tidb 5.3.0 5.2.3 tiup-cluster deploy 失败

fighter · 2022 年1 月 24 日 04:27

tiup-cluster deploy 失败以下是在目标节点上执行的log 有人知道是什么原因吗？topology.yaml应该没问题，tikv 其他两个节点已经检查通过了

tiup check 已经全部通过

/ssd2/tidb-deploy/tikv-8216/bin/tikv-server --config-check --config=/ssd2/tidb-deploy/tikv-8216/conf/tikv.toml --pd “” --data-dir “/ssd2/tidb-data/tikv-8216”
thread ‘main’ panicked at ‘called Option::unwrap() on a None value’, components/tikv_util/src/sys/cgroup.rs:97:72
stack backtrace:
0: 0x5610aa70bbc3 - std::backtrace_rs::backtrace::libunwind::trace::h99dbb39dca18857d
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/…/…/backtrace/src/backtrace/libunwind.rs:90:5
1: 0x5610aa70bbc3 - std::backtrace_rs::backtrace::trace_unsynchronized::h832861927e9cfedf
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/…/…/backtrace/src/backtrace/mod.rs:66:5
2: 0x5610aa70bbc3 - std::sys_common::backtrace::_print_fmt::h3d18154c77dcf310
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/sys_common/backtrace.rs:67:5
3: 0x5610aa70bbc3 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::he312f4ad5b9bb346
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/sys_common/backtrace.rs:46:22
4: 0x5610a95f861c - core::fmt::write::h9a6d9c74526a6c1b
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/core/src/fmt/mod.rs:1115:17
5: 0x5610aa709f04 - std::io::Write::write_fmt::h6aced00850e8186f
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/io/mod.rs:1665:15
6: 0x5610aa70ac7b - std::sys_common::backtrace::_print::h65d996766de40da4
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/sys_common/backtrace.rs:49:5
7: 0x5610aa70ac7b - std::sys_common::backtrace::print::h40df9727e635f303
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/sys_common/backtrace.rs:36:9
8: 0x5610aa70ac7b - std::panicking::default_hook::{{closure}}::hd2da4327dea91a51
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/panicking.rs:208:50
9: 0x5610aa709b95 - std::panicking::default_hook::h3d55120ad6ada158
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/panicking.rs:225:9
10: 0x5610aa709b95 - std::panicking::rust_panic_with_hook::hf85dd0bb545e3b55
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/panicking.rs:622:17
11: 0x5610aa723ace - std::panicking::begin_panic_handler::{{closure}}::h736ae969434da9fa
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/panicking.rs:517:13
12: 0x5610aa723a6c - std::sys_common::backtrace::__rust_end_short_backtrace::h6133bb80b1d6c3e0
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/sys_common/backtrace.rs:141:18
13: 0x5610aa723a1d - rust_begin_unwind
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/panicking.rs:515:5
14: 0x5610a936ded0 - core::panicking::panic_fmt::hcf5f6d96e1dd7099
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/core/src/panicking.rs:92:14
15: 0x5610a936e02c - core::panicking::panic::hd695e3b1d0dd4ef4
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/core/src/panicking.rs:50:5
16: 0x5610aae2e550 - core::option::Option::unwrap::h1836d1e2c72564be
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/core/src/option.rs:722:21
17: 0x5610aae2e550 - tikv_util::sys::cgroup::CGroupSys::cpuset_cores::h7a14dd62b9a774f4
at /home/jenkins/agent/workspace/optimization-build-tidb-linux-amd/go/src/github.com/pingcap/tikv/components/tikv_util/src/sys/cgroup.rs:97:39
18: 0x5610aae2ddfd - tikv_util::sys::SysQuota::cpu_cores_quota::h09e53662b5700dc3
at /home/jenkins/agent/workspace/optimization-build-tidb-linux-amd/go/src/github.com/pingcap/tikv/components/tikv_util/src/sys/mod.rs:34:28
19: 0x5610aaa46be0 - <tikv::config::UnifiedReadPoolConfig as core::default::Default>::default::hc6d034eed8b94d54
at /home/jenkins/agent/workspace/optimization-build-tidb-linux-amd/go/src/github.com/pingcap/tikv/src/config.rs:1705:23

▽
20: 0x5610aaa46be0 - <tikv::config::ReadPoolConfig as core::default::Default>::default::hf24deecb8cd38e0f
at /home/jenkins/agent/workspace/optimization-build-tidb-linux-amd/go/src/github.com/pingcap/tikv/src/config.rs:2008:5
21: 0x5610aaa46be0 - <tikv::config::TiKvConfig as core::default::Default>::default::h02d60621c02d2ec1
at /home/jenkins/agent/workspace/optimization-build-tidb-linux-amd/go/src/github.com/pingcap/tikv/src/config.rs:2451:23
22: 0x5610aaa829bd - <tikv::config::_IMPL_DESERIALIZE_FOR_TiKvConfig::::deserialize::__Visitor as serde::Visitor>::visit_map::h54dbad94605c2fe2
at /home/jenkins/agent/workspace/optimization-build-tidb-linux-amd/go/src/github.com/pingcap/tikv/src/config.rs:2325:28
23: 0x5610aaa56be3 - <serde_ignored::Wrap<X,F> as serde::Visitor>::visit_map::h3b61444105b0e26a
at /rust/registry/src/github.com-1ecc6299db9ec823/serde_ignored-0.1.2/src/lib.rs:618:9
24: 0x5610aaa56be3 - <&mut toml::Deserializer as serde::Deserializer>::deserialize_any::hb6fa006ab859005c
at /rust/registry/src/github.com-1ecc6299db9ec823/toml-0.5.7/src/de.rs:220:19
25: 0x5610aaa56be3 - <&mut toml::Deserializer as serde::Deserializer>::deserialize_struct::h5e5d016457efe4c4
at /rust/registry/src/github.com-1ecc6299db9ec823/toml-0.5.7/src/de.rs:315:9
26: 0x5610aaa56be3 - <serde_ignored::Deserializer<D,F> as serde::Deserializer>::deserialize_struct::hc59060f3d5eab5f1
at /rust/registry/src/github.com-1ecc6299db9ec823/serde_ignored-0.1.2/src/lib.rs:398:9
27: 0x5610aaa56be3 - tikv::config::_IMPL_DESERIALIZE_FOR_TiKvConfig::::deserialize::h0a017b4a2b68d84c
at /home/jenkins/agent/workspace/optimization-build-tidb-linux-amd/go/src/github.com/pingcap/tikv/src/config.rs:2325:28
28: 0x5610aaa56be3 - serde_ignored::deserialize::h70ecbcb6bfec514b
at /rust/registry/src/github.com-1ecc6299db9ec823/serde_ignored-0.1.2/src/lib.rs5
29: 0x5610aaa56be3 - tikv::config::TiKvConfig::from_file::{{closure}}::h08a1c792e5b9e208
at /home/jenkins/agent/workspace/optimization-build-tidb-linux-amd/go/src/github.com/pingcap/tikv/src/config.rs:2844:17
30: 0x5610aaa56be3 - tikv::config::TiKvConfig::from_file::h6d0fe71e97774b3c
at /home/jenkins/agent/workspace/optimization-build-tidb-linux-amd/go/src/github.com/pingcap/tikv/src/config.rs:2840:9
31: 0x5610a947c6da - tikv_server::main::{{closure}}::h01bb911625797dd5
at /home/jenkins/agent/workspace/optimization-build-tidb-linux-amd/go/src/github.com/pingcap/tikv/cmd/tikv-server/src/main.rs:162:13
32: 0x5610a947c6da - core::option::Option::map_or_else::h23bfa87dcd5678ac
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/core/src/option.rs:873:24
33: 0x5610a947c6da - tikv_server::main::hca52c63b811adddf
at /home/jenkins/agent/workspace/optimization-build-tidb-linux-amd/go/src/github.com/pingcap/tikv/cmd/tikv-server/src/main.rs:159:22
34: 0x5610a9465213 - core::ops::function::FnOnce::call_once::h006ef8e504fd7a25
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/core/src/ops/function.rs:227:5
35: 0x5610a9465213 - std::sys_common::backtrace::__rust_begin_short_backtrace::hf5c8a350ecf9179d
at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/sys_common/backtrace.rs:125:18
36: 0x5610a947ee04 - main
37: 0x7f3c8df663d5 - __libc_start_main
38: 0x5610a9464327 -
39: 0x0 -

Christophe · 2022 年1 月 24 日 05:06

tiup报啥错?

fighter · 2022 年1 月 24 日 07:04

Christophe · 2022 年1 月 24 日 07:20

应该还是配置有问题，解析阶段就挂了

caiyfc · 2022 年1 月 24 日 07:21

根据这个信息，需要查看一下配置文件：/ssd2/tidb-deploy/tikv-8216/conf/tikv.toml，然后看看这个文件的权限

fighter · 2022 年1 月 24 日 07:28

这是topology.yaml 有关kv的配置

fighter · 2022 年1 月 24 日 07:32

-rw-r–r-- 1 tidb tidb 325 Jan 24 15:01 /ssd2/tidb-deploy/tikv-8216/conf/tikv.toml
这个配置文件都是空的

caiyfc · 2022 年1 月 24 日 07:39

感觉是这个文件不太对，你看看另外两个tikv上是不是有同样的文件，给复制过来看看

fighter · 2022 年1 月 24 日 07:45

3台机器都一样刚check过了，这个配置文件里面什么都没有，是正确的吗？

caiyfc · 2022 年1 月 24 日 08:07

没遇到过这个错误，所以在这个阶段我也不太清楚里面是否会有配置信息，但是部署完成以后是肯定有数据的。如果另外两个正常的tikv节点也是没数据的话，那应该就是正常的。报错提示的是这个文件，所以还可以从权限问题进行分析

Win-Man · 2022 年1 月 24 日 12:13

cat /etc/redhat-release
uname -a

看下 3 个 tikv 节点的服务器有没有什么区别？

fighter · 2022 年1 月 24 日 12:37

os一致：CentOS Linux release 7.5.1804 (Core)
内核：
4.14.0_1-0-0-44 #1 SMP Tue Sep 14 17:12:22 CST 2021 x86_64 x86_64 x86_64 GNU/Linux
4.14.0_1-0-0-44 #1 SMP Tue Sep 14 17:12:22 CST 2021 x86_64 x86_64 x86_64 GNU/Linux
4.14.0_1-0-0-45 #2 SMP Tue Oct 19 18:27:28 CST 2021 x86_64 x86_64 x86_64 GNU/Linux

Win-Man · 2022 年1 月 25 日 02:37

在报错的目标机器上，手动执行下这个命令看下会不会报错。
判断一下是 ssh 远程执行命令的问题还是说执行命令本身有问题

fighter · 2022 年1 月 25 日 06:03

请看我帖子内容

qizheng · 2022 年1 月 25 日 06:31

在报错节点试试执行 tiup playground v5.2.3 --mode tikv-slim 能否正常启动 tikv，如果 deploy v5.1.3 或 v5.0.6 版本也有报错么

fighter · 2022 年1 月 25 日 09:05

5.3.0，5.2.3 两个版本的报错跟 deploy报错一致，应该是这台机器哪些配置有问题呢？

fighter · 2022 年1 月 25 日 11:56

v5.1.3 v5.0.6 能成功！大佬能不能分享下原因，感激涕零：）

qizheng · 2022 年1 月 25 日 14:06

这里有个类似报错的问题 Can't run tikv inside container (without cgroups) · Issue #10889 · tikv/tikv · GitHub ，但在 v5.2.2 已经修复了，所以可能是新的问题。

可以贴一下操作系统和 CPU 型号，以及 cat /proc/self/cgroup 和 /proc/self/mountinfo，在 github 提个 issue GitHub - tikv/tikv: Distributed transactional key-value database, originally created to complement TiDB

fighter · 2022 年1 月 26 日 06:27

os:CentOS Linux release 7.5.1804 (Core)
kernel: 4.14.0_1-0-0-44 #1 SMP Tue Sep 14 17:12:22 CST 2021 x86_64 x86_64 x86_64 GNU/Linux
cpu: 96 Intel(R) Xeon(R) Gold 6271C CPU @ 2.60GHz

fighter · 2022 年1 月 26 日 06:30

cat /proc/self/mountinfo
20 69 0:20 / /sys rw,relatime - sysfs sysfs rw
21 69 0:4 / /proc rw,relatime - proc proc rw
22 69 0:6 / /dev rw,nosuid - devtmpfs devtmpfs rw,size=131671036k,nr_inodes=32917759,mode=755
24 22 0:21 / /dev/shm rw - tmpfs tmpfs rw
25 22 0:22 / /dev/pts rw,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=000
26 69 0:23 / /run rw,nosuid,nodev - tmpfs tmpfs rw,mode=755
27 20 0:24 / /sys/fs/cgroup ro,nosuid,nodev,noexec - tmpfs tmpfs ro,mode=755
28 27 0:25 / /sys/fs/cgroup/systemd rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd
69 0 8:2 / / rw,relatime - ext2 /dev/sda2 rw,errors=continue,user_xattr,acl
78 69 8:7 / /tmp rw,relatime - ext2 /dev/sda7 rw,errors=continue,user_xattr,acl
80 69 8:5 / /matrix rw,relatime - ext2 /dev/sda5 rw,errors=continue,user_xattr,acl
82 69 8:6 / /has rw,relatime - ext2 /dev/sda6 rw,errors=continue,user_xattr,acl
85 69 8:4 / /noah rw,relatime - ext2 /dev/sda4 rw,errors=continue,user_xattr,acl
84 69 8:3 / /var rw,relatime - ext2 /dev/sda3 rw,errors=continue,user_xattr,acl
89 69 8:8 / /home rw,relatime - ext4 /dev/sda8 rw,data=ordered
90 69 259:1 / /ssd1 rw,relatime - ext4 /dev/nvme0n1 rw,data=ordered
93 85 0:44 / /noah/modules rw,relatime - tmpfs none rw,size=819200k,mode=755
94 85 0:45 / /noah/tmp rw,relatime - tmpfs none rw,size=409600k,mode=755
18 69 259:2 / /ssd2 rw,noatime - ext4 /dev/nvme1n1p1 rw,nodelalloc,data=ordered