重启tidb起不来了

看样子是失败了,~/.tiup/data/hunanhongse/ 下面有 tikv-0,tikv-1,tikv-2 这样的目录,把里面的 tikv.log 都上传上来可以吗

另外这个命令的结果麻烦发一下

是三个tikv都要上传还是只上传可以,我看是有点多的

tail -100 tikv.log 吧

ytes-limit":“256GiB”,“force-consistency-checks”:false,“prop-size-index-distance”:4194304,“prop-keys-index-distance”:40960,“enable-doubly-skiplist”:true,“enable-compaction-guard”:false,“compaction-guard-min-output-file-size”:“8MiB”,“compaction-guard-max-output-file-size”:“128MiB”,“bottommost-level-compression”:“disable”,“bottommost-zstd-compression-dict-size”:0,“bottommost-zstd-compression-sample-size”:0,“titan”:{“min-blob-size”:“1KiB”,“blob-file-compression”:“lz4”,“blob-cache-size”:“0KiB”,“min-gc-batch-size”:“16MiB”,“max-gc-batch-size”:“64MiB”,“discardable-ratio”:0.5,“sample-ratio”:0.1,“merge-small-file-threshold”:“8MiB”,“blob-run-mode”:“normal”,“level-merge”:false,“range-merge”:true,“max-sorted-runs”:20,“gc-merge-rewrite”:false}},“titan”:{“enabled”:false,“dirname”:"",“disable-gc”:false,“max-background-gc”:4,“purge-obsolete-files-period”:“10s”}},“raft-engine”:{“enable”:false,“dir”:"/root/.tiup/data/hunanhongse/tikv-0/data/raft-engine",“recovery-mode”:“tolerate-corrupted-tail-records”,“bytes-per-sync”:“256KiB”,“target-file-size”:“128MiB”,“purge-threshold”:“10GiB”,“cache-limit”:“1GiB”},“security”:{“ca-path”:"",“cert-path”:"",“key-path”:"",“cert-allowed-cn”:[],“redact-info-log”:null,“encryption”:{“data-encryption-method”:“plaintext”,“data-key-rotation-period”:“7d”,“enable-file-dictionary-log”:true,“file-dictionary-rewrite-threshold”:1000000,“master-key”:{“type”:“plaintext”},“previous-master-key”:{“type”:“plaintext”}}},“import”:{“num-threads”:8,“stream-channel-window”:128,“import-mode-timeout”:“10m”},“backup”:{“num-threads”:6,“batch-size”:8,“sst-max-size”:“144MiB”},“pessimistic-txn”:{“wait-for-lock-timeout”:“1s”,“wake-up-delay-duration”:“20ms”,“pipelined”:true},“gc”:{“ratio-threshold”:1.1,“batch-keys”:512,“max-write-bytes-per-sec”:“0KiB”,“enable-compaction-filter”:true,“compaction-filter-skip-version-check”:false},“split”:{“qps-threshold”:3000,“split-balance-score”:0.25,“split-contained-score”:0.5,“detect-times”:10,“sample-num”:20,“sample-threshold”:100,“byte-threshold”:31457280},“cdc”:{“min-ts-interval”:“1s”,“old-value-cache-size”:1024,“hibernate-regions-compatible”:true,“incremental-scan-speed-limit”:“128MiB”}}"]
[2021/06/10 20:05:48.088 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:48.088 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:48.088 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:49.090 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:49.090 +08:00] [INFO] [util.rs:512] [“connecting to PD endpoint”] [endpoints=http://172.16.203.1:2384]
[2021/06/10 20:05:49.091 +08:00] [INFO] [] [“New connected subchannel at 0x7fbcb742f390 for subchannel 0x7fbcbc217f40”]
[2021/06/10 20:05:49.091 +08:00] [INFO] [util.rs:512] [“connecting to PD endpoint”] [endpoints=http://172.16.203.1:2382]
[2021/06/10 20:05:49.092 +08:00] [INFO] [util.rs:627] [“connected to PD member”] [endpoints=http://172.16.203.1:2382]
[2021/06/10 20:05:49.092 +08:00] [INFO] [util.rs:150] [“heartbeat sender and receiver are stale, refreshing …”]
[2021/06/10 20:05:49.092 +08:00] [INFO] [util.rs:181] [“update pd client”] [forworded_host=] [prev_forwarded_host=]
[2021/06/10 20:05:49.092 +08:00] [INFO] [util.rs:313] [“tring to update PD client done”] [spend=1.761527ms]
[2021/06/10 20:05:49.092 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:50.288 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:50.288 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:50.288 +08:00] [INFO] [util.rs:512] [“connecting to PD endpoint”] [endpoints=http://172.16.203.1:2384]
[2021/06/10 20:05:50.289 +08:00] [INFO] [] [“New connected subchannel at 0x7fbcb8a2f510 for subchannel 0x7fbcbc219980”]
[2021/06/10 20:05:50.289 +08:00] [INFO] [util.rs:512] [“connecting to PD endpoint”] [endpoints=http://172.16.203.1:2382]
[2021/06/10 20:05:50.290 +08:00] [INFO] [util.rs:627] [“connected to PD member”] [endpoints=http://172.16.203.1:2382]
[2021/06/10 20:05:50.290 +08:00] [INFO] [util.rs:150] [“heartbeat sender and receiver are stale, refreshing …”]
[2021/06/10 20:05:50.290 +08:00] [INFO] [util.rs:181] [“update pd client”] [forworded_host=] [prev_forwarded_host=]
[2021/06/10 20:05:50.290 +08:00] [INFO] [util.rs:313] [“tring to update PD client done”] [spend=2.284728ms]
[2021/06/10 20:05:50.291 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:50.291 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:50.291 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:51.292 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:51.292 +08:00] [INFO] [util.rs:512] [“connecting to PD endpoint”] [endpoints=http://172.16.203.1:2384]
[2021/06/10 20:05:51.293 +08:00] [INFO] [] [“New connected subchannel at 0x7fbcb7a2e480 for subchannel 0x7fbcbc217840”]
[2021/06/10 20:05:51.294 +08:00] [INFO] [util.rs:512] [“connecting to PD endpoint”] [endpoints=http://172.16.203.1:2382]
[2021/06/10 20:05:51.294 +08:00] [INFO] [util.rs:627] [“connected to PD member”] [endpoints=http://172.16.203.1:2382]
[2021/06/10 20:05:51.294 +08:00] [INFO] [util.rs:150] [“heartbeat sender and receiver are stale, refreshing …”]
[2021/06/10 20:05:51.294 +08:00] [INFO] [util.rs:181] [“update pd client”] [forworded_host=] [prev_forwarded_host=]
[2021/06/10 20:05:51.294 +08:00] [INFO] [util.rs:313] [“tring to update PD client done”] [spend=1.992546ms]
[2021/06/10 20:05:51.294 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:51.295 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:51.295 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:52.296 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:52.296 +08:00] [INFO] [util.rs:512] [“connecting to PD endpoint”] [endpoints=http://172.16.203.1:2384]
[2021/06/10 20:05:52.297 +08:00] [INFO] [] [“New connected subchannel at 0x7fbcb702f300 for subchannel 0x7fbcbc217f40”]
[2021/06/10 20:05:52.298 +08:00] [INFO] [util.rs:512] [“connecting to PD endpoint”] [endpoints=http://172.16.203.1:2382]
[2021/06/10 20:05:52.298 +08:00] [INFO] [util.rs:627] [“connected to PD member”] [endpoints=http://172.16.203.1:2382]
[2021/06/10 20:05:52.298 +08:00] [INFO] [util.rs:150] [“heartbeat sender and receiver are stale, refreshing …”]
[2021/06/10 20:05:52.298 +08:00] [INFO] [util.rs:181] [“update pd client”] [forworded_host=] [prev_forwarded_host=]
[2021/06/10 20:05:52.298 +08:00] [INFO] [util.rs:313] [“tring to update PD client done”] [spend=2.1104ms]
[2021/06/10 20:05:52.299 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:52.299 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:52.299 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:53.300 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:53.300 +08:00] [INFO] [util.rs:512] [“connecting to PD endpoint”] [endpoints=http://172.16.203.1:2384]
[2021/06/10 20:05:53.301 +08:00] [INFO] [] [“New connected subchannel at 0x7fbcb862e480 for subchannel 0x7fbcbc219980”]
[2021/06/10 20:05:53.301 +08:00] [INFO] [util.rs:512] [“connecting to PD endpoint”] [endpoints=http://172.16.203.1:2382]
[2021/06/10 20:05:53.302 +08:00] [INFO] [util.rs:627] [“connected to PD member”] [endpoints=http://172.16.203.1:2382]
[2021/06/10 20:05:53.302 +08:00] [INFO] [util.rs:150] [“heartbeat sender and receiver are stale, refreshing …”]
[2021/06/10 20:05:53.302 +08:00] [INFO] [util.rs:181] [“update pd client”] [forworded_host=] [prev_forwarded_host=]
[2021/06/10 20:05:53.302 +08:00] [INFO] [util.rs:313] [“tring to update PD client done”] [spend=2.02399ms]
[2021/06/10 20:05:53.302 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:53.303 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:53.303 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:54.304 +08:00] [ERROR] [util.rs:392] [“request failed, retry”] [err_code=KV:PD:gRPC] [err=“Grpc(RpcFailure(RpcStatus { status: 2-UNKNOWN, details: Some(”[PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster") }))"]
[2021/06/10 20:05:54.759 +08:00] [FATAL] [lib.rs:465] [“failed to get timestamp from PD: Other(”[components/pd_client/src/util.rs:357]: request retry exceeds limit")"] [backtrace=“stack backtrace:\ 0: tikv_util::set_panic_hook::{{closure}}\ at /home/jenkins/agent/workspace/build_tikv_multi_branch_v5.0.0/tikv/components/tikv_util/src/lib.rs:464\ 1: std::panicking::rust_panic_with_hook\ at /rustc/bc39d4d9c514e5fdb40a5782e6ca08924f979c35//library/std/src/panicking.rs:595\ 2: std::panicking::begin_panic_handler::{{closure}}\ at /rustc/bc39d4d9c514e5fdb40a5782e6ca08924f979c35//library/std/src/panicking.rs:497\ 3: std::sys_common::backtrace::__rust_end_short_backtrace\ at /rustc/bc39d4d9c514e5fdb40a5782e6ca08924f979c35//library/std/src/sys_common/backtrace.rs:141\ 4: rust_begin_unwind\ at /rustc/bc39d4d9c514e5fdb40a5782e6ca08924f979c35//library/std/src/panicking.rs:493\ 5: core::panicking::panic_fmt\ at /rustc/bc39d4d9c514e5fdb40a5782e6ca08924f979c35//library/core/src/panicking.rs:92\ 6: core::option::expect_none_failed\ at /rustc/bc39d4d9c514e5fdb40a5782e6ca08924f979c35//library/core/src/option.rs:1266\ 7: core::result::Result<T,E>::expect\ at /rustc/bc39d4d9c514e5fdb40a5782e6ca08924f979c35/library/core/src/result.rs:929\ cmd::server::TiKVServer::init\ at /home/jenkins/agent/workspace/build_tikv_multi_branch_v5.0.0/tikv/cmd/src/server.rs:223\ cmd::server::run_tikv\ at /home/jenkins/agent/workspace/build_tikv_multi_branch_v5.0.0/tikv/cmd/src/server.rs:138\ 8: tikv_server::main\ at /home/jenkins/agent/workspace/build_tikv_multi_branch_v5.0.0/tikv/cmd/src/bin/tikv-server.rs:181\ 9: core::ops::function::FnOnce::call_once\ at /rustc/bc39d4d9c514e5fdb40a5782e6ca08924f979c35/library/core/src/ops/function.rs:227\ std::sys_common::backtrace::__rust_begin_short_backtrace\ at /rustc/bc39d4d9c514e5fdb40a5782e6ca08924f979c35/library/std/src/sys_common/backtrace.rs:125\ 10: main\ 11: __libc_start_main\ 12: \ ”] [location=cmd/src/server.rs:223] [thread_name=main]

有多少个 tikv 呀,ls ~/.tiup/data/hunanhongse/ 看一下

3个 现在需要把三个都发给你吗?

不用,curl 172.16.203.1:2379/pd/api/v1/stores 这个命令的结果能发下吗?

[root@tidb tikv-0]# curl 172.16.203.1:2379/pd/api/v1/stores
{
“count”: 4,
“stores”: [
{
“store”: {
“id”: 5,
“address”: “172.16.203.1:20161”,
“version”: “5.0.0”,
“status_address”: “172.16.203.1:20181”,
“git_hash”: “7706b9634bd901c9fe8dbe6a556025abbfd0793d”,
“start_timestamp”: 1621504002,
“deploy_path”: “/root/.tiup/components/tikv/v5.0.0”,
“last_heartbeat”: 1622021623843929394,
“state_name”: “Down”
},
“status”: {
“capacity”: “0B”,
“available”: “0B”,
“used_size”: “0B”,
“leader_count”: 0,
“leader_weight”: 1,
“leader_score”: 0,
“leader_size”: 0,
“region_count”: 0,
“region_weight”: 1,
“region_score”: 0,
“region_size”: 0,
“start_ts”: “2021-05-20T17:46:42+08:00”,
“last_heartbeat_ts”: “2021-05-26T17:33:43.843929394+08:00”,
“uptime”: “143h47m1.843929394s”
}
},
{
“store”: {
“id”: 52,
“address”: “172.16.203.1:3930”,
“labels”: [
{
“key”: “engine”,
“value”: “tiflash”
}
],
“version”: “v5.0.0”,
“peer_address”: “172.16.203.1:20170”,
“status_address”: “172.16.203.1:20292”,
“git_hash”: “347fafb0821d8352e24857cc72a8fb7ffd429818”,
“start_timestamp”: 1621504004,
“deploy_path”: “/root/.tiup/components/tiflash/v5.0.0/tiflash”,
“last_heartbeat”: 1623316443141066939,
“state_name”: “Down”
},
“status”: {
“capacity”: “0B”,
“available”: “0B”,
“used_size”: “0B”,
“leader_count”: 0,
“leader_weight”: 1,
“leader_score”: 0,
“leader_size”: 0,
“region_count”: 0,
“region_weight”: 1,
“region_score”: 0,
“region_size”: 0,
“start_ts”: “2021-05-20T17:46:44+08:00”,
“last_heartbeat_ts”: “2021-06-10T17:14:03.141066939+08:00”,
“uptime”: “503h27m19.141066939s”
}
},
{
“store”: {
“id”: 1,
“address”: “172.16.203.1:20160”,
“version”: “5.0.0”,
“status_address”: “172.16.203.1:20180”,
“git_hash”: “7706b9634bd901c9fe8dbe6a556025abbfd0793d”,
“start_timestamp”: 1621504002,
“deploy_path”: “/root/.tiup/components/tikv/v5.0.0”,
“last_heartbeat”: 1623314281335186205,
“state_name”: “Down”
},
“status”: {
“capacity”: “0B”,
“available”: “0B”,
“used_size”: “0B”,
“leader_count”: 0,
“leader_weight”: 1,
“leader_score”: 0,
“leader_size”: 0,
“region_count”: 0,
“region_weight”: 1,
“region_score”: 0,
“region_size”: 0,
“start_ts”: “2021-05-20T17:46:42+08:00”,
“last_heartbeat_ts”: “2021-06-10T16:38:01.335186205+08:00”,
“uptime”: “502h51m19.335186205s”
}
},
{
“store”: {
“id”: 4,
“address”: “172.16.203.1:20162”,
“version”: “5.0.0”,
“status_address”: “172.16.203.1:20182”,
“git_hash”: “7706b9634bd901c9fe8dbe6a556025abbfd0793d”,
“start_timestamp”: 1621504002,
“deploy_path”: “/root/.tiup/components/tikv/v5.0.0”,
“last_heartbeat”: 1623316437622697835,
“state_name”: “Down”
},
“status”: {
“capacity”: “0B”,
“available”: “0B”,
“used_size”: “0B”,
“leader_count”: 0,
“leader_weight”: 1,
“leader_score”: 0,
“leader_size”: 0,
“region_count”: 0,
“region_weight”: 1,
“region_score”: 0,
“region_size”: 0,
“start_ts”: “2021-05-20T17:46:42+08:00”,
“last_heartbeat_ts”: “2021-06-10T17:13:57.622697835+08:00”,
“uptime”: “503h27m15.622697835s”
}
}
]
}

看看这个能否启动

Sorry,是两个横线,被编辑器转义掉了,请把所有参数都换成两个横线 --addr=xxx 这样

-–pd=172.16.203.1:2379 换成 -–pd=172.16.203.1:2379,172.16.203.1:2382,172.16.203.1:2384 再试试

curl 172.16.203.1:2382/pd/api/v1/leader

curl 172.16.203.1:2384/pd/api/v1/leader

用这个命令看看,这两个 PD 都正常吗

服务没有跑起来

那麻烦拿一下 pd-1/pd.log 和 pd-2/pd.log 最后的几行日志