k8s部署tidb3.0 三个工作节点 其中一个节点宕机后。现在恢复后 tikv tipd如何恢复

[root@master ~]# kubectl get po -n denali-tidb
NAME READY STATUS RESTARTS AGE
tidb-cluster-discovery-84d6cf454c-2s7mq 1/1 Running 2 24h
tidb-cluster-monitor-6c6645d89f-d8jc8 3/3 Running 0 20h
tidb-cluster-pd-0 1/1 Running 2 369d
tidb-cluster-pd-1 1/1 Running 0 21h
tidb-cluster-pd-2 0/1 Pending 0 5m34s
tidb-cluster-pd-3 0/1 Pending 0 20h
tidb-cluster-tidb-0 2/2 Running 0 21h
tidb-cluster-tidb-1 2/2 Running 4 36d
tidb-cluster-tikv-0 1/1 Running 0 21h
tidb-cluster-tikv-1 1/1 Running 2 369d
tidb-cluster-tikv-2 0/1 CrashLoopBackOff 5 6m39s
tidb-cluster-tikv-3 0/1 Pending 0 20h
tidb-controller-manager-5b65b7c6c-sw846 1/1 Running 2 20h
tidb-scheduler-54f8f8d699-r2rhm 2/2 Running 2 21h
[root@master ~]#

[2020/11/11 06:47:22.653 +00:00] [WARN] [store.rs:1116] [“set thread priority for raftstore failed”] [error=“Os { code: 13, kind: PermissionDenied, message: “Permission denied” }”]
[2020/11/11 06:47:22.654 +00:00] [INFO] [node.rs:159] [“put store to PD”] [store=“Store { id: 20001, address: “tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160”, state: Up, labels: [], version: “4.0.0-alpha”, unknown_fields: UnknownFields { fields: N
one }, cached_size: CachedSize { size: 0 } }”][2020/11/11 06:47:22.657 +00:00] [ERROR] [util.rs:326] [“request failed”] [err="Grpc(RpcFailure(RpcStatus { status: RpcStatusCode(2), details: Some("duplicated store address: id:20001 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” vers
ion:\“4.0.0-alpha\” , already registered by id:41 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” labels:<key:\“host\” value:\“node2\” > version:\“4.0.0-alpha\” “) }))”][2020/11/11 06:47:22.658 +00:00] [ERROR] [util.rs:326] [“request failed”] [err="Grpc(RpcFailure(RpcStatus { status: RpcStatusCode(2), details: Some("duplicated store address: id:20001 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” vers
ion:\“4.0.0-alpha\” , already registered by id:41 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” labels:<key:\“host\” value:\“node2\” > version:\“4.0.0-alpha\” “) }))”][2020/11/11 06:47:22.658 +00:00] [ERROR] [util.rs:326] [“request failed”] [err="Grpc(RpcFailure(RpcStatus { status: RpcStatusCode(2), details: Some("duplicated store address: id:20001 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” vers
ion:\“4.0.0-alpha\” , already registered by id:41 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” labels:<key:\“host\” value:\“node2\” > version:\“4.0.0-alpha\” “) }))”][2020/11/11 06:47:22.659 +00:00] [ERROR] [util.rs:326] [“request failed”] [err="Grpc(RpcFailure(RpcStatus { status: RpcStatusCode(2), details: Some("duplicated store address: id:20001 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” vers
ion:\“4.0.0-alpha\” , already registered by id:41 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” labels:<key:\“host\” value:\“node2\” > version:\“4.0.0-alpha\” “) }))”][2020/11/11 06:47:22.660 +00:00] [ERROR] [util.rs:326] [“request failed”] [err="Grpc(RpcFailure(RpcStatus { status: RpcStatusCode(2), details: Some("duplicated store address: id:20001 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” vers
ion:\“4.0.0-alpha\” , already registered by id:41 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” labels:<key:\“host\” value:\“node2\” > version:\“4.0.0-alpha\” “) }))”][2020/11/11 06:47:22.661 +00:00] [ERROR] [util.rs:326] [“request failed”] [err="Grpc(RpcFailure(RpcStatus { status: RpcStatusCode(2), details: Some("duplicated store address: id:20001 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” vers
ion:\“4.0.0-alpha\” , already registered by id:41 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” labels:<key:\“host\” value:\“node2\” > version:\“4.0.0-alpha\” “) }))”][2020/11/11 06:47:22.662 +00:00] [ERROR] [util.rs:326] [“request failed”] [err="Grpc(RpcFailure(RpcStatus { status: RpcStatusCode(2), details: Some("duplicated store address: id:20001 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” vers
ion:\“4.0.0-alpha\” , already registered by id:41 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” labels:<key:\“host\” value:\“node2\” > version:\“4.0.0-alpha\” “) }))”][2020/11/11 06:47:22.663 +00:00] [ERROR] [util.rs:326] [“request failed”] [err="Grpc(RpcFailure(RpcStatus { status: RpcStatusCode(2), details: Some("duplicated store address: id:20001 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” vers
ion:\“4.0.0-alpha\” , already registered by id:41 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” labels:<key:\“host\” value:\“node2\” > version:\“4.0.0-alpha\” “) }))”][2020/11/11 06:47:22.663 +00:00] [ERROR] [util.rs:326] [“request failed”] [err="Grpc(RpcFailure(RpcStatus { status: RpcStatusCode(2), details: Some("duplicated store address: id:20001 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” vers
ion:\“4.0.0-alpha\” , already registered by id:41 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” labels:<key:\“host\” value:\“node2\” > version:\“4.0.0-alpha\” “) }))”][2020/11/11 06:47:22.664 +00:00] [ERROR] [util.rs:326] [“request failed”] [err=“Grpc(RpcFailure(RpcStatus { status: RpcStatusCode(2), details: Some(“duplicated store address: id:20001 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” vers
ion:\“4.0.0-alpha\” , already registered by id:41 address:\“tidb-cluster-tikv-2.tidb-cluster-tikv-peer.denali-tidb.svc:20160\” labels:<key:\“host\” value:\“node2\” > version:\“4.0.0-alpha\” “) }))”][2020/11/11 06:47:22.664 +00:00] [FATAL] [server.rs:273] [“failed to start node: Other(”[components/pd_client/src/util.rs:334]: fail to request”)”]

这个报错很常见,可以参考 Tikv启动不了