我刚才尝试在每个健康的kv节点上执行./tikv-ctl --db /home/tidb/deploy/data/db unsafe-recover remove-fail-stores -s 38546296 --all-regions,试图删除已经强制下线节点的regions,然后重启集群。qi zh其中kv节点240的日志如图
store如图
store
{
“count”: 5,
“stores”: [
{
“store”: {
“id”: 38833310,
“address”: “10.12.5.147:20160”,
“version”: “4.0.6”,
“status_address”: “10.12.5.147:20180”,
“git_hash”: “ca2475bfbcb49a7c34cf783596acb3edd05fc88f”,
“start_timestamp”: 1617248660,
“deploy_path”: “/home/tidb/deploy/bin”,
“last_heartbeat”: 1617278938753846245,
“state_name”: “Up”
},
“status”: {
“capacity”: “5.952TiB”,
“available”: “3.685TiB”,
“used_size”: “2.078TiB”,
“leader_count”: 3456,
“leader_weight”: 2,
“leader_score”: 1728,
“leader_size”: 210224,
“region_count”: 98428,
“region_weight”: 2,
“region_score”: 393091.5,
“region_size”: 786183,
“sending_snap_count”: 2,
“receiving_snap_count”: 1,
“start_ts”: “2021-04-01T03:44:20Z”,
“last_heartbeat_ts”: “2021-04-01T12:08:58.753846245Z”,
“uptime”: “8h24m38.753846245s”
}
},
{
“store”: {
“id”: 256634687,
“address”: “10.12.5.12:20160”,
“version”: “4.0.6”,
“status_address”: “10.12.5.12:20180”,
“git_hash”: “ca2475bfbcb49a7c34cf783596acb3edd05fc88f”,
“start_timestamp”: 1617248849,
“deploy_path”: “/home/tidb/deploy/bin”,
“last_heartbeat”: 1617277699444568860,
“state_name”: “Disconnected”
},
“status”: {
“capacity”: “2.952TiB”,
“available”: “2.35TiB”,
“used_size”: “604.9GiB”,
“leader_count”: 0,
“leader_weight”: 1,
“leader_score”: 0,
“leader_size”: 0,
“region_count”: 26342,
“region_weight”: 1,
“region_score”: 18281,
“region_size”: 18281,
“start_ts”: “2021-04-01T03:47:29Z”,
“last_heartbeat_ts”: “2021-04-01T11:48:19.44456886Z”,
“uptime”: “8h0m50.44456886s”
}
},
{
“store”: {
“id”: 24478148,
“address”: “10.12.5.236:20160”,
“version”: “4.0.6”,
“status_address”: “10.12.5.236:20180”,
“git_hash”: “ca2475bfbcb49a7c34cf783596acb3edd05fc88f”,
“start_timestamp”: 1617248731,
“deploy_path”: “/home/tidb/deploy/bin”,
“last_heartbeat”: 1617278932097802241,
“state_name”: “Up”
},
“status”: {
“capacity”: “5.952TiB”,
“available”: “2.803TiB”,
“used_size”: “2.033TiB”,
“leader_count”: 12024,
“leader_weight”: 2,
“leader_score”: 6012,
“leader_size”: 720637,
“region_count”: 101746,
“region_weight”: 2,
“region_score”: 416939,
“region_size”: 833878,
“sending_snap_count”: 1,
“start_ts”: “2021-04-01T03:45:31Z”,
“last_heartbeat_ts”: “2021-04-01T12:08:52.097802241Z”,
“uptime”: “8h23m21.097802241s”
}
},
{
“store”: {
“id”: 24480822,
“address”: “10.12.5.239:20160”,
“version”: “4.0.6”,
“status_address”: “10.12.5.239:20180”,
“git_hash”: “ca2475bfbcb49a7c34cf783596acb3edd05fc88f”,
“start_timestamp”: 1617248780,
“deploy_path”: “/home/tidb/deploy/bin”,
“last_heartbeat”: 1617277712905685930,
“state_name”: “Disconnected”
},
“status”: {
“capacity”: “5.952TiB”,
“available”: “3.861TiB”,
“used_size”: “2TiB”,
“leader_count”: 3,
“leader_weight”: 2,
“leader_score”: 1.5,
“leader_size”: 134,
“region_count”: 88650,
“region_weight”: 2,
“region_score”: 88273.5,
“region_size”: 176547,
“start_ts”: “2021-04-01T03:46:20Z”,
“last_heartbeat_ts”: “2021-04-01T11:48:32.90568593Z”,
“uptime”: “8h2m12.90568593s”
}
},
{
“store”: {
“id”: 24590972,
“address”: “10.12.5.240:20160”,
“version”: “4.0.6”,
“status_address”: “10.12.5.240:20180”,
“git_hash”: “ca2475bfbcb49a7c34cf783596acb3edd05fc88f”,
“start_timestamp”: 1617248859,
“deploy_path”: “/home/tidb/deploy/bin”,
“last_heartbeat”: 1617277713365695442,
“state_name”: “Disconnected”
},
“status”: {
“capacity”: “5.952TiB”,
“available”: “4.029TiB”,
“used_size”: “1.828TiB”,
“leader_count”: 0,
“leader_weight”: 2,
“leader_score”: 0,
“leader_size”: 0,
“region_count”: 85146,
“region_weight”: 2,
“region_score”: 340616.5,
“region_size”: 681233,
“start_ts”: “2021-04-01T03:47:39Z”,
“last_heartbeat_ts”: “2021-04-01T11:48:33.365695442Z”,
“uptime”: “8h0m54.365695442s”
}
}
]
}