tikv 报错：Some(\"Deadline Exceeded\") }))"]

🪐 TiDB 技术问题 部署&运维管理
1
  • 【TiDB 版本】：v4.0.6
  • 【问题描述】：

[2020/09/30 11:19:58.496 +08:00] [ERROR] [util.rs:301] [“request failed, retry”] [err_code=KV:Unknown] [err=“Grpc(RpcFailure(RpcStatus { status: 4-DEADLINE_EXCEEDED, details: Some("Deadline Exceeded") }))”]

监控显示leader为0，region正常：

image
image1920×937 173 KB

这个要怎么解决啊。

2

拿一下 pd-ctl 执行 store 和 config show all 的结果看下

3

» store
{
“count”: 6,
“stores”: [
{
“store”: {
“id”: 5,
“address”: “172.16.188.150:20160”,
“version”: “4.0.6”,
“status_address”: “172.16.188.150:20180”,
“git_hash”: “ca2475bfbcb49a7c34cf783596acb3edd05fc88f”,
“start_timestamp”: 1601275956,
“deploy_path”: “/data/deploy/tikv-20160/bin”,
“last_heartbeat”: 1601444410988390825,
“state_name”: “Up”
},
“status”: {
“capacity”: “500GiB”,
“available”: “120.2GiB”,
“used_size”: “368.2GiB”,
“leader_count”: 4421,
“leader_weight”: 1,
“leader_score”: 4421,
“leader_size”: 398020,
“region_count”: 12211,
“region_weight”: 1,
“region_score”: 640538927.9096346,
“region_size”: 1105359,
“start_ts”: “2020-09-28T14:52:36+08:00”,
“last_heartbeat_ts”: “2020-09-30T13:40:10.988390825+08:00”,
“uptime”: “46h47m34.988390825s”
}
},
{
“store”: {
“id”: 6,
“address”: “172.16.188.123:20160”,
“version”: “4.0.6”,
“status_address”: “172.16.188.123:20180”,
“git_hash”: “ca2475bfbcb49a7c34cf783596acb3edd05fc88f”,
“start_timestamp”: 1601207903,
“deploy_path”: “/data/deploy/tikv-20160/bin”,
“last_heartbeat”: 1601444406177946969,
“state_name”: “Up”
},
“status”: {
“capacity”: “500GiB”,
“available”: “122.8GiB”,
“used_size”: “322GiB”,
“leader_count”: 4426,
“leader_weight”: 1,
“leader_score”: 4426,
“leader_size”: 393229,
“region_count”: 10950,
“region_weight”: 1,
“region_score”: 585035287.0808649,
“region_size”: 978488,
“start_ts”: “2020-09-27T19:58:23+08:00”,
“last_heartbeat_ts”: “2020-09-30T13:40:06.177946969+08:00”,
“uptime”: “65h41m43.177946969s”
}
},
{
“store”: {
“id”: 7,
“address”: “172.16.188.132:20160”,
“version”: “4.0.6”,
“status_address”: “172.16.188.132:20180”,
“git_hash”: “ca2475bfbcb49a7c34cf783596acb3edd05fc88f”,
“start_timestamp”: 1601207891,
“deploy_path”: “/data/deploy/tikv-20160/bin”,
“last_heartbeat”: 1601444408014556942,
“state_name”: “Up”
},
“status”: {
“capacity”: “500GiB”,
“available”: “122.5GiB”,
“used_size”: “318.3GiB”,
“leader_count”: 4424,
“leader_weight”: 1,
“leader_score”: 4424,
“leader_size”: 404291,
“region_count”: 10772,
“region_weight”: 1,
“region_score”: 590858790.1227074,
“region_size”: 978127,
“start_ts”: “2020-09-27T19:58:11+08:00”,
“last_heartbeat_ts”: “2020-09-30T13:40:08.014556942+08:00”,
“uptime”: “65h41m57.014556942s”
}
},
{
“store”: {
“id”: 8,
“address”: “172.16.188.162:20160”,
“version”: “4.0.6”,
“status_address”: “172.16.188.162:20180”,
“git_hash”: “ca2475bfbcb49a7c34cf783596acb3edd05fc88f”,
“start_timestamp”: 1601207902,
“deploy_path”: “/data/deploy/tikv-20160/bin”,
“last_heartbeat”: 1601444407203203832,
“state_name”: “Up”
},
“status”: {
“capacity”: “500GiB”,
“available”: “122.2GiB”,
“used_size”: “347.2GiB”,
“leader_count”: 4425,
“leader_weight”: 1,
“leader_score”: 4425,
“leader_size”: 396471,
“region_count”: 11621,
“region_weight”: 1,
“region_score”: 596699970.8531256,
“region_size”: 1046381,
“sending_snap_count”: 1,
“start_ts”: “2020-09-27T19:58:22+08:00”,
“last_heartbeat_ts”: “2020-09-30T13:40:07.203203832+08:00”,
“uptime”: “65h41m45.203203832s”
}
},
{
“store”: {
“id”: 1,
“address”: “172.16.188.107:20160”,
“version”: “4.0.6”,
“status_address”: “172.16.188.107:20180”,
“git_hash”: “ca2475bfbcb49a7c34cf783596acb3edd05fc88f”,
“start_timestamp”: 1601207913,
“deploy_path”: “/data/deploy/tikv-20160/bin”,
“last_heartbeat”: 1601444404532983865,
“state_name”: “Up”
},
“status”: {
“capacity”: “500GiB”,
“available”: “123.7GiB”,
“used_size”: “316.8GiB”,
“leader_count”: 4424,
“leader_weight”: 1,
“leader_score”: 4424,
“leader_size”: 391354,
“region_count”: 10740,
“region_weight”: 1,
“region_score”: 565009906.342618,
“region_size”: 940931,
“start_ts”: “2020-09-27T19:58:33+08:00”,
“last_heartbeat_ts”: “2020-09-30T13:40:04.532983865+08:00”,
“uptime”: “65h41m31.532983865s”
}
},
{
“store”: {
“id”: 4,
“address”: “172.16.188.109:20160”,
“version”: “4.0.6”,
“status_address”: “172.16.188.109:20180”,
“git_hash”: “ca2475bfbcb49a7c34cf783596acb3edd05fc88f”,
“start_timestamp”: 1601207917,
“deploy_path”: “/data/deploy/tikv-20160/bin”,
“last_heartbeat”: 1601444408108050521,
“state_name”: “Up”
},
“status”: {
“capacity”: “500GiB”,
“available”: “121.3GiB”,
“used_size”: “304.6GiB”,
“leader_count”: 0,
“leader_weight”: 1,
“leader_score”: 0,
“leader_size”: 0,
“region_count”: 10066,
“region_weight”: 1,
“region_score”: 616364714.1864471,
“region_size”: 901095,
“start_ts”: “2020-09-27T19:58:37+08:00”,
“last_heartbeat_ts”: “2020-09-30T13:40:08.108050521+08:00”,
“uptime”: “65h41m31.108050521s”
}
}
]
}

» config show all
{
“client-urls”: “http://0.0.0.0:2379”,
“peer-urls”: “http://172.16.188.144:2380”,
“advertise-client-urls”: “http://172.16.188.144:2379”,
“advertise-peer-urls”: “http://172.16.188.144:2380”,
“name”: “pd-172.16.188.144-2379”,
“data-dir”: “/data/tidata/pd-2379”,
“force-new-cluster”: false,
“enable-grpc-gateway”: true,
“initial-cluster”: “pd-172.16.188.113-2379=http://172.16.188.113:2380,pd-172.16.188.120-2379=http://172.16.188.120:2380,pd-172.16.188.144-2379=http://172.16.188.144:2380”,
“initial-cluster-state”: “new”,
“initial-cluster-token”: “pd-cluster”,
“join”: “”,
“lease”: 3,
“log”: {
“level”: “”,
“format”: “text”,
“disable-timestamp”: false,
“file”: {
“filename”: “/data/deploy/pd-2379/log/pd.log”,
“max-size”: 300,
“max-days”: 0,
“max-backups”: 0
},
“development”: false,
“disable-caller”: false,
“disable-stacktrace”: false,
“disable-error-verbose”: true,
“sampling”: null
},
“tso-save-interval”: “3s”,
“metric”: {
“job”: “pd-172.16.188.144-2379”,
“address”: “”,
“interval”: “15s”
},
“schedule”: {
“max-snapshot-count”: 3,
“max-pending-peer-count”: 16,
“max-merge-region-size”: 20,
“max-merge-region-keys”: 200000,
“split-merge-interval”: “1h0m0s”,
“enable-one-way-merge”: “false”,
“enable-cross-table-merge”: “false”,
“patrol-region-interval”: “100ms”,
“max-store-down-time”: “30m0s”,
“leader-schedule-limit”: 4,
“leader-schedule-policy”: “count”,
“region-schedule-limit”: 2048,
“replica-schedule-limit”: 64,
“merge-schedule-limit”: 8,
“hot-region-schedule-limit”: 4,
“hot-region-cache-hits-threshold”: 3,
“store-limit”: {
“1”: {
“add-peer”: 15,
“remove-peer”: 15
},
“4”: {
“add-peer”: 15,
“remove-peer”: 15
},
“5”: {
“add-peer”: 15,
“remove-peer”: 15
},
“6”: {
“add-peer”: 15,
“remove-peer”: 15
},
“7”: {
“add-peer”: 15,
“remove-peer”: 15
},
“8”: {
“add-peer”: 15,
“remove-peer”: 15
}
},
“tolerant-size-ratio”: 0,
“low-space-ratio”: 0.8,
“high-space-ratio”: 0.7,
“scheduler-max-waiting-operator”: 5,
“enable-remove-down-replica”: “true”,
“enable-replace-offline-replica”: “true”,
“enable-make-up-replica”: “true”,
“enable-remove-extra-replica”: “true”,
“enable-location-replacement”: “true”,
“enable-debug-metrics”: “false”,
“schedulers-v2”: [
{
“type”: “balance-region”,
“args”: null,
“disable”: false,
“args-payload”: “”
},
{
“type”: “balance-leader”,
“args”: null,
“disable”: false,
“args-payload”: “”
},
{
“type”: “hot-region”,
“args”: null,
“disable”: false,
“args-payload”: “”
},
{
“type”: “label”,
“args”: null,
“disable”: false,
“args-payload”: “”
},
{
“type”: “evict-leader”,
“args”: [
“4”
],
“disable”: false,
“args-payload”: “”
}
],
“schedulers-payload”: {
“balance-hot-region-scheduler”: null,
“balance-leader-scheduler”: {
“name”: “balance-leader-scheduler”,
“ranges”: [
{
“end-key”: “”,
“start-key”: “”
}
]
},
“balance-region-scheduler”: {
“name”: “balance-region-scheduler”,
“ranges”: [
{
“end-key”: “”,
“start-key”: “”
}
]
},
“evict-leader-scheduler”: {
“store-id-ranges”: {
“4”: [
{
“end-key”: “”,
“start-key”: “”
}
]
}
},
“label-scheduler”: {
“name”: “label-scheduler”,
“ranges”: [
{
“end-key”: “”,
“start-key”: “”
}
]
}
},
“store-limit-mode”: “manual”
},
“replication”: {
“max-replicas”: 3,
“location-labels”: “”,
“strictly-match-label”: “false”,
“enable-placement-rules”: “false”
},
“pd-server”: {
“use-region-storage”: “true”,
“max-gap-reset-ts”: “24h0m0s”,
“key-type”: “table”,
“runtime-services”: “”,
“metric-storage”: “http://172.16.188.106:9090”,
“dashboard-address”: “http://172.16.188.144:2379”,
“trace-region-flow”: “false”
},
“cluster-version”: “4.0.6”,
“quota-backend-bytes”: “8GiB”,
“auto-compaction-mode”: “periodic”,
“auto-compaction-retention-v2”: “1h”,
“TickInterval”: “500ms”,
“ElectionInterval”: “3s”,
“PreVote”: true,
“security”: {
“cacert-path”: “”,
“cert-path”: “”,
“key-path”: “”,
“cert-allowed-cn”: null
},
“label-property”: {},
“WarningMsgs”: null,
“DisableStrictReconfigCheck”: false,
“HeartbeatStreamBindInterval”: “1m0s”,
“LeaderPriorityCheckInterval”: “1m0s”,
“dashboard”: {
“tidb-cacert-path”: “”,
“tidb-cert-path”: “”,
“tidb-key-path”: “”,
“public-path-prefix”: “”,
“internal-proxy”: false,
“enable-telemetry”: true,
“enable-experimental”: false
},
“replication-mode”: {
“replication-mode”: “majority”,
“dr-auto-sync”: {
“label-key”: “”,
“primary”: “”,
“dr”: “”,
“primary-replicas”: 0,
“dr-replicas”: 0,
“wait-store-timeout”: “1m0s”,
“wait-sync-timeout”: “1m0s”
}
}
}

4

109 是 id 为4， 看到evict-leader 中有4， 可以试着删除evict-leader

scheduler remove evict-leader-scheduler 4

image

5

此话题已在最后回复的 1 分钟后被自动关闭。不再允许新回复。