为提高效率,提问时请提供以下信息,问题描述清晰可优先响应。
- 【TiDB 版本】:v4.0.5
- 【问题描述】:
由 v4.0.4 升级到 v4.0.5 后,tikv 不均衡
偶数节点,tikv_cnt = 2, replica = 2
晚22时进了版本升级
下图中 2:15 有一次日常全量数据备份
TiKV
PD
为提高效率,提问时请提供以下信息,问题描述清晰可优先响应。
由 v4.0.4 升级到 v4.0.5 后,tikv 不均衡
偶数节点,tikv_cnt = 2, replica = 2
晚22时进了版本升级
下图中 2:15 有一次日常全量数据备份
TiKV
PD
tiup ctl pd config show
Starting component ctl
: /root/.tiup/components/ctl/v4.0.5/ctl pd config show
{
"replication": {
"enable-placement-rules": "true",
"location-labels": "",
"max-replicas": 2,
"strictly-match-label": "false"
},
"schedule": {
"enable-cross-table-merge": "true",
"enable-debug-metrics": "false",
"enable-location-replacement": "true",
"enable-make-up-replica": "true",
"enable-one-way-merge": "false",
"enable-remove-down-replica": "true",
"enable-remove-extra-replica": "true",
"enable-replace-offline-replica": "true",
"high-space-ratio": 0.7,
"hot-region-cache-hits-threshold": 3,
"hot-region-schedule-limit": 4,
"leader-schedule-limit": 4,
"leader-schedule-policy": "count",
"low-space-ratio": 0.8,
"max-merge-region-keys": 200000,
"max-merge-region-size": 20,
"max-pending-peer-count": 16,
"max-snapshot-count": 3,
"max-store-down-time": "30m0s",
"merge-schedule-limit": 8,
"patrol-region-interval": "100ms",
"region-schedule-limit": 2048,
"replica-schedule-limit": 64,
"scheduler-max-waiting-operator": 5,
"split-merge-interval": "1h0m0s",
"store-limit-mode": "manual",
"tolerant-size-ratio": 0
}
}
# tiup ctl pd config show all
Starting component `ctl`: /root/.tiup/components/ctl/v4.0.5/ctl pd config show all
{
"client-urls": "http://0.0.0.0:2379",
"peer-urls": "http://172.19.20.139:2380",
"advertise-client-urls": "http://172.19.20.139:2379",
"advertise-peer-urls": "http://172.19.20.139:2380",
"name": "pd-172.19.20.139-2379",
"data-dir": "/data/tidb/data/pd-2379",
"force-new-cluster": false,
"enable-grpc-gateway": true,
"initial-cluster": "pd-172.19.20.138-2379=http://172.19.20.138:2380,pd-172.19.20.139-2379=http://172.19.20.139:2380",
"initial-cluster-state": "new",
"join": "",
"lease": 3,
"log": {
"level": "",
"format": "text",
"disable-timestamp": false,
"file": {
"filename": "/data/tidb/deploy/pd-2379/log/pd.log",
"max-size": 300,
"max-days": 0,
"max-backups": 0
},
"development": false,
"disable-caller": false,
"disable-stacktrace": false,
"disable-error-verbose": true,
"sampling": null
},
"tso-save-interval": "3s",
"metric": {
"job": "pd-172.19.20.139-2379",
"address": "",
"interval": "15s"
},
"schedule": {
"max-snapshot-count": 3,
"max-pending-peer-count": 16,
"max-merge-region-size": 20,
"max-merge-region-keys": 200000,
"split-merge-interval": "1h0m0s",
"enable-one-way-merge": "false",
"enable-cross-table-merge": "true",
"patrol-region-interval": "100ms",
"max-store-down-time": "30m0s",
"leader-schedule-limit": 4,
"leader-schedule-policy": "count",
"region-schedule-limit": 2048,
"replica-schedule-limit": 64,
"merge-schedule-limit": 8,
"hot-region-schedule-limit": 4,
"hot-region-cache-hits-threshold": 3,
"store-limit": {
"1": {
"add-peer": 15,
"remove-peer": 15
},
"18157": {
"add-peer": 15,
"remove-peer": 15
},
"2": {
"add-peer": 15,
"remove-peer": 15
},
"68": {
"add-peer": 30,
"remove-peer": 30
}
},
"tolerant-size-ratio": 0,
"low-space-ratio": 0.8,
"high-space-ratio": 0.7,
"scheduler-max-waiting-operator": 5,
"enable-remove-down-replica": "true",
"enable-replace-offline-replica": "true",
"enable-make-up-replica": "true",
"enable-remove-extra-replica": "true",
"enable-location-replacement": "true",
"enable-debug-metrics": "false",
"schedulers-v2": [
{
"type": "balance-region",
"args": null,
"disable": false,
"args-payload": ""
},
{
"type": "balance-leader",
"args": null,
"disable": false,
"args-payload": ""
},
{
"type": "hot-region",
"args": null,
"disable": false,
"args-payload": ""
},
{
"type": "label",
"args": null,
"disable": false,
"args-payload": ""
},
{
"type": "evict-leader",
"args": [
"2"
],
"disable": false,
"args-payload": ""
}
],
"schedulers-payload": {
"balance-hot-region-scheduler": null,
"balance-leader-scheduler": {
"name": "balance-leader-scheduler",
"ranges": [
{
"end-key": "",
"start-key": ""
}
]
},
"balance-region-scheduler": {
"name": "balance-region-scheduler",
"ranges": [
{
"end-key": "",
"start-key": ""
}
]
},
"evict-leader-scheduler": {
"store-id-ranges": {
"2": [
{
"end-key": "",
"start-key": ""
}
]
}
},
"label-scheduler": {
"name": "label-scheduler",
"ranges": [
{
"end-key": "",
"start-key": ""
}
]
}
},
"store-limit-mode": "manual"
},
"replication": {
"max-replicas": 2,
"location-labels": "",
"strictly-match-label": "false",
"enable-placement-rules": "true"
},
"pd-server": {
"use-region-storage": "true",
"max-gap-reset-ts": "24h0m0s",
"key-type": "table",
"runtime-services": "",
"metric-storage": "http://172.19.20.139:9090",
"dashboard-address": "http://172.19.20.139:2379",
"trace-region-flow": "true"
},
"cluster-version": "4.0.5",
"quota-backend-bytes": "8GiB",
"auto-compaction-mode": "periodic",
"auto-compaction-retention-v2": "1h",
"TickInterval": "500ms",
"ElectionInterval": "3s",
"PreVote": true,
"security": {
"cacert-path": "",
"cert-path": "",
"key-path": "",
"cert-allowed-cn": null
},
"label-property": {},
"WarningMsgs": null,
"DisableStrictReconfigCheck": false,
"HeartbeatStreamBindInterval": "1m0s",
"LeaderPriorityCheckInterval": "1m0s",
"dashboard": {
"tidb-cacert-path": "",
"tidb-cert-path": "",
"tidb-key-path": "",
"public-path-prefix": "",
"internal-proxy": false,
"enable-telemetry": true
},
"replication-mode": {
"replication-mode": "majority",
"dr-auto-sync": {
"label-key": "",
"primary": "",
"dr": "",
"primary-replicas": 0,
"dr-replicas": 0,
"wait-store-timeout": "1m0s",
"wait-sync-timeout": "1m0s"
}
}
}
# tiup ctl pd store show all
Starting component `ctl`: /root/.tiup/components/ctl/v4.0.5/ctl pd store show all
{
"count": 4,
"stores": [
{
"store": {
"id": 2,
"address": "172.19.20.139:20160",
"version": "4.0.5",
"status_address": "172.19.20.139:20180",
"git_hash": "f39927a3529d40a6bb4e6c54854a94fdac996e92",
"start_timestamp": 1599055311,
"deploy_path": "/data/tidb/deploy/tikv-20160/bin",
"last_heartbeat": 1599108179304313172,
"state_name": "Up"
},
"status": {
"capacity": "196.7GiB",
"available": "147.5GiB",
"used_size": "29.15GiB",
"leader_count": 0,
"leader_weight": 1,
"leader_score": 0,
"leader_size": 0,
"region_count": 1232,
"region_weight": 1,
"region_score": 107999,
"region_size": 107999,
"start_ts": "2020-09-02T22:01:51+08:00",
"last_heartbeat_ts": "2020-09-03T12:42:59.304313172+08:00",
"uptime": "14h41m8.304313172s"
}
},
{
"store": {
"id": 68,
"address": "172.19.20.138:3930",
"labels": [
{
"key": "engine",
"value": "tiflash"
}
],
"version": "v4.0.5",
"peer_address": "172.19.20.138:20170",
"status_address": "172.19.20.138:20292",
"git_hash": "47883266d561b299eb900cd04e32a816fc621a69",
"start_timestamp": 1599055245,
"deploy_path": "/data/tidb/deploy/tiflash-9000/bin/tiflash",
"last_heartbeat": 1599108182884884167,
"state_name": "Up"
},
"status": {
"capacity": "196.7GiB",
"available": "143.3GiB",
"used_size": "2.207GiB",
"leader_count": 0,
"leader_weight": 1,
"leader_score": 0,
"leader_size": 0,
"region_count": 65,
"region_weight": 1,
"region_score": 5446,
"region_size": 5446,
"start_ts": "2020-09-02T22:00:45+08:00",
"last_heartbeat_ts": "2020-09-03T12:43:02.884884167+08:00",
"uptime": "14h42m17.884884167s"
}
},
{
"store": {
"id": 18157,
"address": "172.19.20.139:3930",
"labels": [
{
"key": "engine",
"value": "tiflash"
}
],
"version": "v4.0.5",
"peer_address": "172.19.20.139:20170",
"status_address": "172.19.20.139:20292",
"git_hash": "47883266d561b299eb900cd04e32a816fc621a69",
"start_timestamp": 1599055249,
"deploy_path": "/data/tidb/deploy/tiflash-9000/bin/tiflash",
"last_heartbeat": 1599108186633336992,
"state_name": "Up"
},
"status": {
"capacity": "196.7GiB",
"available": "138.5GiB",
"used_size": "1.278GiB",
"leader_count": 0,
"leader_weight": 1,
"leader_score": 0,
"leader_size": 0,
"region_count": 56,
"region_weight": 1,
"region_score": 4535,
"region_size": 4535,
"start_ts": "2020-09-02T22:00:49+08:00",
"last_heartbeat_ts": "2020-09-03T12:43:06.633336992+08:00",
"uptime": "14h42m17.633336992s"
}
},
{
"store": {
"id": 1,
"address": "172.19.20.138:20160",
"version": "4.0.5",
"status_address": "172.19.20.138:20180",
"git_hash": "f39927a3529d40a6bb4e6c54854a94fdac996e92",
"start_timestamp": 1599055270,
"deploy_path": "/data/tidb/deploy/tikv-20160/bin",
"last_heartbeat": 1599108188443281053,
"state_name": "Up"
},
"status": {
"capacity": "196.7GiB",
"available": "152.3GiB",
"used_size": "29.41GiB",
"leader_count": 1232,
"leader_weight": 1,
"leader_score": 1232,
"leader_size": 107999,
"region_count": 1232,
"region_weight": 1,
"region_score": 107999,
"region_size": 107999,
"start_ts": "2020-09-02T22:01:10+08:00",
"last_heartbeat_ts": "2020-09-03T12:43:08.443281053+08:00",
"uptime": "14h41m58.443281053s"
}
}
]
}
木有有尝试执行以下指令:
tiup ctl pd scheduler remove evict-leader-scheduler
您好,查看配置,是否执行过 scheduler add evict-leader-scheduler 2,139 是 store id 2 导致所有leader 迁出
remove 试试
未手动执行过相关指令,应该是使用tiup升级时自动添加的
目前已移除,集群缓慢均衡中
好的,问题我们反馈下,多谢。
此话题已在最后回复的 1 分钟后被自动关闭。不再允许新回复。