【TiDB 版本】
Cluster version: v5.0.1
【问题描述】
出现了TiKV_GC_can_not_work告警，根据https://docs.pingcap.com/zh/tidb/stable/alert-rules#tikv_gc_can_not_work介绍，找到目标db-server，grep日志出现以下信息。
[2021/05/07 13:37:37.076 +08:00] [WARN] [gc_worker.go:957] [“[gc worker] distributed mode will be used”] [“invalid gc mode”=]
以下是该db-server这一天的日志。
[2021/05/07 13:25:54.962 +08:00] [INFO] [printer.go:33] [“Welcome to TiDB.”] [“Release Version”=v5.0.1] [Edition=Community] [“Git Commit Hash”=1145e347d3469d8e89f88dce86f6926ca44b3cd8] [“Git Branch”=heads/refs/tags/v5.0.1] [“UTC Build Time”=“2021-04-23 05:51:17”] [GoVersion=go1.13] [“Race Enabled”=false] [“Check Table Before Drop”=false] [“TiKV Min Version”=v3.0.0-60965b006877ca7234adaced7890d7b029ed1306][2021/05/07 13:25:54.962 +08:00] [INFO] [trackerRecorder.go:28] [“Mem Profile Tracker started”][2021/05/07 13:25:54.963 +08:00] [INFO] [printer.go:47] [“loaded config”] [config=“{"host":"0.0.0.0","advertise-address":"10.37.77.164","port":4000,"cors":"","store":"tikv","path":"10.37.77.167:2379,10.37.77.170:2379,10.37.77.164:2379","socket":"","lease":"45s","run-ddl":true,"split-table":true,"token-limit":1000,"oom-use-tmp-storage":true,"tmp-storage-path":"/tmp/1000_tidb/MC4wLjAuMDo0MDAwLzAuMC4wLjA6MTAwODA=/tmp-storage","oom-action":"cancel","mem-quota-query":1073741824,"tmp-storage-quota":-1,"enable-streaming":false,"enable-batch-dml":false,"lower-case-table-names":2,"server-version":"","log":{"level":"warn","format":"text","disable-timestamp":null,"enable-timestamp":null,"disable-error-stack":null,"enable-error-stack":null,"file":{"filename":"/tidb/log/tidb-3/tidb.log","max-size":64,"max-days":3,"max-backups":3},"enable-slow-log":true,"slow-query-file":"log/tidb_slow_query.log","slow-threshold":300,"expensive-threshold":10000,"query-log-max-len":4096,"record-plan-in-slow-log":1},"security":{"skip-grant-table":false,"ssl-ca":"","ssl-cert":"","ssl-key":"","require-secure-transport":false,"cluster-ssl-ca":"","cluster-ssl-cert":"","cluster-ssl-key":"","cluster-verify-cn":null,"spilled-file-encryption-method":"plaintext"},"status":{"status-host":"0.0.0.0","metrics-addr":"","status-port":10080,"metrics-interval":15,"report-status":true,"record-db-qps":false},"performance":{"max-procs":0,"max-memory":0,"server-memory-quota":0,"memory-usage-alarm-ratio":0.8,"stats-lease":"3s","stmt-count-limit":5000,"feedback-probability":0,"query-feedback-limit":512,"pseudo-estimate-ratio":0.8,"force-priority":"NO_PRIORITY","bind-info-lease":"3s","txn-entry-size-limit":6291456,"txn-total-size-limit":104857600,"tcp-keep-alive":true,"cross-join":true,"run-auto-analyze":true,"agg-push-down-join":false,"committer-concurrency":128,"max-txn-ttl":3600000,"mem-profile-interval":"1m","index-usage-sync-lease":"0s","gogc":100},"prepared-plan-cache":{"enabled":false,"capacity":100,"memory-guard-ratio":0.1},"opentracing":{"enable":false,"rpc-metrics":false,"sampler":{"type":"const","param":1,"sampling-server-url":"","max-operations":0,"sampling-refresh-interval":0},"reporter":{"queue-size":0,"buffer-flush-interval":0,"log-spans":false,"local-agent-host-port":""}},"proxy-protocol":{"networks":"","header-timeout":5},"pd-client":{"pd-server-timeout":3},"tikv-client":{"grpc-connection-count":4,"grpc-keepalive-time":10,"grpc-keepalive-timeout":3,"grpc-compression-type":"none","commit-timeout":"41s","async-commit":{"keys-limit":256,"total-key-size-limit":4096,"safe-window":2000000000,"allowed-clock-drift":500000000},"max-batch-size":128,"overload-threshold":200,"max-batch-wait-time":0,"batch-wait-size":8,"enable-chunk-rpc":true,"region-cache-ttl":600,"store-limit":0,"store-liveness-timeout":"1s","copr-cache":{"capacity-mb":1000},"ttl-refreshed-txn-size":33554432},"binlog":{"enable":false,"ignore-error":false,"write-timeout":"15s","binlog-socket":"","strategy":"range"},"compatible-kill-query":false,"plugin":{"dir":"","load":""},"pessimistic-txn":{"max-retry-count":256},"check-mb4-value-in-utf8":true,"max-index-length":3072,"index-limit":64,"table-column-count-limit":1017,"graceful-wait-before-shutdown":0,"alter-primary-key":false,"treat-old-version-utf8-as-utf8mb4":true,"enable-table-lock":false,"delay-clean-table-lock":0,"split-region-max-num":1000,"stmt-summary":{"enable":true,"enable-internal-query":false,"max-stmt-count":200,"max-sql-length":4096,"refresh-interval":1800,"history-size":24},"repair-mode":false,"repair-table-list":,"isolation-read":{"engines":["tikv","tiflash","tidb"]},"max-server-connections":0,"new_collations_enabled_on_first_bootstrap":false,"experimental":{"allow-expression-index":false},"enable-collect-execution-info":true,"skip-register-to-dashboard":false,"enable-telemetry":true,"labels":{},"enable-global-index":false,"deprecate-integer-display-length":false,"enable-enum-length-limit":true,"stores-refresh-interval":60,"enable-tcp4-only":false,"enable-forwarding":false}”]
[2021/05/07 13:25:55.003 +08:00] [WARN] [session.go:1429] [“run statement failed”] [schemaVersion=0] [error=“[schema:1049]Unknown database ‘mysql’”] [session=“{
"currDBName": "",
"id": 0,
"status": 2,
"strictMode": true,
"user": null
}”]
[2021/05/07 13:25:56.914 +08:00] [WARN] [manager.go:290] [“is not the owner”] [“owner info”=“[ddl] /tidb/ddl/fg/owner ownerManager 82a4660f-5206-4b49-9c22-6bf381cecda3”][2021/05/07 13:37:37.076 +08:00] [WARN] [gc_worker.go:957] [“[gc worker] distributed mode will be used”] [“invalid gc mode”=]
[2021/05/07 18:45:36.923 +08:00] [ERROR] [client.go:599] [“[pd] getTS error”] [dc-location=global] [error=“[PD:client:ErrClientGetTSO]rpc error: code = Unknown desc = [PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster”]
[2021/05/07 18:45:36.926 +08:00] [ERROR] [base_client.go:166] [“[pd] failed updateMember”] [error=“[PD:client:ErrClientGetLeader]get leader from [http://10.37.77.164:2379 http://10.37.77.167:2379 http://10.37.77.170:2379] error”]
[2021/05/07 18:45:39.848 +08:00] [ERROR] [client.go:599] [“[pd] getTS error”] [dc-location=global] [error=“[PD:client:ErrClientGetTSO]rpc error: code = Unknown desc = [PD:tso:ErrGenerateTimestamp]generate timestamp failed, maximum number of retries exceeded”]
[2021/05/07 18:45:39.848 +08:00] [ERROR] [pd.go:215] [“updateTS error”] [txnScope=global] [error=“rpc error: code = Unknown desc = [PD:tso:ErrGenerateTimestamp]generate timestamp failed, maximum number of retries exceeded”]
[2021/05/07 18:45:41.850 +08:00] [ERROR] [client.go:599] [“[pd] getTS error”] [dc-location=global] [error=“[PD:client:ErrClientGetTSO]rpc error: code = Unknown desc = [PD:tso:ErrGenerateTimestamp]generate timestamp failed, maximum number of retries exceeded”]
[2021/05/07 18:45:41.850 +08:00] [WARN] [txn.go:360] [“wait tso failed”] [error=“rpc error: code = Unknown desc = [PD:tso:ErrGenerateTimestamp]generate timestamp failed, maximum number of retries exceeded”]
[2021/05/07 18:45:41.850 +08:00] [WARN] [txn.go:360] [“wait tso failed”] [error=“rpc error: code = Unknown desc = [PD:tso:ErrGenerateTimestamp]generate timestamp failed, maximum number of retries exceeded”]
[2021/05/07 18:45:43.852 +08:00] [ERROR] [client.go:599] [“[pd] getTS error”] [dc-location=global] [error=“[PD:client:ErrClientGetTSO]rpc error: code = Unknown desc = [PD:tso:ErrGenerateTimestamp]generate timestamp failed, maximum number of retries exceeded”]
[2021/05/07 18:45:43.852 +08:00] [ERROR] [pd.go:215] [“updateTS error”] [txnScope=global] [error=“rpc error: code = Unknown desc = [PD:tso:ErrGenerateTimestamp]generate timestamp failed, maximum number of retries exceeded”]
[2021/05/07 18:45:45.854 +08:00] [ERROR] [client.go:599] [“[pd] getTS error”] [dc-location=global] [error=“[PD:client:ErrClientGetTSO]rpc error: code = Unknown desc = [PD:tso:ErrGenerateTimestamp]generate timestamp failed, maximum number of retries exceeded”]
[2021/05/07 18:45:45.854 +08:00] [ERROR] [pd.go:215] [“updateTS error”] [txnScope=global] [error=“rpc error: code = Unknown desc = [PD:tso:ErrGenerateTimestamp]generate timestamp failed, maximum number of retries exceeded”]
[2021/05/07 18:45:47.856 +08:00] [ERROR] [client.go:599] [“[pd] getTS error”] [dc-location=global] [error=“[PD:client:ErrClientGetTSO]rpc error: code = Unknown desc = [PD:tso:ErrGenerateTimestamp]generate timestamp failed, maximum number of retries exceeded”]
[2021/05/07 18:45:47.856 +08:00] [ERROR] [pd.go:215] [“updateTS error”] [txnScope=global] [error=“rpc error: code = Unknown desc = [PD:tso:ErrGenerateTimestamp]generate timestamp failed, maximum number of retries exceeded”]
[2021/05/07 20:06:17.271 +08:00] [WARN] [2pc.go:1406] [“schemaLeaseChecker is not set for this transaction”] [conn=161] [sessionID=161] [startTS=424775300476043270] [commitTS=424775300476043271]
[2021/05/07 20:06:17.406 +08:00] [WARN] [2pc.go:1406] [“schemaLeaseChecker is not set for this transaction”] [conn=161] [sessionID=161] [startTS=424775300515364871] [commitTS=424775300515364872]
[2021/05/07 20:07:10.638 +08:00] [WARN] [2pc.go:1406] [“schemaLeaseChecker is not set for this transaction”] [conn=223] [sessionID=223] [startTS=424775314474532868] [commitTS=424775314474532869]
[2021/05/07 20:10:00.032 +08:00] [WARN] [2pc.go:1406] [“schemaLeaseChecker is not set for this transaction”] [conn=233] [sessionID=233] [startTS=424775358868357196] [commitTS=424775358881726469]
[2021/05/07 20:49:41.548 +08:00] [WARN] [2pc.go:1406] [“schemaLeaseChecker is not set for this transaction”] [conn=489] [sessionID=489] [startTS=424775983177662469] [commitTS=424775983177662470]
[2021/05/07 21:09:08.192 +08:00] [WARN] [2pc.go:1406] [“schemaLeaseChecker is not set for this transaction”] [conn=309] [sessionID=309] [startTS=424776289007960067] [commitTS=424776289007960068]
[2021/05/07 21:24:56.727 +08:00] [WARN] [2pc.go:1406] [“schemaLeaseChecker is not set for this transaction”] [conn=725] [sessionID=725] [startTS=424776537651544070] [commitTS=424776537651544071]
请问下该如何排查解决。谢谢。
我们是to b型产品，峰值压力都很低，负载最高得机器也就使用30%多左右。