tidb突然出现异常

为提高效率,提问时请提供以下信息,问题描述清晰可优先响应。

  • 【TiDB 版本】:
  • 【问题描述】:
    kubectl logs tidb-cluster-tidb-0 -n denali-tidb -c tidb
    start tidb-server …
    /tidb-server --store=tikv --host=0.0.0.0 --path=tidb-cluster-pd:2379 --config=/etc/tidb/tidb.toml
    –log-slow-query=/var/log/tidb/slowlog
    [2020/01/13 08:44:41.986 +00:00] [INFO] [printer.go:41] [“Welcome to TiDB.”] [“Release Version”=v3.0.1] [“Git Commit Hash”=9e4e8da3c58c65123db5f26409759fe1847529f8] [“Git Branch”=HEAD] [“UTC Build Time”=“2019-07-16 01:03:40”] [GoVersion=“go version go1.12 linux/amd64”] [“Race Enabled”=false] [“Check Table Before Drop”=false] [“TiKV Min Version”=2.1.0-alpha.1-ff3dd160846b7d1aed9079c389fc188f7f5ea13e]
    [2020/01/13 08:44:41.988 +00:00] [INFO] [printer.go:54] [“loaded config”] [config=“{"host":"0.0.0.0","advertise-address":"","port":4000,"cors":"","store":"tikv","path":"tidb-cluster-pd:2379","socket":"","lease":"45s","run-ddl":true,"split-table":true,"token-limit":1000,"oom-action":"log","mem-quota-query":34359738368,"enable-streaming":false,"txn-local-latches":{"enabled":true,"capacity":2048000},"lower-case-table-names":2,"log":{"level":"info","format":"text","disable-timestamp":false,"file":{"filename":"","log-rotate":true,"max-size":300,"max-days":0,"max-backups":0},"slow-query-file":"/var/log/tidb/slowlog","slow-threshold":300,"expensive-threshold":10000,"query-log-max-len":2048},"security":{"skip-grant-table":false,"ssl-ca":"","ssl-cert":"","ssl-key":"","cluster-ssl-ca":"","cluster-ssl-cert":"","cluster-ssl-key":""},"status":{"report-status":true,"status-host":"0.0.0.0","status-port":10080,"metrics-addr":"","metrics-interval":15,"record-db-qps":false},"performance":{"max-procs":0,"max-memory":0,"tcp-keep-alive":true,"cross-join":true,"stats-lease":"3s","run-auto-analyze":true,"stmt-count-limit":5000,"feedback-probability":0.05,"query-feedback-limit":1024,"pseudo-estimate-ratio":0.8,"force-priority":"NO_PRIORITY","bind-info-lease":"3s"},"prepared-plan-cache":{"enabled":false,"capacity":100,"memory-guard-ratio":0.1},"opentracing":{"enable":false,"sampler":{"type":"const","param":1,"sampling-server-url":"","max-operations":0,"sampling-refresh-interval":0},"reporter":{"queue-size":0,"buffer-flush-interval":0,"log-spans":false,"local-agent-host-port":""},"rpc-metrics":false},"proxy-protocol":{"networks":"","header-timeout":5},"tikv-client":{"grpc-connection-count":16,"grpc-keepalive-time":10,"grpc-keepalive-timeout":3,"commit-timeout":"41s","max-txn-time-use":590,"max-batch-size":128,"overload-threshold":200,"max-batch-wait-time":0,"batch-wait-size":8},"binlog":{"enable":false,"write-timeout":"15s","ignore-error":false,"binlog-socket":"","strategy":"range"},"compatible-kill-query":false,"plugin":{"dir":"","load":""},"pessimistic-txn":{"enable":false,"default":false,"max-retry-count":256,"ttl":"30s"},"check-mb4-value-in-utf8":true,"treat-old-version-utf8-as-utf8mb4":true}”]
    [2020/01/13 08:44:41.988 +00:00] [INFO] [main.go:250] [“disable Prometheus push client”]
    [2020/01/13 08:44:41.988 +00:00] [INFO] [store.go:69] [“new store”] [path=tikv://tidb-cluster-pd:2379]
    [2020/01/13 08:44:41.988 +00:00] [INFO] [client.go:148] [“[pd] create pd client with endpoints”] [pd-address=“[tidb-cluster-pd:2379]”]
    [2020/01/13 08:44:41.988 +00:00] [INFO] [systime_mon.go:26] [“start system time monitor”]
    [2020/01/13 08:44:41.992 +00:00] [ERROR] [client.go:204] [“[pd] failed to get cluster id”] [error=“rpc error: code = Unavailable desc = all SubConns are in TransientFailure, latest connection error: connection error: desc = "transport: Error while dialing dial tcp 10.96.67.13:2379: connect: connection refused"”] [errorVerbose=“rpc error: code = Unavailable desc = all SubConns are in TransientFailure, latest connection error: connection error: desc = "transport: Error while dialing dial tcp 10.96.67.13:2379: connect: connection refused"
    github.com/pingcap/pd/client.(*client).getMembers
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:239
    github.com/pingcap/pd/client.(*client).initClusterID
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:201
    github.com/pingcap/pd/client.(*client).initRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:188
    github.com/pingcap/pd/client.NewClient
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:161
    github.com/pingcap/tidb/store/tikv.Driver.Open
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/kv.go:78
    github.com/pingcap/tidb/store.newStoreWithRetry.func1
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:70
    github.com/pingcap/tidb/util.RunWithRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/util/misc.go:46
    github.com/pingcap/tidb/store.newStoreWithRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:68
    github.com/pingcap/tidb/store.New
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:52
    main.createStoreAndDomain
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:202
    main.main
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:171
    runtime.main
    \t/usr/local/go/src/runtime/proc.go:200
    runtime.goexit
    \t/usr/local/go/src/runtime/asm_amd64.s:1337”] [stack=“github.com/pingcap/log.Error
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/log@v0.0.0-20190307075452-bd41d9273596/global.go:42
    github.com/pingcap/pd/client.(*client).initClusterID
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:204
    github.com/pingcap/pd/client.(*client).initRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:188
    github.com/pingcap/pd/client.NewClient
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:161
    github.com/pingcap/tidb/store/tikv.Driver.Open
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/kv.go:78
    github.com/pingcap/tidb/store.newStoreWithRetry.func1
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:70
    github.com/pingcap/tidb/util.RunWithRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/util/misc.go:46
    github.com/pingcap/tidb/store.newStoreWithRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:68
    github.com/pingcap/tidb/store.New
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:52
    main.createStoreAndDomain
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:202
    main.main
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:171
    runtime.main
    \t/usr/local/go/src/runtime/proc.go:200”]
    [2020/01/13 08:44:42.992 +00:00] [ERROR] [client.go:204] [“[pd] failed to get cluster id”] [error=“rpc error: code = Unavailable desc = all SubConns are in TransientFailure, latest connection error: connection error: desc = "transport: Error while dialing dial tcp 10.96.67.13:2379: connect: connection refused"”] [errorVerbose=“rpc error: code = Unavailable desc = all SubConns are in TransientFailure, latest connection error: connection error: desc = "transport: Error while dialing dial tcp 10.96.67.13:2379: connect: connection refused"
    github.com/pingcap/pd/client.(*client).getMembers
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:239
    github.com/pingcap/pd/client.(*client).initClusterID
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:201
    github.com/pingcap/pd/client.(*client).initRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:188
    github.com/pingcap/pd/client.NewClient
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:161
    github.com/pingcap/tidb/store/tikv.Driver.Open
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/kv.go:78
    github.com/pingcap/tidb/store.newStoreWithRetry.func1
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:70
    github.com/pingcap/tidb/util.RunWithRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/util/misc.go:46
    github.com/pingcap/tidb/store.newStoreWithRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:68
    github.com/pingcap/tidb/store.New
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:52
    main.createStoreAndDomain
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:202
    main.main
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:171
    runtime.main
    \t/usr/local/go/src/runtime/proc.go:200
    runtime.goexit
    \t/usr/local/go/src/runtime/asm_amd64.s:1337”] [stack=“github.com/pingcap/log.Error
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/log@v0.0.0-20190307075452-bd41d9273596/global.go:42
    github.com/pingcap/pd/client.(*client).initClusterID
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:204
    github.com/pingcap/pd/client.(*client).initRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:188
    github.com/pingcap/pd/client.NewClient
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:161
    github.com/pingcap/tidb/store/tikv.Driver.Open
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/kv.go:78
    github.com/pingcap/tidb/store.newStoreWithRetry.func1
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:70
    github.com/pingcap/tidb/util.RunWithRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/util/misc.go:46
    github.com/pingcap/tidb/store.newStoreWithRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:68
    github.com/pingcap/tidb/store.New
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:52
    main.createStoreAndDomain
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:202
    main.main
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:171
    runtime.main
    \t/usr/local/go/src/runtime/proc.go:200”]
    [2020/01/13 08:44:43.993 +00:00] [ERROR] [client.go:204] [“[pd] failed to get cluster id”] [error=“rpc error: code = Unavailable desc = all SubConns are in TransientFailure, latest connection error: connection error: desc = "transport: Error while dialing dial tcp 10.96.67.13:2379: connect: connection refused"”] [errorVerbose=“rpc error: code = Unavailable desc = all SubConns are in TransientFailure, latest connection error: connection error: desc = "transport: Error while dialing dial tcp 10.96.67.13:2379: connect: connection refused"
    github.com/pingcap/pd/client.(*client).getMembers
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:239
    github.com/pingcap/pd/client.(*client).initClusterID
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:201
    github.com/pingcap/pd/client.(*client).initRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:188
    github.com/pingcap/pd/client.NewClient
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:161
    github.com/pingcap/tidb/store/tikv.Driver.Open
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/kv.go:78
    github.com/pingcap/tidb/store.newStoreWithRetry.func1
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:70
    github.com/pingcap/tidb/util.RunWithRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/util/misc.go:46
    github.com/pingcap/tidb/store.newStoreWithRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:68
    github.com/pingcap/tidb/store.New
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:52
    main.createStoreAndDomain
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:202
    main.main
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:171
    runtime.main
    \t/usr/local/go/src/runtime/proc.go:200
    runtime.goexit
    \t/usr/local/go/src/runtime/asm_amd64.s:1337”] [stack=“github.com/pingcap/log.Error
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/log@v0.0.0-20190307075452-bd41d9273596/global.go:42
    github.com/pingcap/pd/client.(*client).initClusterID
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:204
    github.com/pingcap/pd/client.(*client).initRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:188
    github.com/pingcap/pd/client.NewClient
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:161
    github.com/pingcap/tidb/store/tikv.Driver.Open
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/kv.go:78
    github.com/pingcap/tidb/store.newStoreWithRetry.func1
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:70
    github.com/pingcap/tidb/util.RunWithRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/util/misc.go:46
    github.com/pingcap/tidb/store.newStoreWithRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:68
    github.com/pingcap/tidb/store.New
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:52
    main.createStoreAndDomain
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:202
    main.main
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:171
    runtime.main
    \t/usr/local/go/src/runtime/proc.go:200”]
    [2020/01/13 08:44:44.997 +00:00] [INFO] [client.go:256] [“[pd] switch leader”] [new-leader=http://tidb-cluster-pd-0.tidb-cluster-pd-peer.denali-tidb.svc:2379] [old-leader=]
    [2020/01/13 08:44:44.997 +00:00] [INFO] [client.go:167] [“[pd] init cluster id”] [cluster-id=6779827599285165284]
    [2020/01/13 08:44:45.000 +00:00] [ERROR] [client.go:400] [“[pd] create tso stream error”] [error=“rpc error: code = Unavailable desc = all SubConns are in TransientFailure, latest connection error: connection error: desc = "transport: Error while dialing dial tcp 10.244.1.131:2379: connect: connection refused"”] [stack=“github.com/pingcap/log.Error
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/log@v0.0.0-20190307075452-bd41d9273596/global.go:42
    github.com/pingcap/pd/client.(*client).tsLoop
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:400”]
    [2020/01/13 08:44:45.000 +00:00] [WARN] [store.go:77] [“new store with retry failed”] [error=“rpc error: code = Unavailable desc = all SubConns are in TransientFailure, latest connection error: connection error: desc = "transport: Error while dialing dial tcp 10.244.1.131:2379: connect: connection refused"”] [errorVerbose=“rpc error: code = Unavailable desc = all SubConns are in TransientFailure, latest connection error: connection error: desc = "transport: Error while dialing dial tcp 10.244.1.131:2379: connect: connection refused"
    github.com/pingcap/pd/client.(*client).tsLoop
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:403
    runtime.goexit
    \t/usr/local/go/src/runtime/asm_amd64.s:1337
    github.com/pingcap/pd/client.(*tsoRequest).Wait
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:601
    github.com/pingcap/pd/client.(*client).GetTS
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:617
    github.com/pingcap/tidb/store/tikv/oracle/oracles.(*pdOracle).getTimestamp
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/oracle/oracles/pd.go:103
    github.com/pingcap/tidb/store/tikv/oracle/oracles.(*pdOracle).GetTimestamp
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/oracle/oracles/pd.go:70
    github.com/pingcap/tidb/store/tikv/oracle/oracles.NewPdOracle
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/oracle/oracles/pd.go:53
    github.com/pingcap/tidb/store/tikv.newTikvStore
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/kv.go:178
    github.com/pingcap/tidb/store/tikv.Driver.Open
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/kv.go:104
    github.com/pingcap/tidb/store.newStoreWithRetry.func1
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:70
    github.com/pingcap/tidb/util.RunWithRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/util/misc.go:46
    github.com/pingcap/tidb/store.newStoreWithRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:68
    github.com/pingcap/tidb/store.New
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:52
    main.createStoreAndDomain
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:202
    main.main
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:171
    runtime.main
    \t/usr/local/go/src/runtime/proc.go:200
    runtime.goexit
    \t/usr/local/go/src/runtime/asm_amd64.s:1337
    github.com/pingcap/errors.AddStack
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/errors@v0.11.4/errors.go:174
    github.com/pingcap/errors.Trace
    \t/home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/errors@v0.11.4/juju_adaptor.go:15
    github.com/pingcap/tidb/store/tikv/oracle/oracles.(*pdOracle).getTimestamp
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/oracle/oracles/pd.go:105
    github.com/pingcap/tidb/store/tikv/oracle/oracles.(*pdOracle).GetTimestamp
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/oracle/oracles/pd.go:70
    github.com/pingcap/tidb/store/tikv/oracle/oracles.NewPdOracle
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/oracle/oracles/pd.go:53
    github.com/pingcap/tidb/store/tikv.newTikvStore
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/kv.go:178
    github.com/pingcap/tidb/store/tikv.Driver.Open
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/kv.go:104
    github.com/pingcap/tidb/store.newStoreWithRetry.func1
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:70
    github.com/pingcap/tidb/util.RunWithRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/util/misc.go:46
    github.com/pingcap/tidb/store.newStoreWithRetry
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:68
    github.com/pingcap/tidb/store.New
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:52
    main.createStoreAndDomain
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:202
    main.main
    \t/home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:171
    runtime.main
    \t/usr/local/go/src/runtime/proc.go:200
    runtime.goexit
    \t/usr/local/go/src/runtime/asm_amd64.s:1337”]
    2020/01/13 08:44:45.001 terror.go:337: [fatal] rpc error: code = Unavailable desc = all SubConns are in TransientFailure, latest connection error: connection error: desc = “transport: Error while dialing dial tcp 10.244.1.131:2379: connect: connection refused”
    github.com/pingcap/pd/client.(*client).tsLoop
    /home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:403
    runtime.goexit
    /usr/local/go/src/runtime/asm_amd64.s:1337
    github.com/pingcap/pd/client.(*tsoRequest).Wait
    /home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:601
    github.com/pingcap/pd/client.(*client).GetTS
    /home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/pd@v0.0.0-20190711034019-ee98bf9063e9/client/client.go:617
    github.com/pingcap/tidb/store/tikv/oracle/oracles.(*pdOracle).getTimestamp
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/oracle/oracles/pd.go:103
    github.com/pingcap/tidb/store/tikv/oracle/oracles.(*pdOracle).GetTimestamp
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/oracle/oracles/pd.go:70
    github.com/pingcap/tidb/store/tikv/oracle/oracles.NewPdOracle
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/oracle/oracles/pd.go:53
    github.com/pingcap/tidb/store/tikv.newTikvStore
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/kv.go:178
    github.com/pingcap/tidb/store/tikv.Driver.Open
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/kv.go:104
    github.com/pingcap/tidb/store.newStoreWithRetry.func1
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:70
    github.com/pingcap/tidb/util.RunWithRetry
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/util/misc.go:46
    github.com/pingcap/tidb/store.newStoreWithRetry
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:68
    github.com/pingcap/tidb/store.New
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:52
    main.createStoreAndDomain
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:202
    main.main
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:171
    runtime.main
    /usr/local/go/src/runtime/proc.go:200
    runtime.goexit
    /usr/local/go/src/runtime/asm_amd64.s:1337
    github.com/pingcap/errors.AddStack
    /home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/errors@v0.11.4/errors.go:174
    github.com/pingcap/errors.Trace
    /home/jenkins/workspace/release_tidb_3.0/go/pkg/mod/github.com/pingcap/errors@v0.11.4/juju_adaptor.go:15
    github.com/pingcap/tidb/store/tikv/oracle/oracles.(*pdOracle).getTimestamp
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/oracle/oracles/pd.go:105
    github.com/pingcap/tidb/store/tikv/oracle/oracles.(*pdOracle).GetTimestamp
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/oracle/oracles/pd.go:70
    github.com/pingcap/tidb/store/tikv/oracle/oracles.NewPdOracle
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/oracle/oracles/pd.go:53
    github.com/pingcap/tidb/store/tikv.newTikvStore
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/kv.go:178
    github.com/pingcap/tidb/store/tikv.Driver.Open
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/tikv/kv.go:104
    github.com/pingcap/tidb/store.newStoreWithRetry.func1
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:70
    github.com/pingcap/tidb/util.RunWithRetry
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/util/misc.go:46
    github.com/pingcap/tidb/store.newStoreWithRetry
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:68
    github.com/pingcap/tidb/store.New
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/store/store.go:52
    main.createStoreAndDomain
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:202
    main.main
    /home/jenkins/workspace/release_tidb_3.0/go/src/github.com/pingcap/tidb/tidb-server/main.go:171
    runtime.main
    /usr/local/go/src/runtime/proc.go:200
    runtime.goexit
    /usr/local/go/src/runtime/asm_amd64.s:1337error.log (25.4 KB)error2.log (48.4 KB)

1、请检查 pd 的状态是否正常:

./pd-ctl member -u http://ip:port

./pd-ctl health -u http://ip:port

2、建议查看下 tidb 到 pd 间的网络链接是否正常