tidb运行后自动关闭

TiDBer_4X2DucBd · 2023 年3 月 17 日 10:30

TIDB 部署情况：
三个pd 分别在 192.168.99.227，192.168.99.228，192.168.99.229
三个tidb 分别在 192.168.99.227，192.168.99.228，192.168.99.229
三个tikv 分布在 192.168.99.227，192.168.99.228，192.168.99.229。
系统运行一段时间后自动停止服务。
报错信息：
[2023/03/17 09:46:22.474 +08:00] [INFO] [util.go:47] [“Welcome to Placement Driver (PD)”]
[2023/03/17 09:46:22.474 +08:00] [INFO] [util.go:48] [PD] [release-version=v6.6.0]
[2023/03/17 09:46:22.474 +08:00] [INFO] [util.go:49] [PD] [edition=Community]
[2023/03/17 09:46:22.474 +08:00] [INFO] [util.go:50] [PD] [git-hash=67c0fa283673257b4dc0839f19cd74fec5ef46fb]
[2023/03/17 09:46:22.474 +08:00] [INFO] [util.go:51] [PD] [git-branch=heads/refs/tags/v6.6.0]
[2023/03/17 09:46:22.474 +08:00] [INFO] [util.go:52] [PD] [utc-build-time=“2023-02-14 11:56:24”]
[2023/03/17 09:46:22.474 +08:00] [INFO] [metricutil.go:83] [“disable Prometheus push client”]
[2023/03/17 09:46:22.474 +08:00] [INFO] [server.go:202] [“PD Config”] [config=“{"client-urls":"http://0.0.0.0:2379","peer-urls":"http://0.0.0.0:2380","advertise-client-urls":"http://192.168.99.227:2379","advertise-peer-urls":"http://192.168.99.227:2380","name":"pd-192.168.99.227-2379","data-dir":"/home/tidb/tidb-data/pd-2379","force-new-cluster":false,"enable-grpc-gateway":true,"initial-cluster":"pd-192.168.99.227-2379=http://192.168.99.227:2380,pd-192.168.99.228-2379=http://192.168.99.228:2380,pd-192.168.99.229-2379=http://192.168.99.229:2380","initial-cluster-state":"new","initial-cluster-token":"pd-cluster","join":"","lease":3,"log":{"level":"info","format":"text","disable-timestamp":false,"file":{"filename":"/home/tidb/tidb-deploy/pd-2379/log/pd.log","max-size":300,"max-days":0,"max-backups":0},"development":false,"disable-caller":false,"disable-stacktrace":false,"disable-error-verbose":true,"sampling":null,"error-output-path":""},"tso-save-interval":"3s","tso-update-physical-interval":"50ms","enable-local-tso":false,"metric":{"job":"pd-192.168.99.227-2379","address":"","interval":"15s"},"schedule":{"max-snapshot-count":64,"max-pending-peer-count":64,"max-merge-region-size":20,"max-merge-region-keys":0,"split-merge-interval":"1h0m0s","swtich-witness-interval":"1h0m0s","enable-one-way-merge":"false","enable-cross-table-merge":"true","patrol-region-interval":"10ms","max-store-down-time":"30m0s","max-store-preparing-time":"48h0m0s","leader-schedule-limit":4,"leader-schedule-policy":"count","region-schedule-limit":2048,"witness-schedule-limit":4,"replica-schedule-limit":64,"merge-schedule-limit":8,"hot-region-schedule-limit":4,"hot-region-cache-hits-threshold":3,"store-limit":{},"tolerant-size-ratio":0,"low-space-ratio":0.8,"high-space-ratio":0.7,"region-score-formula-version":"v2","scheduler-max-waiting-operator":5,"enable-remove-down-replica":"true","enable-replace-offline-replica":"true","enable-make-up-replica":"true","enable-remove-extra-replica":"true","enable-location-replacement":"true","enable-debug-metrics":"false","enable-joint-consensus":"true","enable-tikv-split-region":"true","schedulers-v2":[{"type":"balance-region","args":null,"disable":false,"args-payload":""},{"type":"balance-leader","args":null,"disable":false,"args-payload":""},{"type":"balance-witness","args":null,"disable":false,"args-payload":""},{"type":"hot-region","args":null,"disable":false,"args-payload":""},{"type":"split-bucket","args":null,"disable":false,"args-payload":""},{"type":"transfer-witness-leader","args":null,"disable":false,"args-payload":""}],"schedulers-payload":null,"store-limit-mode":"manual","hot-regions-write-interval":"10m0s","hot-regions-reserved-days":7,"enable-diagnostic":"false","enable-witness":"false","slow-store-evicting-affected-store-ratio-threshold":0.3},"replication":{"max-replicas":3,"location-labels":"","strictly-match-label":"false","enable-placement-rules":"true","enable-placement-rules-cache":"false","isolation-level":""},"pd-server":{"use-region-storage":"true","max-gap-reset-ts":"24h0m0s","key-type":"table","runtime-services":"","metric-storage":"","dashboard-address":"auto","trace-region-flow":"true","flow-round-by-digit":3,"min-resolved-ts-persistence-interval":"1s","server-memory-limit":0,"server-memory-limit-gc-trigger":0.7,"enable-gogc-tuner":"false","gc-tuner-threshold":0.6},"cluster-version":"0.0.0","labels":{},"quota-backend-bytes":"8GiB","auto-compaction-mode":"periodic","auto-compaction-retention-v2":"1h","TickInterval":"500ms","ElectionInterval":"3s","PreVote":true,"max-request-bytes":157286400,"security":{"cacert-path":"","cert-path":"","key-path":"","cert-allowed-cn":null,"SSLCABytes":null,"SSLCertBytes":null,"SSLKEYBytes":null,"redact-info-log":false,"encryption":{"data-encryption-method":"plaintext","data-key-rotation-period":"168h0m0s","master-key":{"type":"plaintext","key-id":"","region":"","endpoint":"","path":""}}},"label-property":null,"WarningMsgs":null,"DisableStrictReconfigCheck":false,"HeartbeatStreamBindInterval":"1m0s","LeaderPriorityCheckInterval":"1m0s","dashboard":{"tidb-cacert-path":"","tidb-cert-path":"","tidb-key-path":"","public-path-prefix":"","internal-proxy":false,"enable-telemetry":false,"enable-experimental":false},"replication-mode":{"replication-mode":"majority","dr-auto-sync":{"label-key":"","primary":"","dr":"","primary-replicas":0,"dr-replicas":0,"wait-store-timeout":"1m0s","pause-region-split":"false"}},"keyspace":{"pre-alloc":null}}”]
[2023/03/17 09:46:22.478 +08:00] [INFO] [util.go:220] [“register REST path”] [path=/pd/api/v1]
[2023/03/17 09:46:22.478 +08:00] [INFO] [util.go:220] [“register REST path”] [path=/pd/api/v2/]
[2023/03/17 09:46:22.478 +08:00] [INFO] [util.go:220] [“register REST path”] [path=/swagger/]
[2023/03/17 09:46:22.478 +08:00] [INFO] [util.go:220] [“register REST path”] [path=/autoscaling]
[2023/03/17 09:46:22.478 +08:00] [INFO] [distro.go:51] [“Using distribution strings”] [strings={}]
[2023/03/17 09:46:22.478 +08:00] [INFO] [util.go:220] [“register REST path”] [path=/dashboard/api/]
[2023/03/17 09:46:22.479 +08:00] [INFO] [util.go:220] [“register REST path”] [path=/dashboard/]
[2023/03/17 09:46:22.479 +08:00] [INFO] [util.go:220] [“register REST path”] [path=/resource-manager/api/v1/]
[2023/03/17 09:46:22.479 +08:00] [INFO] [registry.go:92] [“restful API service registered successfully”] [prefix=pd-192.168.99.227-2379] [service-name=ResourceManager]
[2023/03/17 09:46:22.479 +08:00] [INFO] [etcd.go:117] [“configuring peer listeners”] [listen-peer-urls=“[http://0.0.0.0:2380]”]
[2023/03/17 09:46:22.480 +08:00] [INFO] [etcd.go:127] [“configuring client listeners”] [listen-client-urls=“[http://0.0.0.0:2379]”]
[2023/03/17 09:46:22.480 +08:00] [INFO] [etcd.go:611] [“pprof is enabled”] [path=/debug/pprof]
[2023/03/17 09:46:22.479 +08:00] [INFO] [systimemon.go:28] [“start system time monitor”]
[2023/03/17 09:46:22.480 +08:00] [INFO] [etcd.go:305] [“starting an etcd server”] [etcd-version=3.4.21] [git-sha=“Not provided (use ./build instead of go build)”] [go-version=go1.19.5] [go-os=linux] [go-arch=amd64] [max-cpu-set=8] [max-cpu-available=8] [member-initialized=true] [name=pd-192.168.99.227-2379] [data-dir=/home/tidb/tidb-data/pd-2379] [wal-dir=] [wal-dir-dedicated=] [member-dir=/home/tidb/tidb-data/pd-2379/member] [force-new-cluster=false] [heartbeat-interval=500ms] [election-timeout=3s] [initial-election-tick-advance=true] [snapshot-count=100000] [snapshot-catchup-entries=5000] [initial-advertise-peer-urls=“[http://192.168.99.227:2380]”] [listen-peer-urls=“[http://0.0.0.0:2380]”] [advertise-client-urls=“[http://192.168.99.227:2379]”] [listen-client-urls=“[http://0.0.0.0:2379]”] [listen-metrics-urls=“[]”] [cors=“[]“] [host-whitelist=”[]”] [initial-cluster=] [initial-cluster-state=new] [initial-cluster-token=] [quota-backend-bytes=8589934592] [max-request-bytes=157286400] [max-concurrent-streams=4294967295] [pre-vote=true] [initial-corrupt-check=false] [corrupt-check-time-interval=0s] [auto-compaction-mode=periodic] [auto-compaction-retention=1h0m0s] [auto-compaction-interval=1h0m0s] [discovery-url=] [discovery-proxy=]
[2023/03/17 09:46:22.480 +08:00] [WARN] [server.go:297] [“exceeded recommended request limit”] [max-request-bytes=157286400] [max-request-size=“157 MB”] [recommended-request-bytes=10485760] [recommended-request-size=“10 MB”]
[2023/03/17 09:46:22.493 +08:00] [INFO] [backend.go:80] [“opened backend db”] [path=/home/tidb/tidb-data/pd-2379/member/snap/db] [took=12.700311ms]
[2023/03/17 09:46:22.695 +08:00] [INFO] [server.go:462] [“recovered v2 store from snapshot”] [snapshot-index=200002] [snapshot-size=“38 kB”]
[2023/03/17 09:46:22.696 +08:00] [INFO] [kvstore.go:388] [“restored last compact revision”] [meta-bucket-name=meta] [meta-bucket-name-key=finishedCompactRev] [restored-compact-revision=216381]
[2023/03/17 09:46:22.708 +08:00] [INFO] [server.go:480] [“recovered v3 backend from snapshot”] [backend-size-bytes=2629632] [backend-size=“2.6 MB”] [backend-size-in-use-bytes=1773568] [backend-size-in-use=“1.8 MB”]
[2023/03/17 09:46:22.967 +08:00] [INFO] [raft.go:586] [“restarting local member”] [cluster-id=a419421c9ec3f59b] [local-member-id=186de0fde55fedb6] [commit-index=230294]
[2023/03/17 09:46:22.970 +08:00] [INFO] [raft.go:1523] [“186de0fde55fedb6 switched to configuration voters=(1760310410451348918 4030756366632234945 12186352736355005418)”]
[2023/03/17 09:46:22.970 +08:00] [INFO] [raft.go:706] [“186de0fde55fedb6 became follower at term 63”]
[2023/03/17 09:46:22.970 +08:00] [INFO] [raft.go:389] [“newRaft 186de0fde55fedb6 [peers: [186de0fde55fedb6,37f01f8f41422fc1,a91e9f3f660cb3ea], term: 63, commit: 230294, applied: 200002, lastindex: 230295, lastterm: 63]”]
[2023/03/17 09:46:22.970 +08:00] [INFO] [capability.go:76] [“enabled capabilities for version”] [cluster-version=3.4]
[2023/03/17 09:46:22.970 +08:00] [INFO] [cluster.go:256] [“recovered/added member from store”] [cluster-id=a419421c9ec3f59b] [local-member-id=186de0fde55fedb6] [recovered-remote-peer-id=a91e9f3f660cb3ea] [recovered-remote-peer-urls=“[http://192.168.99.228:2380]”]
[2023/03/17 09:46:22.970 +08:00] [INFO] [cluster.go:256] [“recovered/added member from store”] [cluster-id=a419421c9ec3f59b] [local-member-id=186de0fde55fedb6] [recovered-remote-peer-id=186de0fde55fedb6] [recovered-remote-peer-urls=“[http://192.168.99.227:2380]”]
[2023/03/17 09:46:22.970 +08:00] [INFO] [cluster.go:256] [“recovered/added member from store”] [cluster-id=a419421c9ec3f59b] [local-member-id=186de0fde55fedb6] [recovered-remote-peer-id=37f01f8f41422fc1] [recovered-remote-peer-urls=“[http://192.168.99.229:2380]”]
[2023/03/17 09:46:22.970 +08:00] [INFO] [cluster.go:269] [“set cluster version from store”] [cluster-version=3.4]
[2023/03/17 09:46:22.971 +08:00] [WARN] [store.go:1379] [“simple token is not cryptographically signed”]
[2023/03/17 09:46:22.971 +08:00] [INFO] [kvstore.go:388] [“restored last compact revision”] [meta-bucket-name=meta] [meta-bucket-name-key=finishedCompactRev] [restored-compact-revision=216381]
[2023/03/17 09:46:22.986 +08:00] [INFO] [quota.go:126] [“enabled backend quota”] [quota-name=v3-applier] [quota-size-bytes=8589934592] [quota-size=“8.6 GB”]
[2023/03/17 09:46:22.987 +08:00] [INFO] [peer.go:128] [“starting remote peer”] [remote-peer-id=37f01f8f41422fc1]
[2023/03/17 09:46:22.987 +08:00] [INFO] [pipeline.go:71] [“started HTTP pipelining with remote peer”] [local-member-id=186de0fde55fedb6] [remote-peer-id=37f01f8f41422fc1]
[2023/03/17 09:46:22.988 +08:00] [INFO] [stream.go:166] [“started stream writer with remote peer”] [local-member-id=186de0fde55fedb6] [remote-peer-id=37f01f8f41422fc1]
[2023/03/17 09:46:22.988 +08:00] [INFO] [stream.go:166] [“started stream writer with remote peer”] [local-member-id=186de0fde55fedb6] [remote-peer-id=37f01f8f41422fc1]
[2023/03/17 09:46:22.988 +08:00] [INFO] [peer.go:134] [“started remote peer”] [remote-peer-id=37f01f8f41422fc1]
[2023/03/17 09:46:22.988 +08:00] [INFO] [transport.go:327] [“added remote peer”] [local-member-id=186de0fde55fedb6] [remote-peer-id=37f01f8f41422fc1] [remote-peer-urls=“[http://192.168.99.229:2380]”]
[2023/03/17 09:46:22.988 +08:00] [INFO] [peer.go:128] [“starting remote peer”] [remote-peer-id=a91e9f3f660cb3ea]
[2023/03/17 09:46:22.988 +08:00] [INFO] [pipeline.go:71] [“started HTTP pipelining with remote peer”] [local-member-id=186de0fde55fedb6] [remote-peer-id=a91e9f3f660cb3ea]
[2023/03/17 09:46:22.988 +08:00] [INFO] [stream.go:406] [“started stream reader with remote peer”] [stream-reader-type=“stream Message”] [local-member-id=186de0fde55fedb6] [remote-peer-id=37f01f8f41422fc1]
[2023/03/17 09:46:22.988 +08:00] [INFO] [stream.go:406] [“started stream reader with remote peer”] [stream-reader-type=“stream MsgApp v2”] [local-member-id=186de0fde55fedb6] [remote-peer-id=37f01f8f41422fc1]
[2023/03/17 09:46:22.989 +08:00] [INFO] [stream.go:166] [“started stream writer with remote peer”] [local-member-id=186de0fde55fedb6] [remote-peer-id=a91e9f3f660cb3ea]
[2023/03/17 09:46:22.989 +08:00] [INFO] [stream.go:166] [“started stream writer with remote peer”] [local-member-id=186de0fde55fedb6] [remote-peer-id=a91e9f3f660cb3ea]
[2023/03/17 09:46:22.989 +08:00] [INFO] [peer.go:134] [“started remote peer”] [remote-peer-id=a91e9f3f660cb3ea]
[2023/03/17 09:46:22.989 +08:00] [INFO] [transport.go:327] [“added remote peer”] [local-member-id=186de0fde55fedb6] [remote-peer-id=a91e9f3f660cb3ea] [remote-peer-urls=“[http://192.168.99.228:2380]”]
[2023/03/17 09:46:22.989 +08:00] [INFO] [stream.go:406] [“started stream reader with remote peer”] [stream-reader-type=“stream Message”] [local-member-id=186de0fde55fedb6] [remote-peer-id=a91e9f3f660cb3ea]
[2023/03/17 09:46:22.989 +08:00] [INFO] [server.go:803] [“starting etcd server”] [local-member-id=186de0fde55fedb6] [local-server-version=3.4.21] [cluster-id=a419421c9ec3f59b] [cluster-version=3.4]
[2023/03/17 09:46:22.989 +08:00] [INFO] [stream.go:406] [“started stream reader with remote peer”] [stream-reader-type=“stream MsgApp v2”] [local-member-id=186de0fde55fedb6] [remote-peer-id=a91e9f3f660cb3ea]
[2023/03/17 09:46:22.989 +08:00] [INFO] [server.go:704] [“starting initial election tick advance”] [election-ticks=6]
[2023/03/17 09:46:22.990 +08:00] [INFO] [peer_status.go:51] [“peer became active”] [peer-id=a91e9f3f660cb3ea]
[2023/03/17 09:46:22.990 +08:00] [INFO] [stream.go:425] [“established TCP streaming connection with remote peer”] [stream-reader-type=“stream MsgApp v2”] [local-member-id=186de0fde55fedb6] [remote-peer-id=a91e9f3f660cb3ea]
[2023/03/17 09:46:22.991 +08:00] [INFO] [peer_status.go:51] [“peer became active”] [peer-id=37f01f8f41422fc1]
[2023/03/17 09:46:22.991 +08:00] [INFO] [stream.go:425] [“established TCP streaming connection with remote peer”] [stream-reader-type=“stream Message”] [local-member-id=186de0fde55fedb6] [remote-peer-id=a91e9f3f660cb3ea]
[2023/03/17 09:46:22.991 +08:00] [INFO] [stream.go:425] [“established TCP streaming connection with remote peer”] [stream-reader-type=“stream Message”] [local-member-id=186de0fde55fedb6] [remote-peer-id=37f01f8f41422fc1]
[2023/03/17 09:46:22.991 +08:00] [INFO] [stream.go:425] [“established TCP streaming connection with remote peer”] [stream-reader-type=“stream MsgApp v2”] [local-member-id=186de0fde55fedb6] [remote-peer-id=37f01f8f41422fc1]
[2023/03/17 09:46:22.991 +08:00] [INFO] [etcd.go:247] [“now serving peer/client/metrics”] [local-member-id=186de0fde55fedb6] [initial-advertise-peer-urls=“[http://192.168.99.227:2380]”] [listen-peer-urls=“[http://0.0.0.0:2380]”] [advertise-client-urls=“[http://192.168.99.227:2379]”] [listen-client-urls=“[http://0.0.0.0:2379]”] [listen-metrics-urls=“[]”]
[2023/03/17 09:46:22.991 +08:00] [INFO] [etcd.go:585] [“serving peer traffic”] [address=“[::]:2380”]
[2023/03/17 09:46:22.991 +08:00] [INFO] [stream.go:250] [“set message encoder”] [from=186de0fde55fedb6] [to=186de0fde55fedb6] [stream-type=“stream Message”]
[2023/03/17 09:46:22.991 +08:00] [WARN] [stream.go:277] [“established TCP streaming connection with remote peer”] [stream-writer-type=“stream Message”] [local-member-id=186de0fde55fedb6] [remote-peer-id=a91e9f3f660cb3ea]
[2023/03/17 09:46:22.991 +08:00] [INFO] [node.go:327] [“raft.node: 186de0fde55fedb6 elected leader a91e9f3f660cb3ea at term 63”]
[2023/03/17 09:46:22.991 +08:00] [INFO] [stream.go:250] [“set message encoder”] [from=186de0fde55fedb6] [to=186de0fde55fedb6] [stream-type=“stream MsgApp v2”]
[2023/03/17 09:46:22.992 +08:00] [WARN] [stream.go:277] [“established TCP streaming connection with remote peer”] [stream-writer-type=“stream MsgApp v2”] [local-member-id=186de0fde55fedb6] [remote-peer-id=a91e9f3f660cb3ea]
[2023/03/17 09:46:22.992 +08:00] [INFO] [stream.go:250] [“set message encoder”] [from=186de0fde55fedb6] [to=186de0fde55fedb6] [stream-type=“stream Message”]
[2023/03/17 09:46:22.992 +08:00] [WARN] [stream.go:277] [“established TCP streaming connection with remote peer”] [stream-writer-type=“stream Message”] [local-member-id=186de0fde55fedb6] [remote-peer-id=37f01f8f41422fc1]
[2023/03/17 09:46:22.992 +08:00] [INFO] [stream.go:250] [“set message encoder”] [from=186de0fde55fedb6] [to=186de0fde55fedb6] [stream-type=“stream MsgApp v2”]
[2023/03/17 09:46:22.992 +08:00] [WARN] [stream.go:277] [“established TCP streaming connection with remote peer”] [stream-writer-type=“stream MsgApp v2”] [local-member-id=186de0fde55fedb6] [remote-peer-id=37f01f8f41422fc1]
[2023/03/17 09:46:23.019 +08:00] [INFO] [server.go:2069] [“published local member to cluster through raft”] [local-member-id=186de0fde55fedb6] [local-member-attributes=“{Name:pd-192.168.99.227-2379 ClientURLs:[http://192.168.99.227:2379]}”] [request-path=/0/members/186de0fde55fedb6/attributes] [cluster-id=a419421c9ec3f59b] [publish-timeout=11s]
[2023/03/17 09:46:23.019 +08:00] [INFO] [server.go:330] [“create etcd v3 client”] [endpoints=“[http://192.168.99.227:2379]”] [cert=“{"cacert-path":"","cert-path":"","key-path":"","cert-allowed-cn":null,"SSLCABytes":null,"SSLCertBytes":null,"SSLKEYBytes":null,"redact-info-log":false,"encryption":{"data-encryption-method":"plaintext","data-key-rotation-period":"168h0m0s","master-key":{"type":"plaintext","key-id":"","region":"","endpoint":"","path":""}}}”]
[2023/03/17 09:46:23.019 +08:00] [INFO] [registry.go:69] [“gRPC service already registered”] [prefix=pd-192.168.99.227-2379] [service-name=ResourceManager]
[2023/03/17 09:46:23.020 +08:00] [INFO] [serve.go:145] [“serving client traffic insecurely; this is strongly discouraged!”] [address=“[::]:2379”]
[2023/03/17 09:46:23.024 +08:00] [INFO] [server.go:363] [“init cluster id”] [cluster-id=7210713076413628124]
[2023/03/17 09:46:23.036 +08:00] [INFO] [allocator_manager.go:263] [“delete the dc-location key previously written in etcd”] [server-id=1760310410451348918]
[2023/03/17 09:46:23.040 +08:00] [INFO] [server.go:729] [“initialized peer connections; fast-forwarding election ticks”] [local-member-id=186de0fde55fedb6] [forward-ticks=4] [forward-duration=2s] [election-ticks=6] [election-timeout=3s] [active-remote-members=2]
[2023/03/17 09:46:23.044 +08:00] [INFO] [history_buffer.go:154] [“start from history index”] [start-index=6100]
[2023/03/17 09:46:23.048 +08:00] [INFO] [server.go:428] [“triggering the start callback functions”]
[2023/03/17 09:46:23.048 +08:00] [INFO] [manager.go:64] [“resource group manager starts to initialize”] [name=pd-192.168.99.227-2379]
[2023/03/17 09:46:23.054 +08:00] [INFO] [server.go:1562] [“server enable region storage”]
[2023/03/17 09:46:23.054 +08:00] [INFO] [client.go:124] [“region syncer start load region”]
[2023/03/17 09:46:23.054 +08:00] [INFO] [client.go:127] [“region syncer finished load region”] [time-cost=390.754µs]
[2023/03/17 09:46:23.054 +08:00] [INFO] [server.go:1395] [“start to watch pd leader”] [pd-leader=“name:"pd-192.168.99.228-2379" member_id:12186352736355005418 peer_urls:"http://192.168.99.228:2380" client_urls:"http://192.168.99.228:2379" “]
[2023/03/17 09:46:23.056 +08:00] [INFO] [client.go:168] [“server starts to synchronize with leader”] [server=pd-192.168.99.227-2379] [leader=pd-192.168.99.228-2379] [request-index=6100]
[2023/03/17 09:46:24.061 +08:00] [INFO] [dbstore.go:33] [“Dashboard initializing local storage file”] [path=/home/tidb/tidb-data/pd-2379/dashboard.sqlite.db]
[2023/03/17 09:46:24.072 +08:00] [INFO] [version.go:33] [“TiDB Dashboard started”] [internal-version=6.6.0] [standalone=No] [pd-version=v6.6.0] [build-time=“2023-02-14 11:56:24”] [build-git-hash=a58fc2a7e924]
[2023/03/17 09:46:24.072 +08:00] [INFO] [manager.go:197] [“Dashboard server is started”]
[2023/03/17 09:46:24.072 +08:00] [INFO] [proxy.go:211] [“start serve requests to remotes”] [endpoint=127.0.0.1:44473] [remotes=”[]”]
[2023/03/17 09:46:24.072 +08:00] [INFO] [proxy.go:211] [“start serve requests to remotes”] [endpoint=127.0.0.1:45567] [remotes=“[]”]
[2023/03/17 09:46:24.075 +08:00] [INFO] [dynamic_config_manager.go:167] [“Load dynamic config from etcd”] [json=“{"keyvisual":{"auto_collection_disabled":false,"policy":"db","policy_kv_separator":""},"profiling":{"auto_collection_targets":null,"auto_collection_duration_secs":0,"auto_collection_interval_secs":0},"sso":{"core_config":{"enabled":false,"client_id":"","discovery_url":"","is_read_only":false},"auth_url":"","token_url":"","user_info_url":"","sign_out_url":""}}”]
[2023/03/17 09:46:24.078 +08:00] [INFO] [dynamic_config_manager.go:188] [“Save dynamic config to etcd”] [json=“{"keyvisual":{"auto_collection_disabled":false,"policy":"db","policy_kv_separator":""},"profiling":{"auto_collection_targets":null,"auto_collection_duration_secs":0,"auto_collection_interval_secs":0},"sso":{"core_config":{"enabled":false,"client_id":"","discovery_url":"","is_read_only":false},"auth_url":"","token_url":"","user_info_url":"","sign_out_url":""}}”]
[2023/03/17 09:46:24.108 +08:00] [INFO] [manager.go:74] [“Key visual service is started”]
[2023/03/17 09:48:22.634 +08:00] [WARN] [stream.go:436] [“lost TCP streaming connection with remote peer”] [stream-reader-type=“stream Message”] [local-member-id=186de0fde55fedb6] [remote-peer-id=37f01f8f41422fc1] [error=“unexpected EOF”]
[2023/03/17 09:48:22.634 +08:00] [WARN] [peer_status.go:68] [“peer became inactive (message send to peer failed)”] [peer-id=37f01f8f41422fc1] [error=“failed to read 37f01f8f41422fc1 on stream Message (unexpected EOF)”]
[2023/03/17 09:48:22.634 +08:00] [WARN] [stream.go:436] [“lost TCP streaming connection with remote peer”] [stream-reader-type=“stream MsgApp v2”] [local-member-id=186de0fde55fedb6] [remote-peer-id=37f01f8f41422fc1] [error=“unexpected EOF”]
[2023/03/17 09:48:22.636 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.229:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.229:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 09:48:22.738 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.229:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.229:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 09:48:22.896 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.229:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.229:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 09:48:23.103 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.229:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.229:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 09:48:23.541 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.229:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.229:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 09:48:24.264 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.229:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.229:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 09:48:25.357 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.229:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.229:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 09:48:27.282 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.229:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.229:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 09:48:27.989 +08:00] [WARN] [stream.go:193] [“lost TCP streaming connection with remote peer”] [stream-writer-type=“stream MsgApp v2”] [local-member-id=186de0fde55fedb6] [remote-peer-id=37f01f8f41422fc1]
[2023/03/17 09:48:27.990 +08:00] [WARN] [stream.go:193] [“lost TCP streaming connection with remote peer”] [stream-writer-type=“stream Message”] [local-member-id=186de0fde55fedb6] [remote-peer-id=37f01f8f41422fc1]
[2023/03/17 09:48:27.997 +08:00] [WARN] [probing_status.go:70] [“prober detected unhealthy status”] [round-tripper-name=ROUND_TRIPPER_SNAPSHOT] [remote-peer-id=37f01f8f41422fc1] [rtt=1.154587ms] [error=“dial tcp 192.168.99.229:2380: connect: connection refused”]
[2023/03/17 09:48:27.998 +08:00] [WARN] [probing_status.go:70] [“prober detected unhealthy status”] [round-tripper-name=ROUND_TRIPPER_RAFT_MESSAGE] [remote-peer-id=37f01f8f41422fc1] [rtt=1.925701ms] [error=“dial tcp 192.168.99.229:2380: connect: connection refused”]
[2023/03/17 09:48:30.082 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.229:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.229:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 09:48:32.999 +08:00] [WARN] [probing_status.go:70] [“prober detected unhealthy status”] [round-tripper-name=ROUND_TRIPPER_SNAPSHOT] [remote-peer-id=37f01f8f41422fc1] [rtt=1.154587ms] [error=“dial tcp 192.168.99.229:2380: connect: connection refused”]
[2023/03/17 09:48:32.999 +08:00] [WARN] [probing_status.go:70] [“prober detected unhealthy status”] [round-tripper-name=ROUND_TRIPPER_RAFT_MESSAGE] [remote-peer-id=37f01f8f41422fc1] [rtt=1.925701ms] [error=“dial tcp 192.168.99.229:2380: connect: connection refused”]
[2023/03/17 09:48:33.533 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.229:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.229:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 09:48:37.137 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.229:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.229:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 09:48:37.999 +08:00] [WARN] [probing_status.go:70] [“prober detected unhealthy status”] [round-tripper-name=ROUND_TRIPPER_SNAPSHOT] [remote-peer-id=37f01f8f41422fc1] [rtt=1.154587ms] [error=“dial tcp 192.168.99.229:2380: connect: connection refused”]
[2023/03/17 09:48:37.999 +08:00] [WARN] [probing_status.go:70] [“prober detected unhealthy status”] [round-tripper-name=ROUND_TRIPPER_RAFT_MESSAGE] [remote-peer-id=37f01f8f41422fc1] [rtt=1.925701ms] [error=“dial tcp 192.168.99.229:2380: connect: connection refused”]
[2023/03/17 09:48:38.178 +08:00] [INFO] [stream.go:250] [“set message encoder”] [from=186de0fde55fedb6] [to=186de0fde55fedb6] [stream-type=“stream Message”]
[2023/03/17 09:48:38.178 +08:00] [INFO] [peer_status.go:51] [“peer became active”] [peer-id=37f01f8f41422fc1]
[2023/03/17 09:48:38.179 +08:00] [WARN] [stream.go:277] [“established TCP streaming connection with remote peer”] [stream-writer-type=“stream Message”] [local-member-id=186de0fde55fedb6] [remote-peer-id=37f01f8f41422fc1]
[2023/03/17 09:48:38.179 +08:00] [INFO] [stream.go:250] [“set message encoder”] [from=186de0fde55fedb6] [to=186de0fde55fedb6] [stream-type=“stream MsgApp v2”]
[2023/03/17 09:48:38.179 +08:00] [WARN] [stream.go:277] [“established TCP streaming connection with remote peer”] [stream-writer-type=“stream MsgApp v2”] [local-member-id=186de0fde55fedb6] [remote-peer-id=37f01f8f41422fc1]
[2023/03/17 09:48:38.183 +08:00] [INFO] [stream.go:425] [“established TCP streaming connection with remote peer”] [stream-reader-type=“stream Message”] [local-member-id=186de0fde55fedb6] [remote-peer-id=37f01f8f41422fc1]
[2023/03/17 09:48:38.183 +08:00] [INFO] [stream.go:425] [“established TCP streaming connection with remote peer”] [stream-reader-type=“stream MsgApp v2”] [local-member-id=186de0fde55fedb6] [remote-peer-id=37f01f8f41422fc1]
[2023/03/17 10:04:59.344 +08:00] [INFO] [index.go:189] [“compact tree index”] [revision=224983]
[2023/03/17 10:04:59.468 +08:00] [INFO] [kvstore_compaction.go:56] [“finished scheduled compaction”] [compact-revision=224983] [took=123.461357ms]
[2023/03/17 11:04:59.363 +08:00] [INFO] [index.go:189] [“compact tree index”] [revision=230499]
[2023/03/17 11:04:59.438 +08:00] [INFO] [kvstore_compaction.go:56] [“finished scheduled compaction”] [compact-revision=230499] [took=74.838388ms]
[2023/03/17 11:17:05.282 +08:00] [ERROR] [client.go:172] [“region sync with leader meet error”] [error=“[PD:grpc:ErrGRPCRecv]rpc error: code = Unavailable desc = transport is closing: rpc error: code = Unavailable desc = transport is closing”]
[2023/03/17 11:17:05.282 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.228:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 11:17:05.283 +08:00] [WARN] [stream.go:436] [“lost TCP streaming connection with remote peer”] [stream-reader-type=“stream MsgApp v2”] [local-member-id=186de0fde55fedb6] [remote-peer-id=a91e9f3f660cb3ea] [error=“unexpected EOF”]
[2023/03/17 11:17:05.283 +08:00] [WARN] [peer_status.go:68] [“peer became inactive (message send to peer failed)”] [peer-id=a91e9f3f660cb3ea] [error=“failed to read a91e9f3f660cb3ea on stream MsgApp v2 (unexpected EOF)”]
[2023/03/17 11:17:05.283 +08:00] [WARN] [stream.go:436] [“lost TCP streaming connection with remote peer”] [stream-reader-type=“stream Message”] [local-member-id=186de0fde55fedb6] [remote-peer-id=a91e9f3f660cb3ea] [error=“unexpected EOF”]
[2023/03/17 11:17:05.283 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {192.168.99.228:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 11:17:05.383 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.228:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 11:17:05.570 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.228:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 11:17:05.794 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.228:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 11:17:06.271 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.228:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 11:17:06.285 +08:00] [ERROR] [client.go:163] [“server failed to establish sync stream with leader”] [server=pd-192.168.99.227-2379] [leader=pd-192.168.99.228-2379] [error=“rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused"”]
[2023/03/17 11:17:06.286 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {192.168.99.228:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 11:17:06.553 +08:00] [WARN] [v3_server.go:814] [“waiting for ReadIndex response took too long, retrying”] [sent-request-id=17129026586274797927] [retry-timeout=500ms]
[2023/03/17 11:17:06.804 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.228:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 11:17:07.053 +08:00] [WARN] [v3_server.go:814] [“waiting for ReadIndex response took too long, retrying”] [sent-request-id=17129026586274797927] [retry-timeout=500ms]
[2023/03/17 11:17:07.054 +08:00] [WARN] [stream.go:224] [“lost TCP streaming connection with remote peer”] [stream-writer-type=“stream Message”] [local-member-id=186de0fde55fedb6] [remote-peer-id=a91e9f3f660cb3ea]
[2023/03/17 11:17:07.286 +08:00] [ERROR] [client.go:163] [“server failed to establish sync stream with leader”] [server=pd-192.168.99.227-2379] [leader=pd-192.168.99.228-2379] [error=“rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused"”]
[2023/03/17 11:17:07.553 +08:00] [WARN] [v3_server.go:814] [“waiting for ReadIndex response took too long, retrying”] [sent-request-id=17129026586274797927] [retry-timeout=500ms]
[2023/03/17 11:17:07.970 +08:00] [INFO] [raft.go:929] [“186de0fde55fedb6 is starting a new election at term 63”]
[2023/03/17 11:17:07.970 +08:00] [INFO] [raft.go:735] [“186de0fde55fedb6 became pre-candidate at term 63”]
[2023/03/17 11:17:07.970 +08:00] [INFO] [raft.go:830] [“186de0fde55fedb6 received MsgPreVoteResp from 186de0fde55fedb6 at term 63”]
[2023/03/17 11:17:07.970 +08:00] [INFO] [raft.go:817] [“186de0fde55fedb6 [logterm: 63, index: 239565] sent MsgPreVote request to 37f01f8f41422fc1 at term 63”]
[2023/03/17 11:17:07.970 +08:00] [INFO] [raft.go:817] [“186de0fde55fedb6 [logterm: 63, index: 239565] sent MsgPreVote request to a91e9f3f660cb3ea at term 63”]
[2023/03/17 11:17:07.970 +08:00] [INFO] [node.go:333] [“raft.node: 186de0fde55fedb6 lost leader a91e9f3f660cb3ea at term 63”]
[2023/03/17 11:17:07.971 +08:00] [INFO] [raft.go:830] [“186de0fde55fedb6 received MsgPreVoteResp from 37f01f8f41422fc1 at term 63”]
[2023/03/17 11:17:07.971 +08:00] [INFO] [raft.go:1295] [“186de0fde55fedb6 has received 2 MsgPreVoteResp votes and 0 vote rejections”]
[2023/03/17 11:17:07.971 +08:00] [INFO] [raft.go:719] [“186de0fde55fedb6 became candidate at term 64”]
[2023/03/17 11:17:07.971 +08:00] [INFO] [raft.go:830] [“186de0fde55fedb6 received MsgVoteResp from 186de0fde55fedb6 at term 64”]
[2023/03/17 11:17:07.971 +08:00] [INFO] [raft.go:817] [“186de0fde55fedb6 [logterm: 63, index: 239565] sent MsgVote request to 37f01f8f41422fc1 at term 64”]
[2023/03/17 11:17:07.971 +08:00] [INFO] [raft.go:817] [“186de0fde55fedb6 [logterm: 63, index: 239565] sent MsgVote request to a91e9f3f660cb3ea at term 64”]
[2023/03/17 11:17:07.973 +08:00] [INFO] [raft.go:830] [“186de0fde55fedb6 received MsgVoteResp from 37f01f8f41422fc1 at term 64”]
[2023/03/17 11:17:07.973 +08:00] [INFO] [raft.go:1295] [“186de0fde55fedb6 has received 2 MsgVoteResp votes and 0 vote rejections”]
[2023/03/17 11:17:07.973 +08:00] [INFO] [raft.go:771] [“186de0fde55fedb6 became leader at term 64”]
[2023/03/17 11:17:07.973 +08:00] [INFO] [node.go:327] [“raft.node: 186de0fde55fedb6 elected leader 186de0fde55fedb6 at term 64”]
[2023/03/17 11:17:07.974 +08:00] [WARN] [util.go:163] [“apply request took too long”] [took=1.92435491s] [expected-duration=100ms] [prefix=“read-only range “] [request=“key:"/pd/7210713076413628124/config" “] [response=] [error=“etcdserver: leader changed”]
[2023/03/17 11:17:07.974 +08:00] [INFO] [trace.go:152] [“trace[1089097347] range”] [detail=”{range_begin:/pd/7210713076413628124/config; range_end:; }”] [duration=1.924781166s] [start=2023/03/17 11:17:06.049 +08:00] [end=2023/03/17 11:17:07.974 +08:00] [steps=”["trace[1089097347] ‘agreement among raft nodes before linearized reading’ (duration: 1.924396639s)"]”]
[2023/03/17 11:17:07.975 +08:00] [INFO] [v3_server.go:802] [“first commit in current term: resending ReadIndex request”]
[2023/03/17 11:17:07.976 +08:00] [WARN] [util.go:163] [“apply request took too long”] [took=906.242155ms] [expected-duration=100ms] [prefix=“read-only range “] [request=“key:"/tidb/store/gcworker/saved_safe_point" “] [response=“range_response_count:1 size:79”] []
[2023/03/17 11:17:07.976 +08:00] [INFO] [trace.go:152] [“trace[72068049] range”] [detail=”{range_begin:/tidb/store/gcworker/saved_safe_point; range_end:; response_count:1; response_revision:238423; }”] [duration=906.439124ms] [start=2023/03/17 11:17:07.069 +08:00] [end=2023/03/17 11:17:07.976 +08:00] [steps=”["trace[72068049] ‘agreement among raft nodes before linearized reading’ (duration: 906.189887ms)"]”]
[2023/03/17 11:17:07.977 +08:00] [WARN] [cluster_util.go:315] [“failed to reach the peer URL”] [address=http://192.168.99.228:2380/version] [remote-member-id=a91e9f3f660cb3ea] [error=“Get "http://192.168.99.228:2380/version\”: dial tcp 192.168.99.228:2380: connect: connection refused"]
[2023/03/17 11:17:07.977 +08:00] [WARN] [cluster_util.go:168] [“failed to get version”] [remote-member-id=a91e9f3f660cb3ea] [error=“Get "http://192.168.99.228:2380/version\”: dial tcp 192.168.99.228:2380: connect: connection refused"]
[2023/03/17 11:17:07.998 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {192.168.99.228:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 11:17:07.998 +08:00] [WARN] [etcdutil.go:121] [“kv gets too slow”] [request-key=/pd/7210713076413628124/config] [cost=1.949798298s] []
[2023/03/17 11:17:08.005 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.228:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 11:17:08.287 +08:00] [ERROR] [client.go:163] [“server failed to establish sync stream with leader”] [server=pd-192.168.99.227-2379] [leader=pd-192.168.99.228-2379] [error=“rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused"”]
[2023/03/17 11:17:09.287 +08:00] [ERROR] [client.go:163] [“server failed to establish sync stream with leader”] [server=pd-192.168.99.227-2379] [leader=pd-192.168.99.228-2379] [error=“rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused"”]
[2023/03/17 11:17:09.634 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.228:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 11:17:09.658 +08:00] [WARN] [stream.go:193] [“lost TCP streaming connection with remote peer”] [stream-writer-type=“stream MsgApp v2”] [local-member-id=186de0fde55fedb6] [remote-peer-id=a91e9f3f660cb3ea]
[2023/03/17 11:17:10.289 +08:00] [ERROR] [client.go:163] [“server failed to establish sync stream with leader”] [server=pd-192.168.99.227-2379] [leader=pd-192.168.99.228-2379] [error=“rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused"”]
[2023/03/17 11:17:10.499 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {192.168.99.228:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 11:17:11.290 +08:00] [ERROR] [client.go:163] [“server failed to establish sync stream with leader”] [server=pd-192.168.99.227-2379] [leader=pd-192.168.99.228-2379] [error=“rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused"”]
[2023/03/17 11:17:11.969 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.228:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 11:17:11.979 +08:00] [WARN] [cluster_util.go:315] [“failed to reach the peer URL”] [address=http://192.168.99.228:2380/version] [remote-member-id=a91e9f3f660cb3ea] [error=“Get "http://192.168.99.228:2380/version\”: dial tcp 192.168.99.228:2380: connect: connection refused"]
[2023/03/17 11:17:11.979 +08:00] [WARN] [cluster_util.go:168] [“failed to get version”] [remote-member-id=a91e9f3f660cb3ea] [error=“Get "http://192.168.99.228:2380/version\”: dial tcp 192.168.99.228:2380: connect: connection refused"]
[2023/03/17 11:17:12.290 +08:00] [ERROR] [client.go:163] [“server failed to establish sync stream with leader”] [server=pd-192.168.99.227-2379] [leader=pd-192.168.99.228-2379] [error=“rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused"”]
[2023/03/17 11:17:12.961 +08:00] [ERROR] [middleware.go:150] [“request failed”] [error=“[PD:http:ErrSendRequest]Get "http://192.168.99.228:2379/pd/api/v1/members\”: dial tcp 192.168.99.228:2379: connect: connection refused: Get "http://192.168.99.228:2379/pd/api/v1/members\“: dial tcp 192.168.99.228:2379: connect: connection refused”]
[2023/03/17 11:17:13.101 +08:00] [ERROR] [middleware.go:150] [“request failed”] [error=“[PD:http:ErrSendRequest]Get "http://192.168.99.228:2379/pd/api/v1/stores\”: dial tcp 192.168.99.228:2379: connect: connection refused: Get "http://192.168.99.228:2379/pd/api/v1/stores\“: dial tcp 192.168.99.228:2379: connect: connection refused”]
[2023/03/17 11:17:13.291 +08:00] [ERROR] [client.go:163] [“server failed to establish sync stream with leader”] [server=pd-192.168.99.227-2379] [leader=pd-192.168.99.228-2379] [error=“rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused"”]
[2023/03/17 11:17:13.593 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {192.168.99.228:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 11:17:14.297 +08:00] [ERROR] [client.go:163] [“server failed to establish sync stream with leader”] [server=pd-192.168.99.227-2379] [leader=pd-192.168.99.228-2379] [error=“rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused"”]
[2023/03/17 11:17:15.103 +08:00] [ERROR] [middleware.go:150] [“request failed”] [error=“[PD:http:ErrSendRequest]Get "http://192.168.99.228:2379/pd/api/v1/stores\”: dial tcp 192.168.99.228:2379: connect: connection refused: Get "http://192.168.99.228:2379/pd/api/v1/stores\“: dial tcp 192.168.99.228:2379: connect: connection refused”]
[2023/03/17 11:17:15.262 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.228:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 11:17:15.298 +08:00] [ERROR] [client.go:163] [“server failed to establish sync stream with leader”] [server=pd-192.168.99.227-2379] [leader=pd-192.168.99.228-2379] [error=“rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused"”]
[2023/03/17 11:17:15.980 +08:00] [WARN] [cluster_util.go:315] [“failed to reach the peer URL”] [address=http://192.168.99.228:2380/version] [remote-member-id=a91e9f3f660cb3ea] [error=“Get "http://192.168.99.228:2380/version\”: dial tcp 192.168.99.228:2380: connect: connection refused"]
[2023/03/17 11:17:15.981 +08:00] [WARN] [cluster_util.go:168] [“failed to get version”] [remote-member-id=a91e9f3f660cb3ea] [error=“Get "http://192.168.99.228:2380/version\”: dial tcp 192.168.99.228:2380: connect: connection refused"]
[2023/03/17 11:17:16.206 +08:00] [INFO] [leadership.go:211] [“current leadership is deleted”] [leader-key=/pd/7210713076413628124/leader] [purpose=“pd leader election”]
[2023/03/17 11:17:16.240 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {192.168.99.228:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 11:17:16.298 +08:00] [INFO] [server.go:1399] [“pd leader has changed, try to re-campaign a pd leader”]
[2023/03/17 11:17:16.299 +08:00] [INFO] [server.go:1417] [“start to campaign pd leader”] [campaign-pd-leader-name=pd-192.168.99.227-2379]
[2023/03/17 11:17:16.301 +08:00] [INFO] [lease.go:65] [“lease granted”] [lease-id=7905654549420022137] [lease-timeout=3] [purpose=“pd leader election”]
[2023/03/17 11:17:16.304 +08:00] [INFO] [leadership.go:122] [“check campaign resp”] [resp=“{"header":{"cluster_id":11824554987328107931,"member_id":1760310410451348918,"revision":238426,"raft_term":64},"succeeded":true,"responses":[{"Response":{"ResponsePut":{"header":{"revision":238426}}}}]}”]
[2023/03/17 11:17:16.305 +08:00] [INFO] [leadership.go:131] [“write leaderData to leaderPath ok”] [leaderPath=/pd/7210713076413628124/leader] [purpose=“pd leader election”]
[2023/03/17 11:17:16.305 +08:00] [INFO] [server.go:1443] [“campaign pd leader ok”] [campaign-pd-leader-name=pd-192.168.99.227-2379]
[2023/03/17 11:17:16.305 +08:00] [INFO] [server.go:1450] [“initializing the global TSO allocator”]
[2023/03/17 11:17:16.305 +08:00] [INFO] [lease.go:135] [“start lease keep alive worker”] [interval=1s] [purpose=“pd leader election”]
[2023/03/17 11:17:16.309 +08:00] [INFO] [tso.go:227] [“sync and save timestamp”] [last=2023/03/17 11:17:05.615 +08:00] [save=2023/03/17 11:17:19.307 +08:00] [next=2023/03/17 11:17:16.307 +08:00]
[2023/03/17 11:17:16.311 +08:00] [INFO] [server.go:1562] [“server enable region storage”]
[2023/03/17 11:17:16.313 +08:00] [INFO] [server.go:1479] [“triggering the leader callback functions”]
[2023/03/17 11:17:16.314 +08:00] [INFO] [manager.go:103] [“resource group manager finishes initialization”]
[2023/03/17 11:17:16.325 +08:00] [INFO] [cluster.go:441] [“load stores”] [count=3] [cost=8.740447ms]
[2023/03/17 11:17:16.325 +08:00] [INFO] [cluster.go:452] [“load regions”] [count=6] [cost=4.19µs]
[2023/03/17 11:17:16.343 +08:00] [INFO] [coordinator.go:319] [“coordinator starts to collect cluster information”]
[2023/03/17 11:17:16.344 +08:00] [INFO] [cluster.go:318] [“memory info”] [total-mem=16655978496]
[2023/03/17 11:17:16.344 +08:00] [INFO] [tuner.go:117] [“new gctuner”] [threshold=9993587097]
[2023/03/17 11:17:16.344 +08:00] [INFO] [cluster.go:331] [“update gc tuner”] [enable-gc-tuner=false] [gc-threshold-bytes=9993587097]
[2023/03/17 11:17:16.344 +08:00] [INFO] [memory_limit_tuner.go:49] [debug.SetMemoryLimit] [limit=9223372036854775807] [ret=9223372036854775807]
[2023/03/17 11:17:16.344 +08:00] [INFO] [cluster.go:338] [“update gc memory limit”] [memory-limit-bytes=0] [memory-limit-gc-trigger-ratio=0.7]
[2023/03/17 11:17:16.346 +08:00] [INFO] [store_config.go:200] [“sync the store config successful”] [store-address=192.168.99.227:20180] [store-config=“{\n "coprocessor": {\n "region-max-size": "144MiB",\n "region-split-size": "96MiB",\n "region-max-keys": 1440000,\n "region-split-keys": 960000,\n "enable-region-bucket": false,\n "region-bucket-size": "96MiB"\n }\n}”]
[2023/03/17 11:17:16.348 +08:00] [INFO] [id.go:174] [“idAllocator allocates a new id”] [new-end=62000] [new-base=61000] [label=idalloc] [check-curr-end=true]
[2023/03/17 11:17:16.348 +08:00] [INFO] [util.go:83] [“load cluster version”] [cluster-version=6.6.0]
[2023/03/17 11:17:16.348 +08:00] [INFO] [server.go:1506] [“PD cluster leader is ready to serve”] [pd-leader-name=pd-192.168.99.227-2379]
[2023/03/17 11:17:16.506 +08:00] [INFO] [server.go:212] [“establish sync region stream”] [requested-server=pd-192.168.99.229-2379] [url=http://192.168.99.229:2379]
[2023/03/17 11:17:16.506 +08:00] [INFO] [server.go:230] [“requested server has already in sync with server”] [requested-server=pd-192.168.99.229-2379] [server=pd-192.168.99.227-2379] [last-index=6321]
[2023/03/17 11:17:17.672 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.228:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 11:17:19.983 +08:00] [WARN] [cluster_util.go:315] [“failed to reach the peer URL”] [address=http://192.168.99.228:2380/version] [remote-member-id=a91e9f3f660cb3ea] [error=“Get "http://192.168.99.228:2380/version\”: dial tcp 192.168.99.228:2380: connect: connection refused"]
[2023/03/17 11:17:19.983 +08:00] [WARN] [cluster_util.go:168] [“failed to get version”] [remote-member-id=a91e9f3f660cb3ea] [error=“Get "http://192.168.99.228:2380/version\”: dial tcp 192.168.99.228:2380: connect: connection refused"]
[2023/03/17 11:17:20.248 +08:00] [WARN] [grpclog.go:60] [“grpc: addrConn.createTransport failed to connect to {http://192.168.99.228:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused". Reconnecting…”]
[2023/03/17 11:17:21.100 +08:00] [INFO] [stream.go:250] [“set message encoder”] [from=186de0fde55fedb6] [to=186de0fde55fedb6] [stream-type=“stream MsgApp v2”]
[2023/03/17 11:17:21.100 +08:00] [INFO] [peer_status.go:51] [“peer became active”] [peer-id=a91e9f3f660cb3ea]
[2023/03/17 11:17:21.100 +08:00] [WARN] [stream.go:277] [“established TCP streaming connection with remote peer”] [stream-writer-type=“stream MsgApp v2”] [local-member-id=186de0fde55fedb6] [remote-peer-id=a91e9f3f660cb3ea]
[2023/03/17 11:17:21.101 +08:00] [INFO] [stream.go:250] [“set message encoder”] [from=186de0fde55fedb6] [to=186de0fde55fedb6] [stream-type=“stream Message”]
[2023/03/17 11:17:21.101 +08:00] [WARN] [stream.go:277] [“established TCP streaming connection with remote peer”] [stream-writer-type=“stream Message”] [local-member-id=186de0fde55fedb6] [remote-peer-id=a91e9f3f660cb3ea]
[2023/03/17 11:17:21.106 +08:00] [INFO] [stream.go:425] [“established TCP streaming connection with remote peer”] [stream-reader-type=“stream Message”] [local-member-id=186de0fde55fedb6] [remote-peer-id=a91e9f3f660cb3ea]
[2023/03/17 11:17:21.106 +08:00] [INFO] [stream.go:425] [“established TCP streaming connection with remote peer”] [stream-reader-type=“stream MsgApp v2”] [local-member-id=186de0fde55fedb6] [remote-peer-id=a91e9f3f660cb3ea]
[2023/03/17 11:17:21.198 +08:00] [INFO] [server.go:212] [“establish sync region stream”] [requested-server=pd-192.168.99.228-2379] [url=http://192.168.99.228:2379]
[2023/03/17 11:17:21.198 +08:00] [INFO] [server.go:295] [“sync the history regions with server”] [server=pd-192.168.99.228-2379] [from-index=6300] [last-index=6321] [records-length=21]
[2023/03/17 11:17:23.789 +08:00] [INFO] [grpc_service.go:1451] [“update service GC safe point”] [service-id=gc_worker] [expire-at=-9223372035175752766] [safepoint=440145659494137856]
[2023/03/17 11:18:19.343 +08:00] [INFO] [coordinator.go:322] [“coordinator has finished cluster information preparation”]
[2023/03/17 11:18:19.343 +08:00] [INFO] [coordinator.go:332] [“coordinator starts to run schedulers”]
[2023/03/17 11:18:19.349 +08:00] [INFO] [coordinator.go:380] [“create scheduler with independent configuration”] [scheduler-name=balance-hot-region-scheduler]
[2023/03/17 11:18:19.353 +08:00] [INFO] [coordinator.go:380] [“create scheduler with independent configuration”] [scheduler-name=balance-leader-scheduler]
[2023/03/17 11:18:19.355 +08:00] [INFO] [coordinator.go:380] [“create scheduler with independent configuration”] [scheduler-name=balance-region-scheduler]
[2023/03/17 11:18:19.357 +08:00] [INFO] [coordinator.go:380] [“create scheduler with independent configuration”] [scheduler-name=balance-witness-scheduler]
[2023/03/17 11:18:19.359 +08:00] [INFO] [coordinator.go:380] [“create scheduler with independent configuration”] [scheduler-name=split-bucket-scheduler]
[2023/03/17 11:18:19.361 +08:00] [INFO] [coordinator.go:380] [“create scheduler with independent configuration”] [scheduler-name=transfer-witness-leader-scheduler]
[2023/03/17 11:18:19.363 +08:00] [INFO] [coordinator.go:402] [“create scheduler”] [scheduler-name=balance-region-scheduler] [scheduler-args=“[]”]
[2023/03/17 11:18:19.366 +08:00] [INFO] [coordinator.go:402] [“create scheduler”] [scheduler-name=balance-leader-scheduler] [scheduler-args=“[]”]
[2023/03/17 11:18:19.368 +08:00] [INFO] [coordinator.go:402] [“create scheduler”] [scheduler-name=balance-witness-scheduler] [scheduler-args=“[]”]
[2023/03/17 11:18:19.370 +08:00] [INFO] [coordinator.go:402] [“create scheduler”] [scheduler-name=balance-hot-region-scheduler] [scheduler-args=“[]”]
[2023/03/17 11:18:19.372 +08:00] [INFO] [coordinator.go:402] [“create scheduler”] [scheduler-name=split-bucket-scheduler] [scheduler-args=“[]”]
[2023/03/17 11:18:19.374 +08:00] [INFO] [coordinator.go:402] [“create scheduler”] [scheduler-name=transfer-witness-leader-scheduler] [scheduler-args=“[]”]
[2023/03/17 11:18:19.376 +08:00] [INFO] [coordinator.go:291] [“coordinator begins to actively drive push operator”]
[2023/03/17 11:18:19.376 +08:00] [INFO] [coordinator.go:124] [“coordinator starts patrol regions”]
[2023/03/17 11:18:19.376 +08:00] [INFO] [coordinator.go:227] [“coordinator begins to check suspect key ranges”]
[2023/03/17 11:18:20.351 +08:00] [INFO] [hot_region_config.go:441] [“query supported changed”] [last-query-support=false] [cluster-version=6.6.0] [config=“{"min-hot-byte-rate":100,"min-hot-key-rate":10,"min-hot-query-rate":10,"max-zombie-rounds":3,"max-peer-number":1000,"byte-rate-rank-step-ratio":0.05,"key-rate-rank-step-ratio":0.05,"query-rate-rank-step-ratio":0.05,"count-rank-step-ratio":0.01,"great-dec-ratio":0.95,"minor-dec-ratio":0.99,"src-tolerance-ratio":1.05,"dst-tolerance-ratio":1.05,"write-leader-priorities":["query","byte"],"write-peer-priorities":["byte","key"],"read-priorities":["query","byte"],"strict-picking-store":"true","enable-for-tiflash":"true","rank-formula-version":"v2","forbid-rw-type":"none"}”] [valid-config=“{"min-hot-byte-rate":100,"min-hot-key-rate":10,"min-hot-query-rate":10,"max-zombie-rounds":3,"max-peer-number":1000,"byte-rate-rank-step-ratio":0.05,"key-rate-rank-step-ratio":0.05,"query-rate-rank-step-ratio":0.05,"count-rank-step-ratio":0.01,"great-dec-ratio":0.95,"minor-dec-ratio":0.99,"src-tolerance-ratio":1.05,"dst-tolerance-ratio":1.05,"write-leader-priorities":["key","byte"],"write-peer-priorities":["byte","key"],"read-priorities":["byte","key"],"strict-picking-store":"true","enable-for-tiflash":"true","rank-formula-version":"v2"}”]
[2023/03/17 11:22:22.996 +08:00] [INFO] [periodic.go:135] [“starting auto periodic compaction”] [revision=232260] [compact-period=1h0m0s]
[2023/03/17 11:22:22.999 +08:00] [INFO] [periodic.go:146] [“completed auto periodic compaction”] [revision=232260] [compact-period=1h0m0s] [took=1h36m0.012553066s]
[2023/03/17 11:22:22.999 +08:00] [INFO] [index.go:189] [“compact tree index”] [revision=232260]
[2023/03/17 11:22:23.016 +08:00] [INFO] [kvstore_compaction.go:56] [“finished scheduled compaction”] [compact-revision=232260] [took=16.143713ms]
[2023/03/17 11:27:23.790 +08:00] [INFO] [grpc_service.go:1451] [“update service GC safe point”] [service-id=gc_worker] [expire-at=-9223372035175752166] [safepoint=440145816780537856]
[2023/03/17 11:29:03.844 +08:00] [INFO] [grpc_service.go:1403] [“updated gc safe point”] [safe-point=440145816780537856]
[2023/03/17 11:37:23.792 +08:00] [INFO] [grpc_service.go:1451] [“update service GC safe point”] [service-id=gc_worker] [expire-at=-9223372035175751566] [safepoint=440145974066937856]
[2023/03/17 11:39:03.841 +08:00] [INFO] [grpc_service.go:1403] [“updated gc safe point”] [safe-point=440145974066937856]
[2023/03/17 11:40:39.438 +08:00] [WARN] [tso.go:331] [“clock offset”] [jet-lag=189.241521ms] [prev-physical=2023/03/17 11:40:39.248 +08:00] [now=2023/03/17 11:40:39.438 +08:00] [update-physical-interval=50ms]
[2023/03/17 11:47:23.789 +08:00] [INFO] [grpc_service.go:1451] [“update service GC safe point”] [service-id=gc_worker] [expire-at=-9223372035175750966] [safepoint=440146131353337856]
[2023/03/17 11:47:50.502 +08:00] [WARN] [tso.go:331] [“clock offset”] [jet-lag=152.879481ms] [prev-physical=2023/03/17 11:47:50.349 +08:00] [now=2023/03/17 11:47:50.502 +08:00] [update-physical-interval=50ms]
[2023/03/17 11:49:03.839 +08:00] [INFO] [grpc_service.go:1403] [“updated gc safe point”] [safe-point=440146131353337856]
[2023/03/17 11:58:23.799 +08:00] [INFO] [grpc_service.go:1451] [“update service GC safe point”] [service-id=gc_worker] [expire-at=-9223372035175750306] [safepoint=440146304368115712]
[2023/03/17 12:00:03.851 +08:00] [INFO] [grpc_service.go:1403] [“updated gc safe point”] [safe-point=440146304368115712]
[2023/03/17 12:08:23.788 +08:00] [INFO] [grpc_service.go:1451] [“update service GC safe point”] [service-id=gc_worker] [expire-at=-9223372035175749706] [safepoint=440146461654777856]
[2023/03/17 12:10:03.837 +08:00] [INFO] [grpc_service.go:1403] [“updated gc safe point”] [safe-point=440146461654777856]

Jellybean · 2023 年3 月 17 日 11:35

看日志里面很多网络相关的链接错误，检查过pd节点直接的网络连接了吗？部署用户的ssh 免密通信是否正常？

[ERROR] [client.go:163] [“server failed to establish sync stream with leader”] [server=pd-192.168.99.227-2379] [leader=pd-192.168.99.228-2379] [error=“rpc error: code = Unavailable desc = connection error: desc = “transport: Error while dialing dial tcp 192.168.99.228:2379: connect: connection refused””]
[/quote]

Running · 2023 年3 月 18 日 09:58

感觉不通信导致的，都用到6.7了，好用吗？

liuis · 2023 年3 月 18 日 11:50

节点都是联通的吗