【 TiDB 使用环境】测试
【 TiDB 版本】V6.5 operator
【复现路径】node服务器是一个不互通的dns地址
【遇到的问题：问题现象及影响】
Server: 10.43.0.10
Address: 10.43.0.10#53
Name: tidb-pd-1.tidb-pd-peer.infra.svc.cluster.local
Address: 10.42.2.9
nslookup domain tidb-pd-1.tidb-pd-peer.infra.svc.svc success
waiting for discovery service to return start args …
waiting for discovery service to return start args …
waiting for discovery service to return start args …
starting pd-server …
/pd-server --data-dir=/var/lib/pd --name=tidb-pd-1 --peer-urls=http://0.0.0.0:2380 --advertise-peer-urls=http://tidb-pd-1.tidb-pd-peer.infra.svc:2380 --client-urls=http://0.0.0.0:2379 --advertise-client-urls=http://tidb-pd-1.tidb-pd-peer.infra.svc:2379 --config=/etc/pd/pd.toml --join=http://tidb-pd-2.tidb-pd-peer.infra.svc:2379,http://tidb-pd-0.tidb-pd-peer.infra.svc:2379
[2023/12/04 07:54:54.566 +00:00] [INFO] [util.go:41] [“Welcome to Placement Driver (PD)”]
[2023/12/04 07:54:54.566 +00:00] [INFO] [util.go:42] [PD] [release-version=v6.5.0]
[2023/12/04 07:54:54.566 +00:00] [INFO] [util.go:43] [PD] [edition=Community]
[2023/12/04 07:54:54.566 +00:00] [INFO] [util.go:44] [PD] [git-hash=d1a4433c3126c77fb2d5bb5720eefa0f2e05c166]
[2023/12/04 07:54:54.566 +00:00] [INFO] [util.go:45] [PD] [git-branch=heads/refs/tags/v6.5.0]
[2023/12/04 07:54:54.566 +00:00] [INFO] [util.go:46] [PD] [utc-build-time=“2022-12-16 08:19:07”]
[2023/12/04 07:54:54.566 +00:00] [INFO] [metricutil.go:83] [“disable Prometheus push client”]
[2023/12/04 07:54:54.566 +00:00] [INFO] [join.go:218] [“failed to open directory, maybe start for the first time”] [error=“open /var/lib/pd/member: no such file or directory”]
[2023/12/04 07:54:54.582 +00:00] [INFO] [server.go:247] [“PD Config”] [config=“{"client-urls":"http://0.0.0.0:2379","peer-urls":"http://0.0.0.0:2380","advertise-client-urls":"http://tidb-pd-1.tidb-pd-peer.infra.svc:2379","advertise-peer-urls":"http://tidb-pd-1.tidb-pd-peer.infra.svc:2380","name":"tidb-pd-1","data-dir":"/var/lib/pd","force-new-cluster":false,"enable-grpc-gateway":true,"initial-cluster":"tidb-pd-2=http://tidb-pd-2.tidb-pd-peer.infra.svc:2380,tidb-pd-1=http://tidb-pd-1.tidb-pd-peer.infra.svc:2380,tidb-pd-0=http://tidb-pd-0.tidb-pd-peer.infra.svc:2380","initial-cluster-state":"existing","initial-cluster-token":"pd-cluster","join":"http://tidb-pd-2.tidb-pd-peer.infra.svc:2379,http://tidb-pd-0.tidb-pd-peer.infra.svc:2379","lease":3,"log":{"level":"info","format":"text","disable-timestamp":false,"file":{"filename":"","max-size":0,"max-days":0,"max-backups":0},"development":false,"disable-caller":false,"disable-stacktrace":false,"disable-error-verbose":true,"sampling":null,"error-output-path":""},"tso-save-interval":"3s","tso-update-physical-interval":"50ms","enable-local-tso":false,"metric":{"job":"tidb-pd-1","address":"","interval":"15s"},"schedule":{"max-snapshot-count":64,"max-pending-peer-count":64,"max-merge-region-size":20,"max-merge-region-keys":0,"split-merge-interval":"1h0m0s","swtich-witness-interval":"1h0m0s","enable-one-way-merge":"false","enable-cross-table-merge":"true","patrol-region-interval":"10ms","max-store-down-time":"30m0s","max-store-preparing-time":"48h0m0s","leader-schedule-limit":4,"leader-schedule-policy":"count","region-schedule-limit":2048,"replica-schedule-limit":64,"merge-schedule-limit":8,"hot-region-schedule-limit":4,"hot-region-cache-hits-threshold":3,"store-limit":{},"tolerant-size-ratio":0,"low-space-ratio":0.8,"high-space-ratio":0.7,"region-score-formula-version":"v2","scheduler-max-waiting-operator":5,"enable-remove-down-replica":"true","enable-replace-offline-replica":"true","enable-make-up-replica":"true","enable-remove-extra-replica":"true","enable-location-replacement":"true","enable-debug-metrics":"false","enable-joint-consensus":"true","enable-tikv-split-region":"true","schedulers-v2":[{"type":"balance-region","args":null,"disable":false,"args-payload":""},{"type":"balance-leader","args":null,"disable":false,"args-payload":""},{"type":"hot-region","args":null,"disable":false,"args-payload":""},{"type":"split-bucket","args":null,"disable":false,"args-payload":""}],"schedulers-payload":null,"store-limit-mode":"manual","hot-regions-write-interval":"10m0s","hot-regions-reserved-days":7,"enable-diagnostic":"false","enable-witness":"false"},"replication":{"max-replicas":3,"location-labels":"","strictly-match-label":"false","enable-placement-rules":"true","enable-placement-rules-cache":"false","isolation-level":""},"pd-server":{"use-region-storage":"true","max-gap-reset-ts":"24h0m0s","key-type":"table","runtime-services":"","metric-storage":"","dashboard-address":"auto","trace-region-flow":"true","flow-round-by-digit":3,"min-resolved-ts-persistence-interval":"1s"},"cluster-version":"0.0.0","labels":{},"quota-backend-bytes":"8GiB","auto-compaction-mode":"periodic","auto-compaction-retention-v2":"1h","TickInterval":"500ms","ElectionInterval":"3s","PreVote":true,"max-request-bytes":157286400,"security":{"cacert-path":"","cert-path":"","key-path":"","cert-allowed-cn":null,"SSLCABytes":null,"SSLCertBytes":null,"SSLKEYBytes":null,"redact-info-log":false,"encryption":{"data-encryption-method":"plaintext","data-key-rotation-period":"168h0m0s","master-key":{"type":"plaintext","key-id":"","region":"","endpoint":"","path":""}}},"label-property":null,"WarningMsgs":null,"DisableStrictReconfigCheck":false,"HeartbeatStreamBindInterval":"1m0s","LeaderPriorityCheckInterval":"1m0s","dashboard":{"tidb-cacert-path":"","tidb-cert-path":"","tidb-key-path":"","public-path-prefix":"","internal-proxy":false,"enable-telemetry":true,"enable-experimental":false},"replication-mode":{"replication-mode":"majority","dr-auto-sync":{"label-key":"","primary":"","dr":"","primary-replicas":0,"dr-replicas":0,"wait-store-timeout":"1m0s","pause-region-split":"false"}}}”]
[2023/12/04 07:54:54.595 +00:00] [INFO] [server.go:222] [“register REST path”] [path=/pd/api/v1]
[2023/12/04 07:54:54.596 +00:00] [INFO] [server.go:222] [“register REST path”] [path=/pd/api/v2/]
[2023/12/04 07:54:54.596 +00:00] [INFO] [server.go:222] [“register REST path”] [path=/swagger/]
[2023/12/04 07:54:54.596 +00:00] [INFO] [server.go:222] [“register REST path”] [path=/autoscaling]
[2023/12/04 07:54:54.596 +00:00] [INFO] [distro.go:51] [“Using distribution strings”] [strings={}]
[2023/12/04 07:54:54.604 +00:00] [INFO] [server.go:222] [“register REST path”] [path=/dashboard/api/]
[2023/12/04 07:54:54.604 +00:00] [INFO] [server.go:222] [“register REST path”] [path=/dashboard/]
[2023/12/04 07:54:54.604 +00:00] [INFO] [systimemon.go:28] [“start system time monitor”]
[2023/12/04 07:54:54.604 +00:00] [INFO] [etcd.go:117] [“configuring peer listeners”] [listen-peer-urls=“[http://0.0.0.0:2380]”]
[2023/12/04 07:54:54.605 +00:00] [INFO] [etcd.go:127] [“configuring client listeners”] [listen-client-urls=“[http://0.0.0.0:2379]”]
[2023/12/04 07:54:54.605 +00:00] [INFO] [etcd.go:611] [“pprof is enabled”] [path=/debug/pprof]
[2023/12/04 07:54:54.605 +00:00] [INFO] [etcd.go:305] [“starting an etcd server”] [etcd-version=3.4.21] [git-sha=“Not provided (use ./build instead of go build)”] [go-version=go1.19.3] [go-os=linux] [go-arch=amd64] [max-cpu-set=12] [max-cpu-available=12] [member-initialized=false] [name=tidb-pd-1] [data-dir=/var/lib/pd] [wal-dir=] [wal-dir-dedicated=] [member-dir=/var/lib/pd/member] [force-new-cluster=false] [heartbeat-interval=500ms] [election-timeout=3s] [initial-election-tick-advance=true] [snapshot-count=100000] [snapshot-catchup-entries=5000] [initial-advertise-peer-urls=“[http://tidb-pd-1.tidb-pd-peer.infra.svc:2380]”] [listen-peer-urls=“[http://0.0.0.0:2380]”] [advertise-client-urls=“[http://tidb-pd-1.tidb-pd-peer.infra.svc:2379]”] [listen-client-urls=“[http://0.0.0.0:2379]”] [listen-metrics-urls=“[]”] [cors=“[]“] [host-whitelist=”[]”] [initial-cluster=“tidb-pd-0=http://tidb-pd-0.tidb-pd-peer.infra.svc:2380,tidb-pd-1=http://tidb-pd-1.tidb-pd-peer.infra.svc:2380,tidb-pd-2=http://tidb-pd-2.tidb-pd-peer.infra.svc:2380”] [initial-cluster-state=existing] [initial-cluster-token=pd-cluster] [quota-backend-bytes=8589934592] [max-request-bytes=157286400] [max-concurrent-streams=4294967295] [pre-vote=true] [initial-corrupt-check=false] [corrupt-check-time-interval=0s] [auto-compaction-mode=periodic] [auto-compaction-retention=1h0m0s] [auto-compaction-interval=1h0m0s] [discovery-url=] [discovery-proxy=]
[2023/12/04 07:54:54.605 +00:00] [WARN] [server.go:297] [“exceeded recommended request limit”] [max-request-bytes=157286400] [max-request-size=“157 MB”] [recommended-request-bytes=10485760] [recommended-request-size=“10 MB”]
2023-12-04 07:54:54.605541 W | pkg/fileutil: check file permission: directory “/var/lib/pd” exist, but the permission is “drwxrwxrwx”. The recommended permission is “-rwx------” to prevent possible unprivileged access to the data.
[2023/12/04 07:54:54.608 +00:00] [INFO] [backend.go:80] [“opened backend db”] [path=/var/lib/pd/member/snap/db] [took=2.803511ms]
[2023/12/04 07:54:54.615 +00:00] [INFO] [netutil.go:112] [“resolved URL Host”] [url=http://tidb-pd-2.tidb-pd-peer.infra.svc:2380] [host=tidb-pd-2.tidb-pd-peer.infra.svc:2380] [resolved-addr=10.42.1.9:2380]
[2023/12/04 07:54:54.617 +00:00] [INFO] [netutil.go:112] [“resolved URL Host”] [url=http://tidb-pd-1.tidb-pd-peer.infra.svc:2380] [host=tidb-pd-1.tidb-pd-peer.infra.svc:2380] [resolved-addr=10.42.2.9:2380]
[2023/12/04 07:54:54.619 +00:00] [INFO] [netutil.go:112] [“resolved URL Host”] [url=http://tidb-pd-2.tidb-pd-peer.infra.svc:2380] [host=tidb-pd-2.tidb-pd-peer.infra.svc:2380] [resolved-addr=10.42.1.9:2380]
[2023/12/04 07:54:54.621 +00:00] [INFO] [netutil.go:112] [“resolved URL Host”] [url=http://tidb-pd-2.tidb-pd-peer.infra.svc:2380] [host=tidb-pd-2.tidb-pd-peer.infra.svc:2380] [resolved-addr=10.42.1.9:2380]
[2023/12/04 07:54:54.622 +00:00] [INFO] [netutil.go:112] [“resolved URL Host”] [url=http://tidb-pd-1.tidb-pd-peer.infra.svc:2380] [host=tidb-pd-1.tidb-pd-peer.infra.svc:2380] [resolved-addr=10.42.2.9:2380]
[2023/12/04 07:54:54.625 +00:00] [INFO] [netutil.go:112] [“resolved URL Host”] [url=http://tidb-pd-1.tidb-pd-peer.infra.svc:2380] [host=tidb-pd-1.tidb-pd-peer.infra.svc:2380] [resolved-addr=10.42.2.9:2380]
[2023/12/04 07:54:54.628 +00:00] [INFO] [netutil.go:112] [“resolved URL Host”] [url=http://tidb-pd-0.tidb-pd-peer.infra.svc:2380] [host=tidb-pd-0.tidb-pd-peer.infra.svc:2380] [resolved-addr=10.42.0.8:2380]
[2023/12/04 07:54:54.631 +00:00] [INFO] [netutil.go:112] [“resolved URL Host”] [url=http://tidb-pd-1.tidb-pd-peer.infra.svc:2380] [host=tidb-pd-1.tidb-pd-peer.infra.svc:2380] [resolved-addr=10.42.2.9:2380]
[2023/12/04 07:54:54.639 +00:00] [INFO] [netutil.go:112] [“resolved URL Host”] [url=http://tidb-pd-0.tidb-pd-peer.infra.svc:2380] [host=tidb-pd-0.tidb-pd-peer.infra.svc:2380] [resolved-addr=10.42.0.8:2380]
[2023/12/04 07:54:54.641 +00:00] [INFO] [netutil.go:112] [“resolved URL Host”] [url=http://tidb-pd-2.tidb-pd-peer.infra.svc:2380] [host=tidb-pd-2.tidb-pd-peer.infra.svc:2380] [resolved-addr=10.42.1.9:2380]
[2023/12/04 07:54:54.642 +00:00] [INFO] [netutil.go:112] [“resolved URL Host”] [url=http://tidb-pd-0.tidb-pd-peer.infra.svc:2380] [host=tidb-pd-0.tidb-pd-peer.infra.svc:2380] [resolved-addr=10.42.0.8:2380]
[2023/12/04 07:54:54.644 +00:00] [INFO] [netutil.go:112] [“resolved URL Host”] [url=http://tidb-pd-0.tidb-pd-peer.infra.svc:2380] [host=tidb-pd-0.tidb-pd-peer.infra.svc:2380] [resolved-addr=10.42.0.8:2380]
[2023/12/04 07:54:54.654 +00:00] [INFO] [raft.go:536] [“starting local member”] [local-member-id=2dba6c0d24356751] [cluster-id=34d0e6766efa769f]
[2023/12/04 07:54:54.654 +00:00] [INFO] [raft.go:1523] [“2dba6c0d24356751 switched to configuration voters=()”]
[2023/12/04 07:54:54.654 +00:00] [INFO] [raft.go:706] [“2dba6c0d24356751 became follower at term 0”]
[2023/12/04 07:54:54.654 +00:00] [INFO] [raft.go:389] [“newRaft 2dba6c0d24356751 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]”]
[2023/12/04 07:54:54.658 +00:00] [WARN] [store.go:1379] [“simple token is not cryptographically signed”]
[2023/12/04 07:54:54.662 +00:00] [INFO] [quota.go:126] [“enabled backend quota”] [quota-name=v3-applier] [quota-size-bytes=8589934592] [quota-size=“8.6 GB”]
[2023/12/04 07:54:54.663 +00:00] [INFO] [pipeline.go:71] [“started HTTP pipelining with remote peer”] [local-member-id=2dba6c0d24356751] [remote-peer-id=2d2aa41c0a8b966c]
[2023/12/04 07:54:54.663 +00:00] [INFO] [transport.go:294] [“added new remote peer”] [local-member-id=2dba6c0d24356751] [remote-peer-id=2d2aa41c0a8b966c] [remote-peer-urls=“[http://tidb-pd-2.tidb-pd-peer.infra.svc:2380]”]
[2023/12/04 07:54:54.663 +00:00] [INFO] [pipeline.go:71] [“started HTTP pipelining with remote peer”] [local-member-id=2dba6c0d24356751] [remote-peer-id=dff99a14cf072f6b]
[2023/12/04 07:54:54.663 +00:00] [INFO] [transport.go:294] [“added new remote peer”] [local-member-id=2dba6c0d24356751] [remote-peer-id=dff99a14cf072f6b] [remote-peer-urls=“[http://tidb-pd-0.tidb-pd-peer.infra.svc:2380]”]
【资源配置】
【附件：截图/日志/监控】
