1 目前整个集群重启后正常使用
还有一个报警持续:
increase(tidb_server_panic_total[10m]) > 0
{instance="192.168.254.32:9384",job="tidb",type="analyze"} ==== 196.9230769230769
2 没有fatal日志, 有panic关键字 但是是info级别的日志 时间也不太对, 进程应该是活着 就是卡死了
[root@data11 log]# cat tikv.log| grep panic
[2021/04/14 07:48:49.189 +08:00] [INFO] [server.rs:212] ["using config"] [config="{\"log-level\":\"info\",\"log-file\":\"/data/tidb-deploy-v2/tikv-9385/log/tikv.log\",\"log-format\":\"text\",\"slow-log-file\":\"\",\"slow-log-threshold\":\"1s\",\"log-rotation-timespan\":\"1d\",\"log-rotation-size\":\"300MiB\",\"panic-when-unexpected-key-or-data\":false,\"readpool\":{\"unified\":{\"min-thread-count\":1,\"max-thread-count\":57,\"stack-size\":\"10MiB\",\"max-tasks-per-worker\":2000},\"storage\":{\"use-unified-pool\":false,\"high-concurrency\":8,\"normal-concurrency\":8,\"low-concurrency\":8,\"max-tasks-per-worker-high\":2000,\"max-tasks-per-worker-normal\":2000,\"max-tasks-per-worker-low\":2000,\"stack-size\":\"10MiB\"},\"coprocessor\":{\"use-unified-pool\":true,\"high-concurrency\":57,\"normal-concurrency\":57,\"low-concurrency\":57,\"max-tasks-per-worker-high\":2000,\"max-tasks-per-worker-normal\":2000,\"max-tasks-per-worker-low\":2000,\"stack-size\":\"10MiB\"}},\"server\":{\"addr\":\"0.0.0.0:9385\",\"advertise-addr\":\"192.168.254.13:9385\",\"status-addr\":\"0.0.0.0:9386\",\"advertise-status-addr\":\"192.168.254.13:9386\",\"status-thread-pool-size\":1,\"max-grpc-send-msg-len\":10485760,\"grpc-compression-type\":\"none\",\"grpc-concurrency\":4,\"grpc-concurrent-stream\":1024,\"grpc-raft-conn-num\":1,\"grpc-memory-pool-quota\":9223372036854775807,\"grpc-stream-initial-window-size\":\"2MiB\",\"grpc-keepalive-time\":\"10s\",\"grpc-keepalive-timeout\":\"3s\",\"concurrent-send-snap-limit\":32,\"concurrent-recv-snap-limit\":32,\"end-point-recursion-limit\":1000,\"end-point-stream-channel-size\":8,\"end-point-batch-row-limit\":64,\"end-point-stream-batch-row-limit\":128,\"end-point-enable-batch-if-possible\":true,\"end-point-request-max-handle-duration\":\"1m\",\"end-point-max-concurrency\":72,\"snap-max-write-bytes-per-sec\":\"100MiB\",\"snap-max-total-size\":\"0KiB\",\"stats-concurrency\":1,\"heavy-load-threshold\":300,\"heavy-load-wait-duration\":\"1ms\",\"enable-request-batch\":true,\"request-batch-enable-cross-command\":false,\"request-batch-wait-duration\":\"1ms\",\"end-point-slow-log-threshold\":\"1s\",\"labels\":{}},\"storage\":{\"data-dir\":\"/data/tidb-data-v2/tikv-9385\",\"gc-ratio-threshold\":1.1,\"max-key-size\":4096,\"scheduler-concurrency\":524288,\"scheduler-worker-pool-size\":8,\"scheduler-pending-write-threshold\":\"100MiB\",\"reserve-space\":\"2GiB\",\"block-cache\":{\"shared\":true,\"capacity\":\"20GiB\",\"num-shard-bits\":6,\"strict-capacity-limit\":false,\"high-pri-pool-ratio\":0.8,\"memory-allocator\":\"nodump\"}},\"pd\":{\"endpoints\":[\"192.168.254.13:9379\",\"192.168.254.31:9379\",\"192.168.254.32:9379\"],\"retry-interval\":\"300ms\",\"retry-max-count\":9223372036854775807,\"retry-log-every\":10,\"update-interval\":\"10m\"},\"metric\":{\"interval\":\"15s\",\"address\":\"\",\"job\":\"tikv\"},\"raftstore\":{\"sync-log\":true,\"prevote\":true,\"raftdb-path\":\"/data/tidb-data-v2/tikv-9385/raft\",\"capacity\":\"0KiB\",\"raft-base-tick-interval\":\"1s\",\"raft-heartbeat-ticks\":2,\"raft-election-timeout-ticks\":10,\"raft-min-election-timeout-ticks\":10,\"raft-max-election-timeout-ticks\":20,\"raft-max-size-per-msg\":\"1MiB\",\"raft-max-inflight-msgs\":256,\"raft-entry-max-size\":\"8MiB\",\"raft-log-gc-tick-interval\":\"10s\",\"raft-log-gc-threshold\":50,\"raft-log-gc-count-limit\":73728,\"raft-log-gc-size-limit\":\"72MiB\",\"raft-entry-cache-life-time\":\"30s\",\"raft-reject-transfer-leader-duration\":\"3s\",\"split-region-check-tick-interval\":\"10s\",\"region-split-check-diff\":\"6MiB\",\"region-compact-check-interval\":\"5m\",\"region-compact-check-step\":100,\"region-compact-min-tombstones\":10000,\"region-compact-tombstones-percent\":30,\"pd-heartbeat-tick-interval\":\"1m\",\"pd-store-heartbeat-tick-interval\":\"10s\",\"snap-mgr-gc-tick-interval\":\"1m\",\"snap-gc-timeout\":\"4h\",\"lock-cf-compact-interval\":\"10m\",\"lock-cf-compact-bytes-threshold\":\"256MiB\",\"notify-capacity\":40960,\"messages-per-tick\":4096,\"max-peer-down-duration\":\"5m\",\"max-leader-missing-duration\":\"2h\",\"abnormal-leader-missing-duration\":\"10m\",\"peer-stale-state-check-interval\":\"5m\",\"leader-transfer-max-log-lag\":128,\"snap-apply-batch-size\":\"10MiB\",\"consistency-check-interval\":\"0s\",\"report-region-flow-interval\":\"1m\",\"raft-store-max-leader-lease\":\"9s\",\"right-derive-when-split\":true,\"allow-remove-leader\":false,\"merge-max-log-gap\":10,\"merge-check-tick-interval\":\"10s\",\"use-delete-range\":false,\"cleanup-import-sst-interval\":\"10m\",\"local-read-batch-size\":1024,\"apply-max-batch-size\":1024,\"apply-pool-size\":2,\"apply-reschedule-duration\":\"5s\",\"store-max-batch-size\":1024,\"store-pool-size\":2,\"store-reschedule-duration\":\"5s\",\"future-poll-size\":1,\"hibernate-regions\":false,\"hibernate-timeout\":\"10m\",\"early-apply\":true,\"dev-assert\":false,\"apply-yield-duration\":\"500ms\",\"perf-level\":1},\"coprocessor\":{\"split-region-on-table\":false,\"batch-split-limit\":10,\"region-max-size\":\"144MiB\",\"region-split-size\":\"96MiB\",\"region-max-keys\":1440000,\"region-split-keys\":960000},\"rocksdb\":{\"wal-recovery-mode\":2,\"wal-dir\":\"\",\"wal-ttl-seconds\":0,\"wal-size-limit\":\"0KiB\",\"max-total-wal-size\":\"4GiB\",\"max-background-jobs\":8,\"max-background-flushes\":2,\"max-manifest-file-size\":\"128MiB\",\"create-if-missing\":true,\"max-open-files\":40960,\"enable-statistics\":true,\"stats-dump-period\":\"10m\",\"compaction-readahead-size\":\"0KiB\",\"info-log-max-size\":\"1GiB\",\"info-log-roll-time\":\"0s\",\"info-log-keep-log-file-num\":10,\"info-log-dir\":\"\",\"rate-bytes-per-sec\":\"0KiB\",\"rate-limiter-mode\":2,\"auto-tuned\":false,\"bytes-per-sync\":\"1MiB\",\"wal-bytes-per-sync\":\"512KiB\",\"max-sub-compactions\":3,\"writable-file-max-buffer-size\":\"1MiB\",\"use-direct-io-for-flush-and-compaction\":false,\"enable-pipelined-write\":true,\"enable-multi-batch-write\":true,\"enable-unordered-write\":false,\"defaultcf\":{\"block-size\":\"64KiB\",\"block-cache-size\":\"47962MiB\",\"disable-block-cache\":false,\"cache-index-and-filter-blocks\":true,\"pin-l0-filter-and-index-blocks\":true,\"use-bloom-filter\":true,\"optimize-filters-for-hits\":true,\"whole-key-filtering\":true,\"bloom-filter-bits-per-key\":10,\"block-based-bloom-filter\":false,\"read-amp-bytes-per-bit\":0,\"compression-per-level\":[\"no\",\"no\",\"lz4\",\"lz4\",\"lz4\",\"zstd\",\"zstd\"],\"write-buffer-size\":\"128MiB\",\"max-write-buffer-number\":5,\"min-write-buffer-number-to-merge\":1,\"max-bytes-for-level-base\":\"512MiB\",\"target-file-size-base\":\"8MiB\",\"level0-file-num-compaction-trigger\":4,\"level0-slowdown-writes-trigger\":20,\"level0-stop-writes-trigger\":36,\"max-compaction-bytes\":\"2GiB\",\"compaction-pri\":3,\"dynamic-level-bytes\":true,\"num-levels\":7,\"max-bytes-for-level-multiplier\":10,\"compaction-style\":0,\"disable-auto-compactions\":false,\"soft-pending-compaction-bytes-limit\":\"64GiB\",\"hard-pending-compaction-bytes-limit\":\"256GiB\",\"force-consistency-checks\":false,\"prop-size-index-distance\":4194304,\"prop-keys-index-distance\":40960,\"enable-doubly-skiplist\":true,\"titan\":{\"min-blob-size\":\"1KiB\",\"blob-file-compression\":\"lz4\",\"blob-cache-size\":\"0KiB\",\"min-gc-batch-size\":\"16MiB\",\"max-gc-batch-size\":\"64MiB\",\"discardable-ratio\":0.5,\"sample-ratio\":0.1,\"merge-small-file-threshold\":\"8MiB\",\"blob-run-mode\":\"normal\",\"level-merge\":false,\"range-merge\":true,\"max-sorted-runs\":20,\"gc-merge-rewrite\":false}},\"writecf\":{\"block-size\":\"64KiB\",\"block-cache-size\":\"28777MiB\",\"disable-block-cache\":false,\"cache-index-and-filter-blocks\":true,\"pin-l0-filter-and-index-blocks\":true,\"use-bloom-filter\":true,\"optimize-filters-for-hits\":false,\"whole-key-filtering\":false,\"bloom-filter-bits-per-key\":10,\"block-based-bloom-filter\":false,\"read-amp-bytes-per-bit\":0,\"compression-per-level\":[\"no\",\"no\",\"lz4\",\"lz4\",\"lz4\",\"zstd\",\"zstd\"],\"write-buffer-size\":\"128MiB\",\"max-write-buffer-number\":5,\"min-write-buffer-number-to-merge\":1,\"max-bytes-for-level-base\":\"512MiB\",\"target-file-size-base\":\"8MiB\",\"level0-file-num-compaction-trigger\":4,\"level0-slowdown-writes-trigger\":20,\"level0-stop-writes-trigger\":36,\"max-compaction-bytes\":\"2GiB\",\"compaction-pri\":3,\"dynamic-level-bytes\":true,\"num-levels\":7,\"max-bytes-for-level-multiplier\":10,\"compaction-style\":0,\"disable-auto-compactions\":false,\"soft-pending-compaction-bytes-limit\":\"64GiB\",\"hard-pending-compaction-bytes-limit\":\"256GiB\",\"force-consistency-checks\":false,\"prop-size-index-distance\":4194304,\"prop-keys-index-distance\":40960,\"enable-doubly-skiplist\":true,\"titan\":{\"min-blob-size\":\"1KiB\",\"blob-file-compression\":\"lz4\",\"blob-cache-size\":\"0KiB\",\"min-gc-batch-size\":\"16MiB\",\"max-gc-batch-size\":\"64MiB\",\"discardable-ratio\":0.5,\"sample-ratio\":0.1,\"merge-small-file-threshold\":\"8MiB\",\"blob-run-mode\":\"read-only\",\"level-merge\":false,\"range-merge\":true,\"max-sorted-runs\":20,\"gc-merge-rewrite\":false}},\"lockcf\":{\"block-size\":\"16KiB\",\"block-cache-size\":\"1GiB\",\"disable-block-cache\":false,\"cache-index-and-filter-blocks\":true,\"pin-l0-filter-and-index-blocks\":true,\"use-bloom-filter\":true,\"optimize-filters-for-hits\":false,\"whole-key-filtering\":true,\"bloom-filter-bits-per-key\":10,\"block-based-bloom-filter\":false,\"read-amp-bytes-per-bit\":0,\"compression-per-level\":[\"no\",\"no\",\"no\",\"no\",\"no\",\"no\",\"no\"],\"write-buffer-size\":\"32MiB\",\"max-write-buffer-number\":5,\"min-write-buffer-number-to-merge\":1,\"max-bytes-for-level-base\":\"128MiB\",\"target-file-size-base\":\"8MiB\",\"level0-file-num-compaction-trigger\":1,\"level0-slowdown-writes-trigger\":20,\"level0-stop-writes-trigger\":36,\"max-compaction-bytes\":\"2GiB\",\"compaction-pri\":0,\"dynamic-level-bytes\":true,\"num-levels\":7,\"max-bytes-for-level-multiplier\":10,\"compaction-style\":0,\"disable-auto-compactions\":false,\"soft-pending-compaction-bytes-limit\":\"64GiB\",\"hard-pending-compaction-bytes-limit\":\"256GiB\",\"force-consistency-checks\":false,\"prop-size-index-distance\":4194304,\"prop-keys-index-distance\":40960,\"enable-doubly-skiplist\":true,\"titan\":{\"min-blob-size\":\"1KiB\",\"blob-file-compression\":\"lz4\",\"blob-cache-size\":\"0KiB\",\"min-gc-batch-size\":\"16MiB\",\"max-gc-batch-size\":\"64MiB\",\"discardable-ratio\":0.5,\"sample-ratio\":0.1,\"merge-small-file-threshold\":\"8MiB\",\"blob-run-mode\":\"read-only\",\"level-merge\":false,\"range-merge\":true,\"max-sorted-runs\":20,\"gc-merge-rewrite\":false}},\"raftcf\":{\"block-size\":\"16KiB\",\"block-cache-size\":\"128MiB\",\"disable-block-cache\":false,\"cache-index-and-filter-blocks\":true,\"pin-l0-filter-and-index-blocks\":true,\"use-bloom-filter\":true,\"optimize-filters-for-hits\":true,\"whole-key-filtering\":true,\"bloom-filter-bits-per-key\":10,\"block-based-bloom-filter\":false,\"read-amp-bytes-per-bit\":0,\"compression-per-level\":[\"no\",\"no\",\"no\",\"no\",\"no\",\"no\",\"no\"],\"write-buffer-size\":\"128MiB\",\"max-write-buffer-number\":5,\"min-write-buffer-number-to-merge\":1,\"max-bytes-for-level-base\":\"128MiB\",\"target-file-size-base\":\"8MiB\",\"level0-file-num-compaction-trigger\":1,\"level0-slowdown-writes-trigger\":20,\"level0-stop-writes-trigger\":36,\"max-compaction-bytes\":\"2GiB\",\"compaction-pri\":0,\"dynamic-level-bytes\":true,\"num-levels\":7,\"max-bytes-for-level-multiplier\":10,\"compaction-style\":0,\"disable-auto-compactions\":false,\"soft-pending-compaction-bytes-limit\":\"64GiB\",\"hard-pending-compaction-bytes-limit\":\"256GiB\",\"force-consistency-checks\":false,\"prop-size-index-distance\":4194304,\"prop-keys-index-distance\":40960,\"enable-doubly-skiplist\":true,\"titan\":{\"min-blob-size\":\"1KiB\",\"blob-file-compression\":\"lz4\",\"blob-cache-size\":\"0KiB\",\"min-gc-batch-size\":\"16MiB\",\"max-gc-batch-size\":\"64MiB\",\"discardable-ratio\":0.5,\"sample-ratio\":0.1,\"merge-small-file-threshold\":\"8MiB\",\"blob-run-mode\":\"read-only\",\"level-merge\":false,\"range-merge\":true,\"max-sorted-runs\":20,\"gc-merge-rewrite\":false}},\"titan\":{\"enabled\":false,\"dirname\":\"\",\"disable-gc\":false,\"max-background-gc\":4,\"purge-obsolete-files-period\":\"10s\"}},\"raftdb\":{\"wal-recovery-mode\":2,\"wal-dir\":\"\",\"wal-ttl-seconds\":0,\"wal-size-limit\":\"0KiB\",\"max-total-wal-size\":\"4GiB\",\"max-background-jobs\":4,\"max-background-flushes\":1,\"max-manifest-file-size\":\"20MiB\",\"create-if-missing\":true,\"max-open-files\":40960,\"enable-statistics\":true,\"stats-dump-period\":\"10m\",\"compaction-readahead-size\":\"0KiB\",\"info-log-max-size\":\"1GiB\",\"info-log-roll-time\":\"0s\",\"info-log-keep-log-file-num\":10,\"info-log-dir\":\"\",\"max-sub-compactions\":2,\"writable-file-max-buffer-size\":\"1MiB\",\"use-direct-io-for-flush-and-compaction\":false,\"enable-pipelined-write\":true,\"enable-unordered-write\":false,\"allow-concurrent-memtable-write\":true,\"bytes-per-sync\":\"1MiB\",\"wal-bytes-per-sync\":\"512KiB\",\"defaultcf\":{\"block-size\":\"64KiB\",\"block-cache-size\":\"2GiB\",\"disable-block-cache\":false,\"cache-index-and-filter-blocks\":true,\"pin-l0-filter-and-index-blocks\":true,\"use-bloom-filter\":false,\"optimize-filters-for-hits\":true,\"whole-key-filtering\":true,\"bloom-filter-bits-per-key\":10,\"block-based-bloom-filter\":false,\"read-amp-bytes-per-bit\":0,\"compression-per-level\":[\"no\",\"no\",\"lz4\",\"lz4\",\"lz4\",\"zstd\",\"zstd\"],\"write-buffer-size\":\"128MiB\",\"max-write-buffer-number\":5,\"min-write-buffer-number-to-merge\":1,\"max-bytes-for-level-base\":\"512MiB\",\"target-file-size-base\":\"8MiB\",\"level0-file-num-compaction-trigger\":4,\"level0-slowdown-writes-trigger\":20,\"level0-stop-writes-trigger\":36,\"max-compaction-bytes\":\"2GiB\",\"compaction-pri\":0,\"dynamic-level-bytes\":true,\"num-levels\":7,\"max-bytes-for-level-multiplier\":10,\"compaction-style\":0,\"disable-auto-compactions\":false,\"soft-pending-compaction-bytes-limit\":\"64GiB\",\"hard-pending-compaction-bytes-limit\":\"256GiB\",\"force-consistency-checks\":false,\"prop-size-index-distance\":4194304,\"prop-keys-index-distance\":40960,\"enable-doubly-skiplist\":true,\"titan\":{\"min-blob-size\":\"1KiB\",\"blob-file-compression\":\"lz4\",\"blob-cache-size\":\"0KiB\",\"min-gc-batch-size\":\"16MiB\",\"max-gc-batch-size\":\"64MiB\",\"discardable-ratio\":0.5,\"sample-ratio\":0.1,\"merge-small-file-threshold\":\"8MiB\",\"blob-run-mode\":\"normal\",\"level-merge\":false,\"range-merge\":true,\"max-sorted-runs\":20,\"gc-merge-rewrite\":false}},\"titan\":{\"enabled\":false,\"dirname\":\"\",\"disable-gc\":false,\"max-background-gc\":4,\"purge-obsolete-files-period\":\"10s\"}},\"security\":{\"ca-path\":\"\",\"cert-path\":\"\",\"key-path\":\"\",\"cert-allowed-cn\":[],\"redact-info-log\":null,\"encryption\":{\"data-encryption-method\":\"plaintext\",\"data-key-rotation-period\":\"7d\",\"enable-file-dictionary-log\":true,\"file-dictionary-rewrite-threshold\":1000000,\"master-key\":{\"type\":\"plaintext\"},\"previous-master-key\":{\"type\":\"plaintext\"}}},\"import\":{\"num-threads\":8,\"stream-channel-window\":128},\"backup\":{\"num-threads\":32,\"batch-size\":8,\"sst-max-size\":\"144MiB\"},\"pessimistic-txn\":{\"enabled\":true,\"wait-for-lock-timeout\":\"1s\",\"wake-up-delay-duration\":\"20ms\",\"pipelined\":false},\"gc\":{\"ratio-threshold\":1.1,\"batch-keys\":512,\"max-write-bytes-per-sec\":\"0KiB\"},\"split\":{\"qps-threshold\":3000,\"split-balance-score\":0.25,\"split-contained-score\":0.5,\"detect-times\":10,\"sample-num\":20,\"sample-threshold\":100,\"size-threshold\":4194304,\"key-threshold\":40960},\"cdc\":{\"min-ts-interval\":\"1s\"}}"]
[2021/04/14 07:48:49.226 +08:00] [INFO] [server.rs:212] ["using config"] [config="{\"log-level\":\"info\",\"log-file\":\"/data/tidb-deploy-v2/tikv-9385/log/tikv.log\",\"log-format\":\"text\",\"slow-log-file\":\"\",\"slow-log-threshold\":\"1s\",\"log-rotation-timespan\":\"1d\",\"log-rotation-size\":\"300MiB\",\"panic-when-unexpected-key-or-data\":false,\"readpool\":{\"unified\":{\"min-thread-count\":1,\"max-thread-count\":57,\"stack-size\":\"10MiB\",\"max-tasks-per-worker\":2000},\"storage\":{\"use-unified-pool\":false,\"high-concurrency\":8,\"normal-concurrency\":8,\"low-concurrency\":8,\"max-tasks-per-worker-high\":2000,\"max-tasks-per-worker-normal\":2000,\"max-tasks-per-worker-low\":2000,\"stack-size\":\"10MiB\"},\"coprocessor\":{\"use-unified-pool\":true,\"high-concurrency\":57,\"normal-concurrency\":57,\"low-concurrency\":57,\"max-tasks-per-worker-high\":2000,\"max-tasks-per-worker-normal\":2000,\"max-tasks-per-worker-low\":2000,\"stack-size\":\"10MiB\"}},\"server\":{\"addr\":\"0.0.0.0:9385\",\"advertise-addr\":\"192.168.254.31:9385\",\"status-addr\":\"0.0.0.0:9386\",\"advertise-status-addr\":\"192.168.254.31:9386\",\"status-thread-pool-size\":1,\"max-grpc-send-msg-len\":10485760,\"grpc-compression-type\":\"none\",\"grpc-concurrency\":4,\"grpc-concurrent-stream\":1024,\"grpc-raft-conn-num\":1,\"grpc-memory-pool-quota\":9223372036854775807,\"grpc-stream-initial-window-size\":\"2MiB\",\"grpc-keepalive-time\":\"10s\",\"grpc-keepalive-timeout\":\"3s\",\"concurrent-send-snap-limit\":32,\"concurrent-recv-snap-limit\":32,\"end-point-recursion-limit\":1000,\"end-point-stream-channel-size\":8,\"end-point-batch-row-limit\":64,\"end-point-stream-batch-row-limit\":128,\"end-point-enable-batch-if-possible\":true,\"end-point-request-max-handle-duration\":\"1m\",\"end-point-max-concurrency\":72,\"snap-max-write-bytes-per-sec\":\"100MiB\",\"snap-max-total-size\":\"0KiB\",\"stats-concurrency\":1,\"heavy-load-threshold\":300,\"heavy-load-wait-duration\":\"1ms\",\"enable-request-batch\":true,\"request-batch-enable-cross-command\":false,\"request-batch-wait-duration\":\"1ms\",\"end-point-slow-log-threshold\":\"1s\",\"labels\":{}},\"storage\":{\"data-dir\":\"/data/tidb-data-v2/tikv-9385\",\"gc-ratio-threshold\":1.1,\"max-key-size\":4096,\"scheduler-concurrency\":524288,\"scheduler-worker-pool-size\":8,\"scheduler-pending-write-threshold\":\"100MiB\",\"reserve-space\":\"2GiB\",\"block-cache\":{\"shared\":true,\"capacity\":\"20GiB\",\"num-shard-bits\":6,\"strict-capacity-limit\":false,\"high-pri-pool-ratio\":0.8,\"memory-allocator\":\"nodump\"}},\"pd\":{\"endpoints\":[\"192.168.254.13:9379\",\"192.168.254.31:9379\",\"192.168.254.32:9379\"],\"retry-interval\":\"300ms\",\"retry-max-count\":9223372036854775807,\"retry-log-every\":10,\"update-interval\":\"10m\"},\"metric\":{\"interval\":\"15s\",\"address\":\"\",\"job\":\"tikv\"},\"raftstore\":{\"sync-log\":true,\"prevote\":true,\"raftdb-path\":\"/data/tidb-data-v2/tikv-9385/raft\",\"capacity\":\"0KiB\",\"raft-base-tick-interval\":\"1s\",\"raft-heartbeat-ticks\":2,\"raft-election-timeout-ticks\":10,\"raft-min-election-timeout-ticks\":10,\"raft-max-election-timeout-ticks\":20,\"raft-max-size-per-msg\":\"1MiB\",\"raft-max-inflight-msgs\":256,\"raft-entry-max-size\":\"8MiB\",\"raft-log-gc-tick-interval\":\"10s\",\"raft-log-gc-threshold\":50,\"raft-log-gc-count-limit\":73728,\"raft-log-gc-size-limit\":\"72MiB\",\"raft-entry-cache-life-time\":\"30s\",\"raft-reject-transfer-leader-duration\":\"3s\",\"split-region-check-tick-interval\":\"10s\",\"region-split-check-diff\":\"6MiB\",\"region-compact-check-interval\":\"5m\",\"region-compact-check-step\":100,\"region-compact-min-tombstones\":10000,\"region-compact-tombstones-percent\":30,\"pd-heartbeat-tick-interval\":\"1m\",\"pd-store-heartbeat-tick-interval\":\"10s\",\"snap-mgr-gc-tick-interval\":\"1m\",\"snap-gc-timeout\":\"4h\",\"lock-cf-compact-interval\":\"10m\",\"lock-cf-compact-bytes-threshold\":\"256MiB\",\"notify-capacity\":40960,\"messages-per-tick\":4096,\"max-peer-down-duration\":\"5m\",\"max-leader-missing-duration\":\"2h\",\"abnormal-leader-missing-duration\":\"10m\",\"peer-stale-state-check-interval\":\"5m\",\"leader-transfer-max-log-lag\":128,\"snap-apply-batch-size\":\"10MiB\",\"consistency-check-interval\":\"0s\",\"report-region-flow-interval\":\"1m\",\"raft-store-max-leader-lease\":\"9s\",\"right-derive-when-split\":true,\"allow-remove-leader\":false,\"merge-max-log-gap\":10,\"merge-check-tick-interval\":\"10s\",\"use-delete-range\":false,\"cleanup-import-sst-interval\":\"10m\",\"local-read-batch-size\":1024,\"apply-max-batch-size\":1024,\"apply-pool-size\":2,\"apply-reschedule-duration\":\"5s\",\"store-max-batch-size\":1024,\"store-pool-size\":2,\"store-reschedule-duration\":\"5s\",\"future-poll-size\":1,\"hibernate-regions\":false,\"hibernate-timeout\":\"10m\",\"early-apply\":true,\"dev-assert\":false,\"apply-yield-duration\":\"500ms\",\"perf-level\":1},\"coprocessor\":{\"split-region-on-table\":false,\"batch-split-limit\":10,\"region-max-size\":\"144MiB\",\"region-split-size\":\"96MiB\",\"region-max-keys\":1440000,\"region-split-keys\":960000},\"rocksdb\":{\"wal-recovery-mode\":2,\"wal-dir\":\"\",\"wal-ttl-seconds\":0,\"wal-size-limit\":\"0KiB\",\"max-total-wal-size\":\"4GiB\",\"max-background-jobs\":8,\"max-background-flushes\":2,\"max-manifest-file-size\":\"128MiB\",\"create-if-missing\":true,\"max-open-files\":40960,\"enable-statistics\":true,\"stats-dump-period\":\"10m\",\"compaction-readahead-size\":\"0KiB\",\"info-log-max-size\":\"1GiB\",\"info-log-roll-time\":\"0s\",\"info-log-keep-log-file-num\":10,\"info-log-dir\":\"\",\"rate-bytes-per-sec\":\"0KiB\",\"rate-limiter-mode\":2,\"auto-tuned\":false,\"bytes-per-sync\":\"1MiB\",\"wal-bytes-per-sync\":\"512KiB\",\"max-sub-compactions\":3,\"writable-file-max-buffer-size\":\"1MiB\",\"use-direct-io-for-flush-and-compaction\":false,\"enable-pipelined-write\":true,\"enable-multi-batch-write\":true,\"enable-unordered-write\":false,\"defaultcf\":{\"block-size\":\"64KiB\",\"block-cache-size\":\"47962MiB\",\"disable-block-cache\":false,\"cache-index-and-filter-blocks\":true,\"pin-l0-filter-and-index-blocks\":true,\"use-bloom-filter\":true,\"optimize-filters-for-hits\":true,\"whole-key-filtering\":true,\"bloom-filter-bits-per-key\":10,\"block-based-bloom-filter\":false,\"read-amp-bytes-per-bit\":0,\"compression-per-level\":[\"no\",\"no\",\"lz4\",\"lz4\",\"lz4\",\"zstd\",\"zstd\"],\"write-buffer-size\":\"128MiB\",\"max-write-buffer-number\":5,\"min-write-buffer-number-to-merge\":1,\"max-bytes-for-level-base\":\"512MiB\",\"target-file-size-base\":\"8MiB\",\"level0-file-num-compaction-trigger\":4,\"level0-slowdown-writes-trigger\":20,\"level0-stop-writes-trigger\":36,\"max-compaction-bytes\":\"2GiB\",\"compaction-pri\":3,\"dynamic-level-bytes\":true,\"num-levels\":7,\"max-bytes-for-level-multiplier\":10,\"compaction-style\":0,\"disable-auto-compactions\":false,\"soft-pending-compaction-bytes-limit\":\"64GiB\",\"hard-pending-compaction-bytes-limit\":\"256GiB\",\"force-consistency-checks\":false,\"prop-size-index-distance\":4194304,\"prop-keys-index-distance\":40960,\"enable-doubly-skiplist\":true,\"titan\":{\"min-blob-size\":\"1KiB\",\"blob-file-compression\":\"lz4\",\"blob-cache-size\":\"0KiB\",\"min-gc-batch-size\":\"16MiB\",\"max-gc-batch-size\":\"64MiB\",\"discardable-ratio\":0.5,\"sample-ratio\":0.1,\"merge-small-file-threshold\":\"8MiB\",\"blob-run-mode\":\"normal\",\"level-merge\":false,\"range-merge\":true,\"max-sorted-runs\":20,\"gc-merge-rewrite\":false}},\"writecf\":{\"block-size\":\"64KiB\",\"block-cache-size\":\"28777MiB\",\"disable-block-cache\":false,\"cache-index-and-filter-blocks\":true,\"pin-l0-filter-and-index-blocks\":true,\"use-bloom-filter\":true,\"optimize-filters-for-hits\":false,\"whole-key-filtering\":false,\"bloom-filter-bits-per-key\":10,\"block-based-bloom-filter\":false,\"read-amp-bytes-per-bit\":0,\"compression-per-level\":[\"no\",\"no\",\"lz4\",\"lz4\",\"lz4\",\"zstd\",\"zstd\"],\"write-buffer-size\":\"128MiB\",\"max-write-buffer-number\":5,\"min-write-buffer-number-to-merge\":1,\"max-bytes-for-level-base\":\"512MiB\",\"target-file-size-base\":\"8MiB\",\"level0-file-num-compaction-trigger\":4,\"level0-slowdown-writes-trigger\":20,\"level0-stop-writes-trigger\":36,\"max-compaction-bytes\":\"2GiB\",\"compaction-pri\":3,\"dynamic-level-bytes\":true,\"num-levels\":7,\"max-bytes-for-level-multiplier\":10,\"compaction-style\":0,\"disable-auto-compactions\":false,\"soft-pending-compaction-bytes-limit\":\"64GiB\",\"hard-pending-compaction-bytes-limit\":\"256GiB\",\"force-consistency-checks\":false,\"prop-size-index-distance\":4194304,\"prop-keys-index-distance\":40960,\"enable-doubly-skiplist\":true,\"titan\":{\"min-blob-size\":\"1KiB\",\"blob-file-compression\":\"lz4\",\"blob-cache-size\":\"0KiB\",\"min-gc-batch-size\":\"16MiB\",\"max-gc-batch-size\":\"64MiB\",\"discardable-ratio\":0.5,\"sample-ratio\":0.1,\"merge-small-file-threshold\":\"8MiB\",\"blob-run-mode\":\"read-only\",\"level-merge\":false,\"range-merge\":true,\"max-sorted-runs\":20,\"gc-merge-rewrite\":false}},\"lockcf\":{\"block-size\":\"16KiB\",\"block-cache-size\":\"1GiB\",\"disable-block-cache\":false,\"cache-index-and-filter-blocks\":true,\"pin-l0-filter-and-index-blocks\":true,\"use-bloom-filter\":true,\"optimize-filters-for-hits\":false,\"whole-key-filtering\":true,\"bloom-filter-bits-per-key\":10,\"block-based-bloom-filter\":false,\"read-amp-bytes-per-bit\":0,\"compression-per-level\":[\"no\",\"no\",\"no\",\"no\",\"no\",\"no\",\"no\"],\"write-buffer-size\":\"32MiB\",\"max-write-buffer-number\":5,\"min-write-buffer-number-to-merge\":1,\"max-bytes-for-level-base\":\"128MiB\",\"target-file-size-base\":\"8MiB\",\"level0-file-num-compaction-trigger\":1,\"level0-slowdown-writes-trigger\":20,\"level0-stop-writes-trigger\":36,\"max-compaction-bytes\":\"2GiB\",\"compaction-pri\":0,\"dynamic-level-bytes\":true,\"num-levels\":7,\"max-bytes-for-level-multiplier\":10,\"compaction-style\":0,\"disable-auto-compactions\":false,\"soft-pending-compaction-bytes-limit\":\"64GiB\",\"hard-pending-compaction-bytes-limit\":\"256GiB\",\"force-consistency-checks\":false,\"prop-size-index-distance\":4194304,\"prop-keys-index-distance\":40960,\"enable-doubly-skiplist\":true,\"titan\":{\"min-blob-size\":\"1KiB\",\"blob-file-compression\":\"lz4\",\"blob-cache-size\":\"0KiB\",\"min-gc-batch-size\":\"16MiB\",\"max-gc-batch-size\":\"64MiB\",\"discardable-ratio\":0.5,\"sample-ratio\":0.1,\"merge-small-file-threshold\":\"8MiB\",\"blob-run-mode\":\"read-only\",\"level-merge\":false,\"range-merge\":true,\"max-sorted-runs\":20,\"gc-merge-rewrite\":false}},\"raftcf\":{\"block-size\":\"16KiB\",\"block-cache-size\":\"128MiB\",\"disable-block-cache\":false,\"cache-index-and-filter-blocks\":true,\"pin-l0-filter-and-index-blocks\":true,\"use-bloom-filter\":true,\"optimize-filters-for-hits\":true,\"whole-key-filtering\":true,\"bloom-filter-bits-per-key\":10,\"block-based-bloom-filter\":false,\"read-amp-bytes-per-bit\":0,\"compression-per-level\":[\"no\",\"no\",\"no\",\"no\",\"no\",\"no\",\"no\"],\"write-buffer-size\":\"128MiB\",\"max-write-buffer-number\":5,\"min-write-buffer-number-to-merge\":1,\"max-bytes-for-level-base\":\"128MiB\",\"target-file-size-base\":\"8MiB\",\"level0-file-num-compaction-trigger\":1,\"level0-slowdown-writes-trigger\":20,\"level0-stop-writes-trigger\":36,\"max-compaction-bytes\":\"2GiB\",\"compaction-pri\":0,\"dynamic-level-bytes\":true,\"num-levels\":7,\"max-bytes-for-level-multiplier\":10,\"compaction-style\":0,\"disable-auto-compactions\":false,\"soft-pending-compaction-bytes-limit\":\"64GiB\",\"hard-pending-compaction-bytes-limit\":\"256GiB\",\"force-consistency-checks\":false,\"prop-size-index-distance\":4194304,\"prop-keys-index-distance\":40960,\"enable-doubly-skiplist\":true,\"titan\":{\"min-blob-size\":\"1KiB\",\"blob-file-compression\":\"lz4\",\"blob-cache-size\":\"0KiB\",\"min-gc-batch-size\":\"16MiB\",\"max-gc-batch-size\":\"64MiB\",\"discardable-ratio\":0.5,\"sample-ratio\":0.1,\"merge-small-file-threshold\":\"8MiB\",\"blob-run-mode\":\"read-only\",\"level-merge\":false,\"range-merge\":true,\"max-sorted-runs\":20,\"gc-merge-rewrite\":false}},\"titan\":{\"enabled\":false,\"dirname\":\"\",\"disable-gc\":false,\"max-background-gc\":4,\"purge-obsolete-files-period\":\"10s\"}},\"raftdb\":{\"wal-recovery-mode\":2,\"wal-dir\":\"\",\"wal-ttl-seconds\":0,\"wal-size-limit\":\"0KiB\",\"max-total-wal-size\":\"4GiB\",\"max-background-jobs\":4,\"max-background-flushes\":1,\"max-manifest-file-size\":\"20MiB\",\"create-if-missing\":true,\"max-open-files\":40960,\"enable-statistics\":true,\"stats-dump-period\":\"10m\",\"compaction-readahead-size\":\"0KiB\",\"info-log-max-size\":\"1GiB\",\"info-log-roll-time\":\"0s\",\"info-log-keep-log-file-num\":10,\"info-log-dir\":\"\",\"max-sub-compactions\":2,\"writable-file-max-buffer-size\":\"1MiB\",\"use-direct-io-for-flush-and-compaction\":false,\"enable-pipelined-write\":true,\"enable-unordered-write\":false,\"allow-concurrent-memtable-write\":true,\"bytes-per-sync\":\"1MiB\",\"wal-bytes-per-sync\":\"512KiB\",\"defaultcf\":{\"block-size\":\"64KiB\",\"block-cache-size\":\"2GiB\",\"disable-block-cache\":false,\"cache-index-and-filter-blocks\":true,\"pin-l0-filter-and-index-blocks\":true,\"use-bloom-filter\":false,\"optimize-filters-for-hits\":true,\"whole-key-filtering\":true,\"bloom-filter-bits-per-key\":10,\"block-based-bloom-filter\":false,\"read-amp-bytes-per-bit\":0,\"compression-per-level\":[\"no\",\"no\",\"lz4\",\"lz4\",\"lz4\",\"zstd\",\"zstd\"],\"write-buffer-size\":\"128MiB\",\"max-write-buffer-number\":5,\"min-write-buffer-number-to-merge\":1,\"max-bytes-for-level-base\":\"512MiB\",\"target-file-size-base\":\"8MiB\",\"level0-file-num-compaction-trigger\":4,\"level0-slowdown-writes-trigger\":20,\"level0-stop-writes-trigger\":36,\"max-compaction-bytes\":\"2GiB\",\"compaction-pri\":0,\"dynamic-level-bytes\":true,\"num-levels\":7,\"max-bytes-for-level-multiplier\":10,\"compaction-style\":0,\"disable-auto-compactions\":false,\"soft-pending-compaction-bytes-limit\":\"64GiB\",\"hard-pending-compaction-bytes-limit\":\"256GiB\",\"force-consistency-checks\":false,\"prop-size-index-distance\":4194304,\"prop-keys-index-distance\":40960,\"enable-doubly-skiplist\":true,\"titan\":{\"min-blob-size\":\"1KiB\",\"blob-file-compression\":\"lz4\",\"blob-cache-size\":\"0KiB\",\"min-gc-batch-size\":\"16MiB\",\"max-gc-batch-size\":\"64MiB\",\"discardable-ratio\":0.5,\"sample-ratio\":0.1,\"merge-small-file-threshold\":\"8MiB\",\"blob-run-mode\":\"normal\",\"level-merge\":false,\"range-merge\":true,\"max-sorted-runs\":20,\"gc-merge-rewrite\":false}},\"titan\":{\"enabled\":false,\"dirname\":\"\",\"disable-gc\":false,\"max-background-gc\":4,\"purge-obsolete-files-period\":\"10s\"}},\"security\":{\"ca-path\":\"\",\"cert-path\":\"\",\"key-path\":\"\",\"cert-allowed-cn\":[],\"redact-info-log\":null,\"encryption\":{\"data-encryption-method\":\"plaintext\",\"data-key-rotation-period\":\"7d\",\"enable-file-dictionary-log\":true,\"file-dictionary-rewrite-threshold\":1000000,\"master-key\":{\"type\":\"plaintext\"},\"previous-master-key\":{\"type\":\"plaintext\"}}},\"import\":{\"num-threads\":8,\"stream-channel-window\":128},\"backup\":{\"num-threads\":32,\"batch-size\":8,\"sst-max-size\":\"144MiB\"},\"pessimistic-txn\":{\"enabled\":true,\"wait-for-lock-timeout\":\"1s\",\"wake-up-delay-duration\":\"20ms\",\"pipelined\":false},\"gc\":{\"ratio-threshold\":1.1,\"batch-keys\":512,\"max-write-bytes-per-sec\":\"0KiB\"},\"split\":{\"qps-threshold\":3000,\"split-balance-score\":0.25,\"split-contained-score\":0.5,\"detect-times\":10,\"sample-num\":20,\"sample-threshold\":100,\"size-threshold\":4194304,\"key-threshold\":40960},\"cdc\":{\"min-ts-interval\":\"1s\"}}"]
3 两台down的tikv 日志都是
[root@data11 log]# cat tikv_stderr.log
<jemalloc>: Malformed conf string
<jemalloc>: Malformed conf string
<jemalloc>: Malformed conf string
<jemalloc>: Malformed conf string
<jemalloc>: Malformed conf string
<jemalloc>: Malformed conf string
[root@saas1 log]# cat tikv_stderr.log
<jemalloc>: Malformed conf string
<jemalloc>: Malformed conf string
<jemalloc>: Malformed conf string
<jemalloc>: Malformed conf string
<jemalloc>: Malformed conf string
<jemalloc>: Malformed conf string
[root@saas1 log]#
从现在运行的dashboard 里 下载4月13日23点到现在所有的错误日志
logs.zip (2.1 MB)
