ticdc 下游停止写入

k8s 平台, 从tidb-primary 往tidb-slave 使用cdc同步:
(1) primary 和slave都初始化sysbench使用的数据库
(2) 配置primary 和 slave的cdc同步:
k8scdccli changefeed create --pd=http://tidb-primary-pd:2379 --sink-uri=“mysql://root:xxxxxx@tidb-slave-tidb:4000/?worker-count=8&max-txn-row=1000” --changefeed-id=“k8s-primary-slave” --sort-engine=“unified”
(3)使用sysbench在primary初始化10张1亿表
问题: slave库上只创建了9张表,在同步了2600行数据后终止了写入,数据量一直无变化;

从server监控上一直在持续工作

pod状态

# kubectl get pod -n tidb-cluster1
NAME                                     READY   STATUS    RESTARTS        AGE
tidb-monitor-monitor-0                   4/4     Running   0               5h29m
tidb-primary-discovery-b5ddb679f-bhqg2   1/1     Running   0               5h51m
tidb-primary-pd-0                        1/1     Running   0               5h51m
tidb-primary-pd-1                        1/1     Running   0               5h51m
tidb-primary-pd-2                        1/1     Running   1 (5h51m ago)   5h51m
tidb-primary-ticdc-0                     1/1     Running   0               43m
tidb-primary-ticdc-1                     1/1     Running   0               6m25s
tidb-primary-ticdc-2                     1/1     Running   0               98m
tidb-primary-tidb-0                      2/2     Running   0               161m
tidb-primary-tidb-1                      2/2     Running   0               162m
tidb-primary-tidb-2                      2/2     Running   0               43m
tidb-primary-tiflash-0                   4/4     Running   0               165m
tidb-primary-tiflash-1                   4/4     Running   0               166m
tidb-primary-tiflash-2                   4/4     Running   0               167m
tidb-primary-tikv-0                      1/1     Running   1 (72m ago)     164m
tidb-primary-tikv-1                      1/1     Running   1 (52m ago)     165m
tidb-primary-tikv-2                      1/1     Running   1 (51m ago)     165m
tidb-slave-discovery-5d945d4f8c-8v6xn    1/1     Running   0               5h44m
tidb-slave-pd-0                          1/1     Running   1 (5h44m ago)   5h44m
tidb-slave-pd-1                          1/1     Running   0               5h44m
tidb-slave-pd-2                          1/1     Running   0               5h44m
tidb-slave-tidb-0                        2/2     Running   0               166m
tidb-slave-tidb-1                        2/2     Running   0               167m
tidb-slave-tidb-2                        2/2     Running   0               168m
tidb-slave-tikv-0                        1/1     Running   0               169m
tidb-slave-tikv-1                        1/1     Running   0               169m
tidb-slave-tikv-2                        1/1     Running   0               170m

changefeed status:

{
  "upstream_id": 7198742155979344387,
  "namespace": "default",
  "id": "k8s-primary-slave",
  "sink_uri": "mysql://root:xxxxx@tidb-slave-tidb:4000/?worker-count=8\u0026max-txn-row=1000",
  "config": {
    "case_sensitive": true,
    "enable_old_value": true,
    "force_replicate": false,
    "ignore_ineligible_table": false,
    "check_gc_safe_point": true,
    "enable_sync_point": false,
    "bdr_mode": false,
    "sync_point_interval": 600000000000,
    "sync_point_retention": 86400000000000,
    "filter": {
      "rules": [
        "*.*"
      ],
      "event_filters": null
    },
    "mounter": {
      "worker_num": 16
    },
    "sink": {
      "protocol": "",
      "schema_registry": "",
      "csv": {
        "delimiter": ",",
        "quote": "\"",
        "null": "\\N",
        "include_commit_ts": false
      },
      "column_selectors": null,
      "transaction_atomicity": "none",
      "encoder_concurrency": 16,
      "terminator": "\r\n",
      "date_separator": "none",
      "enable_partition_separator": false
    },
    "consistent": {
      "level": "none",
      "max_log_size": 64,
      "flush_interval": 2000,
      "storage": ""
    }
  },
  "create_time": "2023-02-11 15:19:15.344",
  "start_ts": 439379546657783822,
  "resolved_ts": 439379577472811021,
  "target_ts": 0,
  "checkpoint_tso": 439379577433489451,
  "checkpoint_time": "2023-02-11 15:21:12.607",
  "sort_engine": "unified",
  "state": "normal",
  "error": null,
  "error_history": null,
  "creator_version": "",
  "task_status": [
    {
      "capture_id": "a507ba7a-c90d-48c1-a4ac-681238190416",
      "table_ids": [
        86,
        92,
        100,
        88
      ],
      "table_operations": null
    },
    {
      "capture_id": "4076692b-7550-4029-83f9-38e58d26162f",
      "table_ids": [],
      "table_operations": null
    },
    {
      "capture_id": "b0a7fc84-0eb5-4c44-a284-5c79994ab8ea",
      "table_ids": [
        94,
        85,
        90,
        96,
        84
      ],
      "table_operations": null
    }
  ]
}

cdc日志:
cdc.rar (125.9 KB)

primary tikv日志

tikv.rar (846.6 KB)

请问磁盘配置是什么样的呢?

我重新建了changefeed好了 磁盘是普通ssd

麻烦用 kubectl logs -p 采集下 TiCDC Pod 重启前的日志

image

看不到,

这个可以记一下,放到版主交流会上讨论~