CDC同步数据错误

qihuajun · 2025 年2 月 21 日 11:49

【 TiDB 使用环境】生产环境
【 TiDB 版本】V8.1.0
【复现路径】正常开启cdc
【遇到的问题：问题现象及影响】
同步的过程中发现noteid字段的值变成了空字符串，但是在上游tidb服务器中这个字段的值是正常的非空字符串。于是开启了一个新的changefeed，把数据变更记录到本地确认变更日志中这个字段的值就是空的，以下是保存到本地的变更日志：

{
    "id": 0,
    "database": "test",
    "table": "brand_note",
    "pkNames": [
        "brandid",
        "noteid"
    ],
    "isDdl": false,
    "type": "INSERT",
    "es": 1740137386485,
    "ts": 1740137387948,
    "sql": "",
    "sqlType": {
        "brandid": 4,
        "noteid": 12,
        "uid": 12
    },
    "mysqlType": {
        "brandid": "int unsigned",
        "noteid": "varchar",
        "uid": "varchar"
    },
    "old": null,
    "data": [
        {
            "brandid": "10",
            "noteid": "",
            "uid": "5ff1bff600000000010016e8"
        }
    ]
}

TiDBer_wk · 2025 年2 月 25 日 03:15

能发下这个字符吗？日志有没有相关报错

Jasper · 2025 年2 月 25 日 05:56

是稳定复现吗？这个 noteid 字段里的字符串最近涉及变更吗？比如是刚insert 进来的？还是update 过的？

qihuajun · 2025 年2 月 26 日 03:27

没有报错，原始的字符就是普通的字符串，类似uid的值：5ff1bff600000000010016e8

qihuajun · 2025 年2 月 26 日 03:28

稳定复现，这个字段最近也没有变更，insert操作和update操作都一样，从sink到本地的文件里看值都是空字符串

qihuajun · 2025 年2 月 26 日 03:32

还发现另外一个情况，我更新changefeed的配置开启一致性验证开启不成功，操作日志显示执行了更新操作，但最终的结果是未生效：

Starting component cdc: /home/tidb/.tiup/components/cdc/v8.1.0/cdc cli changefeed update -c rxbn --config=rxbn.toml
Diff of changefeed config:
{Type:update Path:[Config SyncPointInterval] From:<nil> To:0xc00119b038}
{Type:update Path:[Config SyncPointRetention] From:<nil> To:0xc00119b0d0}
{Type:update Path:[Config Consistent] From:<nil> To:0xc0010e03f0}
{Type:update Path:[Config Integrity IntegrityCheckLevel] From:none To:correctness}
Could you agree to apply changes above to changefeed [Y/N]
Y
Update changefeed config successfully! 
ID: rxbn
Info: 
{
    "upstream_id": 6801054821581547599,
    "namespace": "default",
    "id": "rxbn",
    "sink_uri": "file:///tmp/cdc-rxbn",
    "create_time": "2025-02-21T21:30:16.133726532+08:00",
    "start_ts": 456227220811939865,
    "admin_job_type": 1,
    "config": {
        "memory_quota": 1073741824,
        "case_sensitive": true,
        "force_replicate": false,
        "ignore_ineligible_table": false,
        "check_gc_safe_point": true,
        "enable_table_monitor": false,
        "sync_point_interval": 600000000000,
        "sync_point_retention": 86400000000000,
        "filter": {
            "rules": [
                "mbase.rxbn"
            ],
            "event_filters": [
                {
                    "matcher": [
                        "mbase.rxbn"
                    ],
                    "ignore_event": null,
                    "ignore_sql": null,
                    "ignore_insert_value_expr": "createtime \u003c= '2025-01-01'",
                    "ignore_update_new_value_expr": "",
                    "ignore_update_old_value_expr": "createtime \u003c= '2025-01-01'",
                    "ignore_delete_value_expr": ""
                }
            ]
        },
        "mounter": {
            "worker_num": 4
        },
        "sink": {
            "protocol": "canal-json",
            "csv": {
                "delimiter": ",",
                "quote": "\"",
                "null": "\\N",
                "include_commit_ts": false,
                "binary_encoding_method": "base64",
                "output_old_value": false,
                "output_handle_key": false
            },
            "terminator": "\r\n",
            "date_separator": "day",
            "enable_partition_separator": true,
            "file_index_width": 20,
            "delete_only_output_handle_key_columns": null,
            "content_compatible": null,
            "advance_timeout": 150,
            "send_bootstrap_interval_in_sec": 120,
            "send_bootstrap_in_msg_count": 10000,
            "send_bootstrap_to_all_partition": true,
            "debezium_disable_schema": false,
            "debezium": {
                "output_old_value": true
            },
            "open": {
                "output_old_value": true
            }
        },
        "consistent": {
            "level": "none",
            "max_log_size": 64,
            "flush_interval": 2000,
            "meta_flush_interval": 200,
            "encoding_worker_num": 16,
            "flush_worker_num": 8,
            "use_file_backend": false,
            "memory_usage": {
                "memory_quota_percentage": 50
            }
        },
        "scheduler": {
            "enable_table_across_nodes": false,
            "region_threshold": 100000,
            "write_key_threshold": 0
        },
        "integrity": {
            "integrity_check_level": "none",
            "corruption_handle_level": "warn"
        },
        "changefeed_error_stuck_duration": 1800000000000,
        "synced_status": {
            "synced_check_interval": 300,
            "checkpoint_interval": 15
        }
    },
    "state": "stopped",
    "creator_version": "v8.1.0",
    "resolved_ts": 0,
    "checkpoint_ts": 456227335211581442,
    "checkpoint_time": "2025-02-24 11:52:48.135"
}

TiDBer_wk · 2025 年2 月 26 日 09:09

意思是这里的 uid 字段和 noteid 字段是同一类型，两个字段的值也基本一样，只有 uid 字段会出现字段为空的情况？

qihuajun · 2025 年2 月 27 日 02:40

是这样的

TiDBer_wk · 2025 年2 月 27 日 05:52

请给一个能复现的 sql，我本地测试下

qihuajun · 2025 年2 月 27 日 07:09

这是我们完整的建表语句，执行任何insert或者update操作cdc记录的noteid字段都是空字符串，另外这张表开启了两个tiflash副本。

CREATE TABLE `rxbn` (
  `industryid` int(10) unsigned NOT NULL ,
  `categoryid` int(11) NOT NULL DEFAULT '0' ,
  `brandid` int(10) unsigned NOT NULL ,
  `commodityid` bigint(20) NOT NULL DEFAULT '0' ,
  `noteid` varchar(32) NOT NULL ,
  `uid` varchar(32) NOT NULL ,
  `keyword` varchar(255) NOT NULL ,
  `is_ads` tinyint(1) NOT NULL DEFAULT '0' ,
  `source` tinyint(4) NOT NULL DEFAULT '1' ,
  `pos` tinyint(4) DEFAULT NULL ,
  `user_questions` json DEFAULT NULL ,
  `filtered` tinyint(1) DEFAULT '0' ,
  `weight_partition` tinyint(1) DEFAULT NULL ,
  `filter_time` datetime DEFAULT NULL ,
  `createtime` datetime NOT NULL ,
  `point_time` datetime DEFAULT NULL ,
  `updatetime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP ,
  KEY `inx_updatetime` (`updatetime`),
  PRIMARY KEY (`industryid`,`brandid`,`commodityid`,`categoryid`,`noteid`),
  KEY `idx_brand_id_point_time` (`brandid`,`point_time`),
  KEY `idx_note_id` (`noteid`),
  KEY `inx_note_industry` (`industryid`,`noteid`),
  KEY `idx_filter_time` (`filter_time`),
  KEY `inx_createtime` (`createtime`),
  KEY `idx_uid` (`uid`),
  KEY `idx_commodityid` (`commodityid`),
  KEY `idx_cfb` (`categoryid`,`filtered`,`brandid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

qihuajun · 2025 年2 月 27 日 07:22

生成数据的sql：

insert into rxbn (industryid, categoryid, brandid, commodityid, noteid, uid, keyword, is_ads, source, pos, user_questions, filtered, weight_partition, filter_time, createtime, point_time, updatetime)
values  (0, 2, 0, 0, '62cqaz2334io1002a2f1', '5fc8989d2344328897003e2b', '精华', 0, 2, null, null, 1, null, '2025-02-25 23:01:24', '2025-02-25 23:01:24', null, '2025-02-25 23:01:31'),
        (0, 2, 0, 0, '62cqaz2334io1d02738f', '600e72b8234432889701c7ac', '精华', 0, 2, null, null, 1, null, '2025-02-25 23:01:25', '2025-02-25 23:01:25', null, '2025-02-25 23:01:31'),
        (0, 2, 0, 0, '62cqaz2334io1d01ca9c', '60aeec12234432889701dbcb', '精华', 0, 2, null, null, 1, null, '2025-02-25 23:01:26', '2025-02-25 23:01:26', null, '2025-02-25 23:01:31'),
        (0, 2, 0, 0, '62cqaz2334io1e006b23', '5d08a969234432889700d7cd', '精华', 0, 2, null, null, 1, null, '2025-02-25 23:01:25', '2025-02-25 23:01:25', null, '2025-02-25 23:01:30'),
        (0, 2, 0, 0, '62cqaz2334io1102b0ac', '5e033952234432889700872d', '精华', 0, 2, null, null, 1, null, '2025-02-25 23:01:24', '2025-02-25 23:01:24', null, '2025-02-25 23:01:32'),
        (0, 2, 0, 0, '62cqaz2334io1102e937', '56513a7e234432889733184d', '精华', 0, 2, null, null, 1, null, '2025-02-25 23:01:26', '2025-02-25 23:01:26', null, '2025-02-25 23:01:31'),
        (0, 2, 0, 0, '62cqaz2334io1002a066', '5e5f58e82344328897007d61', '精华', 0, 2, null, null, 1, null, '2025-02-25 23:01:25', '2025-02-25 23:01:25', null, '2025-02-25 23:01:31'),
        (0, 2, 0, 0, '62cqaz2334io1d01f884', '5e344d6f23443288970016a3', '精华', 0, 2, null, null, 1, null, '2025-02-25 23:01:24', '2025-02-25 23:01:24', null, '2025-02-25 23:01:30'),
        (0, 2, 0, 0, '62cqaz2334io1f0051b5', '5f1688d923443288970043a4', '精华', 0, 2, null, null, 1, null, '2025-02-25 23:01:28', '2025-02-25 23:01:28', null, '2025-02-25 23:01:35'),
        (0, 2, 0, 0, '62cqaz2334io11026a94', '5acce1772344328897474314', '精华', 0, 2, null, null, 1, null, '2025-02-25 23:01:30', '2025-02-25 23:01:30', null, '2025-02-25 23:01:36');

TiDBer_wk · 2025 年2 月 28 日 03:55

我在本地测试可以正常显示

{"id":0,"database":"test","table":"rxbn","pkNames":["industryid","categoryid","brandid","commodityid","noteid"],"isDdl":false,"type":"INSERT","es":1740714780302,"ts":1740714783181,"sql":"","sqlType":{"industryid":4,"categoryid":4,"brandid":4,"commodityid":-5,"noteid":12,"uid":12,"keyword":12,"is_ads":-6,"source":-6,"pos":-6,"user_questions":12,"filtered":-6,"weight_partition":-6,"filter_time":93,"createtime":93,"point_time":93,"updatetime":93},"mysqlType":{"source":"tinyint","updatetime":"timestamp","filtered":"tinyint","weight_partition":"tinyint","filter_time":"datetime","brandid":"int unsigned","uid":"varchar","keyword":"varchar","pos":"tinyint","user_questions":"json","industryid":"int unsigned","noteid":"varchar","point_time":"datetime","categoryid":"int","commodityid":"bigint","is_ads":"tinyint","createtime":"datetime"},"old":null,"data":[{"industryid":"0","categoryid":"2","brandid":"0","commodityid":"0","noteid":"62cqaz2334io1002a2f1","uid":"5fc8989d2344328897003e2b","keyword":"精华","is_ads":"0","source":"2","pos":null,"user_questions":null,"filtered":"1","weight_partition":null,"filter_time":"2025-02-25 23:01:24","createtime":"2025-02-25 23:01:24","point_time":null,"updatetime":"2025-02-25 23:01:31"}]}
{"id":0,"database":"test","table":"rxbn","pkNames":["industryid","categoryid","brandid","commodityid","noteid"],"isDdl":false,"type":"INSERT","es":1740714780302,"ts":1740714783182,"sql":"","sqlType":{"industryid":4,"categoryid":4,"brandid":4,"commodityid":-5,"noteid":12,"uid":12,"keyword":12,"is_ads":-6,"source":-6,"pos":-6,"user_questions":12,"filtered":-6,"weight_partition":-6,"filter_time":93,"createtime":93,"point_time":93,"updatetime":93},"mysqlType":{"categoryid":"int","commodityid":"bigint","noteid":"varchar","uid":"varchar","source":"tinyint","updatetime":"timestamp","industryid":"int unsigned","createtime":"datetime","point_time":"datetime","filtered":"tinyint","is_ads":"tinyint","pos":"tinyint","weight_partition":"tinyint","filter_time":"datetime","keyword":"varchar","user_questions":"json","brandid":"int unsigned"},"old":null,"data":[{"industryid":"0","categoryid":"2","brandid":"0","commodityid":"0","noteid":"62cqaz2334io1d02738f","uid":"600e72b8234432889701c7ac","keyword":"精华","is_ads":"0","source":"2","pos":null,"user_questions":null,"filtered":"1","weight_partition":null,"filter_time":"2025-02-25 23:01:25","createtime":"2025-02-25 23:01:25","point_time":null,"updatetime":"2025-02-25 23:01:31"}]}
{"id":0,"database":"test","table":"rxbn","pkNames":["industryid","categoryid","brandid","commodityid","noteid"],"isDdl":false,"type":"INSERT","es":1740714780302,"ts":1740714783182,"sql":"","sqlType":{"industryid":4,"categoryid":4,"brandid":4,"commodityid":-5,"noteid":12,"uid":12,"keyword":12,"is_ads":-6,"source":-6,"pos":-6,"user_questions":12,"filtered":-6,"weight_partition":-6,"filter_time":93,"createtime":93,"point_time":93,"updatetime":93},"mysqlType":{"categoryid":"int","is_ads":"tinyint","updatetime":"timestamp","industryid":"int unsigned","pos":"tinyint","filter_time":"datetime","point_time":"datetime","brandid":"int unsigned","commodityid":"bigint","noteid":"varchar","uid":"varchar","keyword":"varchar","source":"tinyint","user_questions":"json","filtered":"tinyint","createtime":"datetime","weight_partition":"tinyint"},"old":null,"data":[{"industryid":"0","categoryid":"2","brandid":"0","commodityid":"0","noteid":"62cqaz2334io1d01ca9c","uid":"60aeec12234432889701dbcb","keyword":"精华","is_ads":"0","source":"2","pos":null,"user_questions":null,"filtered":"1","weight_partition":null,"filter_time":"2025-02-25 23:01:26","createtime":"2025-02-25 23:01:26","point_time":null,"updatetime":"2025-02-25 23:01:31"}]}
{"id":0,"database":"test","table":"rxbn","pkNames":["industryid","categoryid","brandid","commodityid","noteid"],"isDdl":false,"type":"INSERT","es":1740714780302,"ts":1740714783182,"sql":"","sqlType":{"industryid":4,"categoryid":4,"brandid":4,"commodityid":-5,"noteid":12,"uid":12,"keyword":12,"is_ads":-6,"source":-6,"pos":-6,"user_questions":12,"filtered":-6,"weight_partition":-6,"filter_time":93,"createtime":93,"point_time":93,"updatetime":93},"mysqlType":{"uid":"varchar","keyword":"varchar","commodityid":"bigint","is_ads":"tinyint","source":"tinyint","pos":"tinyint","filtered":"tinyint","industryid":"int unsigned","categoryid":"int","brandid":"int unsigned","point_time":"datetime","updatetime":"timestamp","filter_time":"datetime","createtime":"datetime","noteid":"varchar","user_questions":"json","weight_partition":"tinyint"},"old":null,"data":[{"industryid":"0","categoryid":"2","brandid":"0","commodityid":"0","noteid":"62cqaz2334io1e006b23","uid":"5d08a969234432889700d7cd","keyword":"精华","is_ads":"0","source":"2","pos":null,"user_questions":null,"filtered":"1","weight_partition":null,"filter_time":"2025-02-25 23:01:25","createtime":"2025-02-25 23:01:25","point_time":null,"updatetime":"2025-02-25 23:01:30"}]}
{"id":0,"database":"test","table":"rxbn","pkNames":["industryid","categoryid","brandid","commodityid","noteid"],"isDdl":false,"type":"INSERT","es":1740714780302,"ts":1740714783182,"sql":"","sqlType":{"industryid":4,"categoryid":4,"brandid":4,"commodityid":-5,"noteid":12,"uid":12,"keyword":12,"is_ads":-6,"source":-6,"pos":-6,"user_questions":12,"filtered":-6,"weight_partition":-6,"filter_time":93,"createtime":93,"point_time":93,"updatetime":93},"mysqlType":{"brandid":"int unsigned","is_ads":"tinyint","user_questions":"json","filter_time":"datetime","createtime":"datetime","categoryid":"int","source":"tinyint","filtered":"tinyint","point_time":"datetime","updatetime":"timestamp","commodityid":"bigint","noteid":"varchar","uid":"varchar","weight_partition":"tinyint","industryid":"int unsigned","pos":"tinyint","keyword":"varchar"},"old":null,"data":[{"industryid":"0","categoryid":"2","brandid":"0","commodityid":"0","noteid":"62cqaz2334io1102b0ac","uid":"5e033952234432889700872d","keyword":"精华","is_ads":"0","source":"2","pos":null,"user_questions":null,"filtered":"1","weight_partition":null,"filter_time":"2025-02-25 23:01:24","createtime":"2025-02-25 23:01:24","point_time":null,"updatetime":"2025-02-25 23:01:32"}]}
{"id":0,"database":"test","table":"rxbn","pkNames":["industryid","categoryid","brandid","commodityid","noteid"],"isDdl":false,"type":"INSERT","es":1740714780302,"ts":1740714783182,"sql":"","sqlType":{"industryid":4,"categoryid":4,"brandid":4,"commodityid":-5,"noteid":12,"uid":12,"keyword":12,"is_ads":-6,"source":-6,"pos":-6,"user_questions":12,"filtered":-6,"weight_partition":-6,"filter_time":93,"createtime":93,"point_time":93,"updatetime":93},"mysqlType":{"uid":"varchar","keyword":"varchar","pos":"tinyint","point_time":"datetime","updatetime":"timestamp","brandid":"int unsigned","noteid":"varchar","filter_time":"datetime","commodityid":"bigint","is_ads":"tinyint","source":"tinyint","industryid":"int unsigned","categoryid":"int","weight_partition":"tinyint","createtime":"datetime","user_questions":"json","filtered":"tinyint"},"old":null,"data":[{"industryid":"0","categoryid":"2","brandid":"0","commodityid":"0","noteid":"62cqaz2334io1102e937","uid":"56513a7e234432889733184d","keyword":"精华","is_ads":"0","source":"2","pos":null,"user_questions":null,"filtered":"1","weight_partition":null,"filter_time":"2025-02-25 23:01:26","createtime":"2025-02-25 23:01:26","point_time":null,"updatetime":"2025-02-25 23:01:31"}]}
{"id":0,"database":"test","table":"rxbn","pkNames":["industryid","categoryid","brandid","commodityid","noteid"],"isDdl":false,"type":"INSERT","es":1740714780302,"ts":1740714783182,"sql":"","sqlType":{"industryid":4,"categoryid":4,"brandid":4,"commodityid":-5,"noteid":12,"uid":12,"keyword":12,"is_ads":-6,"source":-6,"pos":-6,"user_questions":12,"filtered":-6,"weight_partition":-6,"filter_time":93,"createtime":93,"point_time":93,"updatetime":93},"mysqlType":{"createtime":"datetime","industryid":"int unsigned","commodityid":"bigint","noteid":"varchar","user_questions":"json","categoryid":"int","brandid":"int unsigned","is_ads":"tinyint","filter_time":"datetime","point_time":"datetime","uid":"varchar","keyword":"varchar","source":"tinyint","pos":"tinyint","filtered":"tinyint","weight_partition":"tinyint","updatetime":"timestamp"},"old":null,"data":[{"industryid":"0","categoryid":"2","brandid":"0","commodityid":"0","noteid":"62cqaz2334io1002a066","uid":"5e5f58e82344328897007d61","keyword":"精华","is_ads":"0","source":"2","pos":null,"user_questions":null,"filtered":"1","weight_partition":null,"filter_time":"2025-02-25 23:01:25","createtime":"2025-02-25 23:01:25","point_time":null,"updatetime":"2025-02-25 23:01:31"}]}
{"id":0,"database":"test","table":"rxbn","pkNames":["industryid","categoryid","brandid","commodityid","noteid"],"isDdl":false,"type":"INSERT","es":1740714780302,"ts":1740714783182,"sql":"","sqlType":{"industryid":4,"categoryid":4,"brandid":4,"commodityid":-5,"noteid":12,"uid":12,"keyword":12,"is_ads":-6,"source":-6,"pos":-6,"user_questions":12,"filtered":-6,"weight_partition":-6,"filter_time":93,"createtime":93,"point_time":93,"updatetime":93},"mysqlType":{"updatetime":"timestamp","industryid":"int unsigned","commodityid":"bigint","uid":"varchar","keyword":"varchar","user_questions":"json","filter_time":"datetime","brandid":"int unsigned","filtered":"tinyint","categoryid":"int","noteid":"varchar","is_ads":"tinyint","createtime":"datetime","point_time":"datetime","source":"tinyint","pos":"tinyint","weight_partition":"tinyint"},"old":null,"data":[{"industryid":"0","categoryid":"2","brandid":"0","commodityid":"0","noteid":"62cqaz2334io1d01f884","uid":"5e344d6f23443288970016a3","keyword":"精华","is_ads":"0","source":"2","pos":null,"user_questions":null,"filtered":"1","weight_partition":null,"filter_time":"2025-02-25 23:01:24","createtime":"2025-02-25 23:01:24","point_time":null,"updatetime":"2025-02-25 23:01:30"}]}
{"id":0,"database":"test","table":"rxbn","pkNames":["industryid","categoryid","brandid","commodityid","noteid"],"isDdl":false,"type":"INSERT","es":1740714780302,"ts":1740714783182,"sql":"","sqlType":{"industryid":4,"categoryid":4,"brandid":4,"commodityid":-5,"noteid":12,"uid":12,"keyword":12,"is_ads":-6,"source":-6,"pos":-6,"user_questions":12,"filtered":-6,"weight_partition":-6,"filter_time":93,"createtime":93,"point_time":93,"updatetime":93},"mysqlType":{"keyword":"varchar","weight_partition":"tinyint","filter_time":"datetime","categoryid":"int","commodityid":"bigint","point_time":"datetime","updatetime":"timestamp","industryid":"int unsigned","noteid":"varchar","uid":"varchar","is_ads":"tinyint","pos":"tinyint","createtime":"datetime","brandid":"int unsigned","source":"tinyint","user_questions":"json","filtered":"tinyint"},"old":null,"data":[{"industryid":"0","categoryid":"2","brandid":"0","commodityid":"0","noteid":"62cqaz2334io1f0051b5","uid":"5f1688d923443288970043a4","keyword":"精华","is_ads":"0","source":"2","pos":null,"user_questions":null,"filtered":"1","weight_partition":null,"filter_time":"2025-02-25 23:01:28","createtime":"2025-02-25 23:01:28","point_time":null,"updatetime":"2025-02-25 23:01:35"}]}
{"id":0,"database":"test","table":"rxbn","pkNames":["industryid","categoryid","brandid","commodityid","noteid"],"isDdl":false,"type":"INSERT","es":1740714780302,"ts":1740714783182,"sql":"","sqlType":{"industryid":4,"categoryid":4,"brandid":4,"commodityid":-5,"noteid":12,"uid":12,"keyword":12,"is_ads":-6,"source":-6,"pos":-6,"user_questions":12,"filtered":-6,"weight_partition":-6,"filter_time":93,"createtime":93,"point_time":93,"updatetime":93},"mysqlType":{"source":"tinyint","weight_partition":"tinyint","point_time":"datetime","updatetime":"timestamp","brandid":"int unsigned","commodityid":"bigint","noteid":"varchar","uid":"varchar","is_ads":"tinyint","user_questions":"json","filtered":"tinyint","createtime":"datetime","industryid":"int unsigned","categoryid":"int","keyword":"varchar","pos":"tinyint","filter_time":"datetime"},"old":null,"data":[{"industryid":"0","categoryid":"2","brandid":"0","commodityid":"0","noteid":"62cqaz2334io11026a94","uid":"5acce1772344328897474314","keyword":"精华","is_ads":"0","source":"2","pos":null,"user_questions":null,"filtered":"1","weight_partition":null,"filter_time":"2025-02-25 23:01:30","createtime":"2025-02-25 23:01:30","point_time":null,"updatetime":"2025-02-25 23:01:36"}]}

qihuajun · 2025 年2 月 28 日 07:16

版本是8.1.0吗？开启TiFlash副本了吗？

我在本地重新建表试了一下还是不行。

TiDBer_wk · 2025 年2 月 28 日 07:38

是 8.1.0, 我没有开 tiflash 副本

qihuajun · 2025 年2 月 28 日 08:15

你开启tiflash副本之后还可以吗？

TiDBer_wk · 2025 年2 月 28 日 08:19

我加了 ALTER TABLE rxbn SET TIFLASH REPLICA 1; 还是成功的。 tiflash 副本和这个应该没啥关系，cdc 是从 tikv 取数据的

db_user · 2025 年2 月 28 日 08:32

上下游表结构发下看看，都一样，还有sql_mode也发下

qihuajun · 2025 年3 月 1 日 13:00

表结构：

上游：
CREATE TABLE `rxbn` (
  `industryid` int(10) unsigned NOT NULL ,
  `categoryid` int(11) NOT NULL DEFAULT '0' ,
  `brandid` int(10) unsigned NOT NULL ,
  `commodityid` bigint(20) NOT NULL DEFAULT '0' ,
  `noteid` varchar(32) NOT NULL ,
  `uid` varchar(32) NOT NULL ,
  `keyword` varchar(255) NOT NULL ,
  `is_ads` tinyint(1) NOT NULL DEFAULT '0' ,
  `source` tinyint(4) NOT NULL DEFAULT '1' ,
  `pos` tinyint(4) DEFAULT NULL ,
  `user_questions` json DEFAULT NULL ,
  `filtered` tinyint(1) DEFAULT '0' ,
  `weight_partition` tinyint(1) DEFAULT NULL ,
  `filter_time` datetime DEFAULT NULL ,
  `createtime` datetime NOT NULL ,
  `point_time` datetime DEFAULT NULL ,
  `updatetime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP ,
  KEY `inx_updatetime` (`updatetime`),
  PRIMARY KEY (`industryid`,`brandid`,`commodityid`,`categoryid`,`noteid`) /*T![clustered_index] CLUSTERED */,
  KEY `idx_brand_id_point_time` (`brandid`,`point_time`),
  KEY `idx_note_id` (`noteid`),
  KEY `inx_note_industry` (`industryid`,`noteid`),
  KEY `idx_filter_time` (`filter_time`),
  KEY `inx_createtime` (`createtime`),
  KEY `idx_uid` (`uid`),
  KEY `idx_commodityid` (`commodityid`),
  KEY `idx_cfb` (`categoryid`,`filtered`,`brandid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin  /*T![placement] PLACEMENT POLICY=`storageonnvme` */;

下游
CREATE TABLE `rxbn` (
  `industryid` int(10) unsigned NOT NULL ,
  `categoryid` int(11) NOT NULL DEFAULT '0' ,
  `brandid` int(10) unsigned NOT NULL ,
  `commodityid` bigint(20) NOT NULL DEFAULT '0' ,
  `noteid` varchar(32) NOT NULL ,
  `uid` varchar(32) NOT NULL ,
  `keyword` varchar(255) NOT NULL ,
  `is_ads` tinyint(1) NOT NULL DEFAULT '0' ,
  `source` tinyint(4) NOT NULL DEFAULT '1' ,
  `pos` tinyint(4) DEFAULT NULL ,
  `user_questions` json DEFAULT NULL ,
  `filtered` tinyint(1) DEFAULT '0' ,
  `weight_partition` tinyint(1) DEFAULT NULL ,
  `filter_time` datetime DEFAULT NULL ,
  `createtime` datetime NOT NULL ,
  `point_time` datetime DEFAULT NULL ,
  `updatetime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP ,
  KEY `inx_updatetime` (`updatetime`),
  PRIMARY KEY (`industryid`,`brandid`,`commodityid`,`categoryid`,`noteid`) /*T![clustered_index] NONCLUSTERED */,
  KEY `idx_brand_id_point_time` (`brandid`,`point_time`),
  KEY `idx_note_id` (`noteid`),
  KEY `inx_note_industry` (`industryid`,`noteid`),
  KEY `idx_filter_time` (`filter_time`),
  KEY `inx_createtime` (`createtime`),
  KEY `idx_uid` (`uid`),
  KEY `idx_commodityid` (`commodityid`),
  KEY `idx_categoryid` (`categoryid`),
  KEY `idx_bfc` (`brandid`,`filtered`,`categoryid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin  /*T![placement] PLACEMENT POLICY=`storageonssd` */;

问题是我现在下游不写入tidb，直接保存到文件里那个字段的值也是空字符串：

{
    "id": 0,
    "database": "test",
    "table": "rxbn",
    "pkNames": [
        "industryid",
        "categoryid",
        "brandid",
        "commodityid",
        "noteid"
    ],
    "isDdl": false,
    "type": "UPDATE",
    "es": 1740833733735,
    "ts": 1740833734635,
    "sql": "",
    "sqlType": {
        "industryid": 4,
        "categoryid": 4,
        "brandid": 4,
        "commodityid": -5,
        "noteid": 12,
        "uid": 12,
        "keyword": 12,
        "is_ads": -6,
        "source": -6,
        "pos": -6,
        "user_questions": 12,
        "filtered": -6,
        "weight_partition": -6,
        "filter_time": 93,
        "createtime": 93,
        "point_time": 93,
        "updatetime": 93
    },
    "mysqlType": {
        "pos": "tinyint",
        "createtime": "datetime",
        "point_time": "datetime",
        "brandid": "int unsigned",
        "uid": "varchar",
        "keyword": "varchar",
        "filter_time": "datetime",
        "updatetime": "timestamp",
        "is_ads": "tinyint",
        "filtered": "tinyint",
        "weight_partition": "tinyint",
        "categoryid": "int",
        "source": "tinyint",
        "user_questions": "json",
        "industryid": "int unsigned",
        "commodityid": "bigint",
        "noteid": "varchar"
    },
    "old": [
        {
            "industryid": "0",
            "categoryid": "2",
            "brandid": "0",
            "commodityid": "0",
            "noteid": "",
            "uid": "300e000000000000101c7ac",
            "keyword": "精华",
            "is_ads": "0",
            "source": "2",
            "pos": null,
            "user_questions": null,
            "filtered": "1",
            "weight_partition": null,
            "filter_time": "2025-02-25 23:01:25",
            "createtime": "2025-02-25 23:01:25",
            "point_time": null,
            "updatetime": "2025-02-25 23:01:31"
        }
    ],
    "data": [
        {
            "industryid": "0",
            "categoryid": "2",
            "brandid": "0",
            "commodityid": "0",
            "noteid": "",
            "uid": "300e000000000000101c7ac",
            "keyword": "精华",
            "is_ads": "0",
            "source": "2",
            "pos": null,
            "user_questions": null,
            "filtered": "1",
            "weight_partition": null,
            "filter_time": "2025-02-25 23:01:25",
            "createtime": "2025-02-25 23:01:25",
            "point_time": "2025-02-25 23:01:31",
            "updatetime": "2025-03-01 20:55:33"
        }
    ]
}

TiDBer_wk · 2025 年3 月 2 日 13:51

看下 changefeed 配置文件，按理说 uid 字段和 noteid 字段应该会表现一致，查下上游 noteid 是不是为空呢

qihuajun · 2025 年3 月 3 日 02:42

肯定不是啊