ticdc 同步报错

blaine · 2022 年8 月 27 日 06:41

【 TiDB 使用环境】生产环境【 TiDB 版本】6.1.0 【遇到的问题】源端的tidb5.3.0 通过ticdc通过到6.1.0，同步任务建立好了，目标端一直报错【复现路径】【问题现象及影响】 [2022/08/27 14:33:07.951 +08:00] [ERROR] [session.go:721] ["assertion failed"] [conn=633048217743437079] [message="[tikv:8141]assertion failed: key: 7480000000000007695f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2a0e320e2e0e2eff0e2a0e2b0e2c0e29ff0e300e300e2d0e2cff0e2d0e2f0e2c0e2eff0e2e000000000000f9, assertion: Exist, start_ts: 435573748162101255, existing start ts: 0, existing commit ts: 0"] ["mvcc history"="{\"key\":\"7480000000000007695F698000000000000001010E2D0E290E290E2AFF0F640E8B10510EC1FF0E330F5B0E8B0EB9FF0E4A1002105E0E30FF0E2A0E320E2E0E2EFF0E2A0E2B0E2C0E29FF0E300E300E2D0E2CFF0E2D0E2F0E2C0E2EFF0E2E000000000000F9\",\"mvcc\":{\"info\":{\"writes\":[{\"type\":3,\"start_ts\":435573742696398855,\"commit_ts\":435573742696398855}]}},\"regionID\":425126}"] [2022/08/27 14:33:07.951 +08:00] [WARN] [session.go:881] ["can not retry txn"] [conn=633048217743437079] [label=general] [error="[tikv:8141]assertion failed: key: 7480000000000007695f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2a0e320e2e0e2eff0e2a0e2b0e2c0e29ff0e300e300e2d0e2cff0e2d0e2f0e2c0e2eff0e2e000000000000f9, assertion: Exist, start_ts: 435573748162101255, existing start ts: 0, existing commit ts: 0"] [IsBatchInsert=false] [IsPessimistic=false] [InRestrictedSQL=false] [tidb_retry_limit=10] [tidb_disable_txn_auto_retry=true] [2022/08/27 14:33:07.951 +08:00] [WARN] [session.go:897] ["commit failed"] [conn=633048217743437079] ["finished txn"="Txn{state=invalid}"] [error="[tikv:8141]assertion failed: key: 7480000000000007695f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2a0e320e2e0e2eff0e2a0e2b0e2c0e29ff0e300e300e2d0e2cff0e2d0e2f0e2c0e2eff0e2e000000000000f9, assertion: Exist, start_ts: 435573748162101255, existing start ts: 0, existing commit ts: 0"] [2022/08/27 14:33:07.951 +08:00] [WARN] [session.go:1941] ["run statement failed"] [conn=633048217743437079] [schemaVersion=889] [error="previous statement: REPLACE INTOactivity_flow_clean.newgamesgcp(id,md5,site_id,member_id,venue_id,venue_bill_no,bet_amount,net_amount,valid_bet_amount,bet_time,settle_time,flag,created_at,updated_at) VALUES (4228667934687145998,_binary'4001newgamesgcp220821301827168',4001,292358,3,_binary'220821301827168','311.0000','-311.0000','311.0000',_binary'2022-08-27T13:23:04+08:00',_binary'2022-08-27T13:23:04+08:00',1,'2022-08-27 13:24:10','2022-08-27 13:24:10'),(4228667934853607502,_binary'4001newgamesgcp220821301827286',4001,82145487,3,_binary'220821301827286','101.0000','-101.0000','101.0000',_binary'2022-08-27T13:24:04+08:00',_binary'2022-08-27T13:24:04+08:00',1,'2022-08-27 13:24:10','2022-08-27 13:24:10'): [tikv:8141]assertion failed: key: 7480000000000007695f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2a0e320e2e0e2eff0e2a0e2b0e2c0e29ff0e300e300e2d0e2cff0e2d0e2f0e2c0e2eff0e2e000000000000f9, assertion: Exist, start_ts: 435573748162101255, existing start ts: 0, existing commit ts: 0"] [session="{\ \"currDBName\": \"\",\ \"id\": 633048217743437079,\ \"status\": 2,\ \"strictMode\": true,\ \"user\": {\ \"Username\": \"dba_tools\",\ \"Hostname\": \"10.10.11.109\",\ \"CurrentUser\": false,\ \"AuthUsername\": \"dba_tools\",\ \"AuthHostname\": \"10.%\"\ }\ }"] [2022/08/27 14:33:07.951 +08:00] [INFO] [conn.go:1149] ["command dispatched failed"] [conn=633048217743437079] [connInfo="id:633048217743437079, addr:10.10.11.109:60563 status:10, collation:utf8mb4_general_ci, user:dba_tools"] [command=Query] [status="inTxn:0, autocommit:1"] [sql=COMMIT] [txn_mode=OPTIMISTIC] [timestamp=435573748162101255] [err="[tikv:8141]assertion failed: key: 7480000000000007695f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2a0e320e2e0e2eff0e2a0e2b0e2c0e29ff0e300e300e2d0e2cff0e2d0e2f0e2c0e2eff0e2e000000000000f9, assertion: Exist, start_ts: 435573748162101255, existing start ts: 0, existing commit ts: 0\ github.com/pingcap/errors.AddStack\ \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20211224045212-9687c2b0f87c/errors.go:174\ github.com/pingcap/errors.(*Error).GenWithStackByArgs\ \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20211224045212-9687c2b0f87c/normalize.go:164\ github.com/pingcap/tidb/session.(*session).handleAssertionFailure\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/session/session.go:679\ github.com/pingcap/tidb/session.(*session).doCommit\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/session/session.go:622\ github.com/pingcap/tidb/session.(*session).doCommitWithRetry\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/session/session.go:861\ github.com/pingcap/tidb/session.(*session).CommitTxn\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/session/session.go:927\ github.com/pingcap/tidb/session.autoCommitAfterStmt\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/session/tidb.go:280\ github.com/pingcap/tidb/session.finishStmt\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/session/tidb.go:246\ github.com/pingcap/tidb/session.runStmt\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/session/session.go:2080\ github.com/pingcap/tidb/session.(*session).ExecuteStmt\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/session/session.go:1938\ github.com/pingcap/tidb/server.(*TiDBContext).ExecuteStmt\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/server/driver_tidb.go:230\ github.com/pingcap/tidb/server.(*clientConn).handleStmt\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/server/conn.go:2022\ github.com/pingcap/tidb/server.(*clientConn).handleQuery\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/server/conn.go:1876\ github.com/pingcap/tidb/server.(*clientConn).dispatch\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/server/conn.go:1371\ github.com/pingcap/tidb/server.(*clientConn).Run\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/server/conn.go:1121\ github.com/pingcap/tidb/server.(*Server).onConn\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/server/server.go:559\ runtime.goexit\ \t/usr/local/go/src/runtime/asm_amd64.s:1571\ previous statement: REPLACE INTOactivity_flow_clean.newgamesgcp(id,md5,site_id,member_id,venue_id,venue_bill_no,bet_amount,net_amount,valid_bet_amount,bet_time,settle_time,flag,created_at,updated_at`) VALUES (4228667934687145998,_binary’4001newgamesgcp220821301827168’,4001,292358,3,_binary’220821301827168’,‘311.0000’,’-311.0000’,‘311.0000’,_binary’2022-08-27T13:23:04+08:00’,_binary’2022-08-27T13:23:04+08:00’,1,‘2022-08-27 13:24:10’,‘2022-08-27 13:24:10’),(4228667934853607502,_binary’4001newgamesgcp220821301827286’,4001,82145487,3,_binary’220821301827286’,‘101.0000’,’-101.0000’,‘101.0000’,_binary’2022-08-27T13:24:04+08:00’,_binary’2022-08-27T13:24:04+08:00’,1,‘2022-08-27 13:24:10’,‘2022-08-27 13:24:10’)"] [2022/08/27 14:33:08.587 +08:00] [WARN] [2pc.go:1802] [“schemaLeaseChecker is not set for this transaction”] [sessionID=633048217743437121] [startTS=435573748830568450] [checkTS=435573748830568451]

hey-hoho · 2022 年8 月 27 日 06:44

changefeed是不是重复了

blaine · 2022 年8 月 27 日 06:47

不会啊，之前的我都remove了

HACK · 2022 年8 月 27 日 06:52

删除同步任务，注意默认任务删除后会继续保留24小时，–force是强制立刻删除。不加该参数，任务删除以后会保留24小时，这个期间不能创建同名称的任务。

执行下面的命令，看看有你删除的任务吗
cdc cli changefeed list --pd=http://xxx.xxx.xxx.xxx:2379

blaine · 2022 年8 月 27 日 07:44

我重新换了一个changefeed ，还是报错
[2022/08/27 15:42:30.631 +08:00] [WARN] [2pc.go:1802] [“schemaLeaseChecker is not set for this transaction”] [sessionID=633048217743450915] [startTS=435574839860527109] [checkTS=435574839860527110] [2022/08/27 15:42:30.946 +08:00] [WARN] [2pc.go:1802] [“schemaLeaseChecker is not set for this transaction”] [sessionID=633048217743450911] [startTS=435574839939432456] [checkTS=435574839952539649] [2022/08/27 15:42:31.890 +08:00] [WARN] [2pc.go:1802] [“schemaLeaseChecker is not set for this transaction”] [sessionID=633048217743450915] [startTS=435574840188469267] [checkTS=435574840188469268] [2022/08/27 15:42:33.952 +08:00] [ERROR] [session.go:721] [“assertion failed”] [conn=633048217743450857] [message="[tikv:8141]assertion failed: key: 7480000000000007ec5f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2a0e320e2e0e2cff0e2d0e300e2b0e2eff0e320e2e0e2c0e2bff0e2b0e2f0e2d0e29ff0e2f000000000000f9, assertion: Exist, start_ts: 435574840254005263, existing start ts: 0, existing commit ts: 0"] [“mvcc history”="{“key”:“7480000000000007EC5F698000000000000001010E2D0E290E290E2AFF0F640E8B10510EC1FF0E330F5B0E8B0EB9FF0E4A1002105E0E30FF0E2A0E320E2E0E2CFF0E2D0E300E2B0E2EFF0E320E2E0E2C0E2BFF0E2B0E2F0E2D0E29FF0E2F000000000000F9”,“mvcc”:{“info”:{“writes”:[{“type”:3,“start_ts”:435574828169166852,“commit_ts”:435574828169166852}]}},“regionID”:547981}"] [2022/08/27 15:42:33.952 +08:00] [WARN] [session.go:881] [“can not retry txn”] [conn=633048217743450857] [label=general] [error="[tikv:8141]assertion failed: key: 7480000000000007ec5f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2a0e320e2e0e2cff0e2d0e300e2b0e2eff0e320e2e0e2c0e2bff0e2b0e2f0e2d0e29ff0e2f000000000000f9, assertion: Exist, start_ts: 435574840254005263, existing start ts: 0, existing commit ts: 0"] [IsBatchInsert=false] [IsPessimistic=false] [InRestrictedSQL=false] [tidb_retry_limit=10] [tidb_disable_txn_auto_retry=true] [2022/08/27 15:42:33.952 +08:00] [WARN] [session.go:897] [“commit failed”] [conn=633048217743450857] [“finished txn”=“Txn{state=invalid}”] [error="[tikv:8141]assertion failed: key: 7480000000000007ec5f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2a0e320e2e0e2cff0e2d0e300e2b0e2eff0e320e2e0e2c0e2bff0e2b0e2f0e2d0e29ff0e2f000000000000f9, assertion: Exist, start_ts: 435574840254005263, existing start ts: 0, existing commit ts: 0"] [2022/08/27 15:42:33.952 +08:00] [WARN] [session.go:1941] [“run statement failed”] [conn=633048217743450857] [schemaVersion=894] [error=“previous statement: REPLACE INTO activity_flow_clean.red_spot_newgame_ybzr(id,site_id,member_id,venue_id,venue_bill_no,bet_amount,net_amount,valid_bet_amount,bet_time,settle_time,flag,is_second,created_at,updated_at) VALUES (4228668095914881033,4001,1463149,26,_binary’771377992140271616’,‘200.0000’,‘190.0000’,‘190.0000’,_binary’2022-08-27T13:20:45+08:00’,_binary’2022-08-27T13:21:10+08:00’,1,0,‘2022-08-27 13:24:19’,‘2022-08-27 13:24:19’),(4228668104974577740,4001,488831,26,_binary’771378251729747968’,‘20.0000’,‘20.0000’,‘20.0000’,_binary’2022-08-27T13:21:47+08:00’,_binary’2022-08-27T13:22:10+08:00’,1,0,‘2022-08-27 13:24:20’,‘2022-08-27 13:24:20’): [tikv:8141]assertion failed: key: 7480000000000007ec5f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2a0e320e2e0e2cff0e2d0e300e2b0e2eff0e320e2e0e2c0e2bff0e2b0e2f0e2d0e29ff0e2f000000000000f9, assertion: Exist, start_ts: 435574840254005263, existing start ts: 0, existing commit ts: 0”] [session="{\ “currDBName”: “”,\ “id”: 633048217743450857,\ “status”: 2,\ “strictMode”: true,\ “user”: {\ “Username”: “dba_tools”,\ “Hostname”: “10.10.11.109”,\ “CurrentUser”: false,\ “AuthUsername”: “dba_tools”,\ “AuthHostname”: “10.%”\ }\ }"]

blaine · 2022 年8 月 27 日 08:00

这个事ticdc的日志报错信息；

[2022/08/27 15:59:22.213 +08:00] [WARN] [mysql.go:935] [“execute DMLs with error, retry later”] [error="[CDC:ErrMySQLTxnError]Error 8141: assertion failed: key: 7480000000000007ec5f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2a0e320e2b0e2dff0e300e2f0e2d0e2aff0e300e2d0e320e2bff0e2b0e2e0e2e0e31ff0e30000000000000f9, assertion: Exist, start_ts: 435575104573538305, existing start ts: 0, existing commit ts: 0: Error 8141: assertion failed: key: 7480000000000007ec5f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2a0e320e2b0e2dff0e300e2f0e2d0e2aff0e300e2d0e320e2bff0e2b0e2e0e2e0e31ff0e30000000000000f9, assertion: Exist, start_ts: 435575104573538305, existing start ts: 0, existing commit ts: 0"] [errorVerbose="[CDC:ErrMySQLTxnError]Error 8141: assertion failed: key: 7480000000000007ec5f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2a0e320e2b0e2dff0e300e2f0e2d0e2aff0e300e2d0e320e2bff0e2b0e2e0e2e0e31ff0e30000000000000f9, assertion: Exist, start_ts: 435575104573538305, existing start ts: 0, existing commit ts: 0: Error 8141: assertion failed: key: 7480000000000007ec5f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2a0e320e2b0e2dff0e300e2f0e2d0e2aff0e300e2d0e320e2bff0e2b0e2e0e2e0e31ff0e30000000000000f9, assertion: Exist, start_ts: 435575104573538305, existing start ts: 0, existing commit ts: 0\ngithub.com/pingcap/errors.AddStack\ \tgithub.com/pingcap/errors@v0.11.5-0.20210513014640-40f9a1999b3b/errors.go:174\ github.com/pingcap/errors.(*Error).GenWithStackByCause\ \tgithub.com/pingcap/errors@v0.11.5-0.20210513014640-40f9a1999b3b/normalize.go:302\ github.com/pingcap/ticdc/pkg/errors.WrapError\ \tgithub.com/pingcap/ticdc/pkg/errors/helper.go:30\ github.com/pingcap/ticdc/cdc/sink.(*mysqlSink).execDMLWithMaxRetries.func1.3\ \tgithub.com/pingcap/ticdc/cdc/sink/mysql.go:1000\ github.com/pingcap/ticdc/cdc/sink.(*Statistics).RecordBatchExecution\ \tgithub.com/pingcap/ticdc/cdc/sink/statistics.go:112\ github.com/pingcap/ticdc/cdc/sink.(*mysqlSink).execDMLWithMaxRetries.func1\ \tgithub.com/pingcap/ticdc/cdc/sink/mysql.go:972\ github.com/pingcap/ticdc/pkg/retry.run\ \tgithub.com/pingcap/ticdc/pkg/retry/retry_with_opt.go:54\ github.com/pingcap/ticdc/pkg/retry.Do\ \tgithub.com/pingcap/ticdc/pkg/retry/retry_with_opt.go:32\ github.com/pingcap/ticdc/cdc/sink.(*mysqlSink).execDMLWithMaxRetries\ \tgithub.com/pingcap/ticdc/cdc/sink/mysql.go:964\ github.com/pingcap/ticdc/cdc/sink.(*mysqlSink).execDMLs\ \tgithub.com/pingcap/ticdc/cdc/sink/mysql.go:1125\ github.com/pingcap/ticdc/cdc/sink.(*mysqlSinkWorker).run.func3\ \tgithub.com/pingcap/ticdc/cdc/sink/mysql.go:820\ github.com/pingcap/ticdc/cdc/sink.(*mysqlSinkWorker).run\ \tgithub.com/pingcap/ticdc/cdc/sink/mysql.go:858\ github.com/pingcap/ticdc/cdc/sink.(*mysqlSink).createSinkWorkers.func1\ \tgithub.com/pingcap/ticdc/cdc/sink/mysql.go:661\ runtime.goexit\ \truntime/asm_amd64.s:1371"]
[2022/08/27 15:59:26.881 +08:00] [WARN] [mysql.go:935] [“execute DMLs with error, retry later”] [error="[CDC:ErrMySQLTxnError]Error 8141: assertion failed: key: 7480000000000007ec5f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2a0e320e2e0e2cff0e2d0e300e2b0e2eff0e320e2e0e2c0e2bff0e2b0e2f0e2d0e29ff0e2f000000000000f9, assertion: Exist, start_ts: 435575105818984451, existing start ts: 0, existing commit ts: 0: Error 8141: assertion failed: key: 7480000000000007ec5f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2a0e320e2e0e2cff0e2d0e300e2b0e2eff0e320e2e0e2c0e2bff0e2b0e2f0e2d0e29ff0e2f000000000000f9, assertion: Exist, start_ts: 435575105818984451, existing start ts: 0, existing commit ts: 0"] [errorVerbose="[CDC:ErrMySQLTxnError]Error 8141: assertion failed: key: 7480000000000007ec5f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2a0e320e2e0e2cff0e2d0e300e2b0e2eff0e320e2e0e2c0e2bff0e2b0e2f0e2d0e29ff0e2f000000000000f9, assertion: Exist, start_ts: 435575105818984451, existing start ts: 0, existing commit ts: 0: Error 8141: assertion failed: key: 7480000000000007ec5f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2a0e320e2e0e2cff0e2d0e300e2b0e2eff0e320e2e0e2c0e2bff0e2b0e2f0e2d0e29ff0e2f000000000000f9, assertion: Exist, start_ts: 435575105818984451, existing start ts: 0, existing commit ts: 0\ngithub.com/pingcap/errors.AddStack\ \tgithub.com/pingcap/errors@v0.11.5-0.20210513014640-40f9a1999b3b/errors.go:174\ github.com/pingcap/errors.(*Error).GenWithStackByCause\ \tgithub.com/pingcap/errors@v0.11.5-0.20210513014640-40f9a1999b3b/normalize.go:302\ github.com/pingcap/ticdc/pkg/errors.WrapError\ \tgithub.com/pingcap/ticdc/pkg/errors/helper.go:30\ github.com/pingcap/ticdc/cdc/sink.(*mysqlSink).execDMLWithMaxRetries.func1.3\ \tgithub.com/pingcap/ticdc/cdc/sink/mysql.go:1000\ github.com/pingcap/ticdc/cdc/sink.(*Statistics).RecordBatchExecution\ \tgithub.com/pingcap/ticdc/cdc/sink/statistics.go:112\ github.com/pingcap/ticdc/cdc/sink.(*mysqlSink).execDMLWithMaxRetries.func1\ \tgithub.com/pingcap/ticdc/cdc/sink/mysql.go:972\ github.com/pingcap/ticdc/pkg/retry.run\ \tgithub.com/pingcap/ticdc/pkg/retry/retry_with_opt.go:54\ github.com/pingcap/ticdc/pkg/retry.Do\ \tgithub.com/pingcap/ticdc/pkg/retry/retry_with_opt.go:32\ github.com/pingcap/ticdc/cdc/sink.(*mysqlSink).execDMLWithMaxRetries\ \tgithub.com/pingcap/ticdc/cdc/sink/mysql.go:964\ github.com/pingcap/ticdc/cdc/sink.(*mysqlSink).execDMLs\ \tgithub.com/pingcap/ticdc/cdc/sink/mysql.go:1125\ github.com/pingcap/ticdc/cdc/sink.(*mysqlSinkWorker).run.func3\ \tgithub.com/pingcap/ticdc/cdc/sink/mysql.go:820\ github.com/pingcap/ticdc/cdc/sink.(*mysqlSinkWorker).run\ \tgithub.com/pingcap/ticdc/cdc/sink/mysql.go:858\ github.com/pingcap/ticdc/cdc/sink.(*mysqlSink).createSinkWorkers.func1\ \tgithub.com/pingcap/ticdc/cdc/sink/mysql.go:661\ runtime.goexit\ \truntime/asm_amd64.s:1371"]

blaine · 2022 年8 月 27 日 10:09

状态也不报错，就是卡在那里

blaine · 2022 年8 月 28 日 05:48

最后我将将整个库的每个表都建立一个任务同步，发现其他的表都可以同步。但是有一个表无法同步。报错如上。我怎么感觉数据都过来了，但是6.1.0的版本没有将数据写入库中。我手动将数据拿出来，REPLACE INTO也是可以进入的。很奇怪。2边的表结构都是一致的。我都是源库备份出来的。
[2022/08/28 13:38:24.207 +08:00] [ERROR] [session.go:721] [“assertion failed”] [conn=633048217743695731] [message="[tikv:8141]assertion failed: key: 7480000000000008605f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2b0e290e290e32ff0e310e2c0e2d0e2eff0e320e2b0e2a0e2eff0e2d0e2d0e2d0e29ff0e31000000000000f9, assertion: Exist, start_ts: 435595536483483650, existing start ts: 0, existing commit ts: 0"] [“mvcc history”="{“key”:“7480000000000008605F698000000000000001010E2D0E290E290E2AFF0F640E8B10510EC1FF0E330F5B0E8B0EB9FF0E4A1002105E0E30FF0E2B0E290E290E32FF0E310E2C0E2D0E2EFF0E320E2B0E2A0E2EFF0E2D0E2D0E2D0E29FF0E31000000000000F9”,“mvcc”:{“info”:{“writes”:[{“type”:3,“start_ts”:435595533141147649,“commit_ts”:435595533141147649}]}},“regionID”:662985}"] [2022/08/28 13:38:24.207 +08:00] [WARN] [session.go:881] [“can not retry txn”] [conn=633048217743695731] [label=general] [error="[tikv:8141]assertion failed: key: 7480000000000008605f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2b0e290e290e32ff0e310e2c0e2d0e2eff0e320e2b0e2a0e2eff0e2d0e2d0e2d0e29ff0e31000000000000f9, assertion: Exist, start_ts: 435595536483483650, existing start ts: 0, existing commit ts: 0"] [IsBatchInsert=false] [IsPessimistic=false] [InRestrictedSQL=false] [tidb_retry_limit=10] [tidb_disable_txn_auto_retry=true] [2022/08/28 13:38:24.207 +08:00] [WARN] [session.go:897] [“commit failed”] [conn=633048217743695731] [“finished txn”=“Txn{state=invalid}”] [error="[tikv:8141]assertion failed: key: 7480000000000008605f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2b0e290e290e32ff0e310e2c0e2d0e2eff0e320e2b0e2a0e2eff0e2d0e2d0e2d0e29ff0e31000000000000f9, assertion: Exist, start_ts: 435595536483483650, existing start ts: 0, existing commit ts: 0"] [2022/08/28 13:38:24.207 +08:00] [WARN] [session.go:1941] [“run statement failed”] [conn=633048217743695731] [schemaVersion=903] [error=“previous statement: REPLACE INTO activity_flow_clean.newgamefbty(id,md5,site_id,member_id,venue_id,venue_bill_no,bet_amount,net_amount,valid_bet_amount,bet_time,settle_time,flag,created_at,updated_at) VALUES (4230125943377014874,_binary’4001newgamefbty720322031375090287’,4001,82066823,44,_binary’720322031375090287’,‘99.0000’,’-22.8700’,‘22.8700’,_binary’2022-08-28T13:28:41+08:00’,_binary’2022-08-28T13:32:49+08:00’,1,‘2022-08-28 13:32:34’,‘2022-08-28 13:36:34’),(4230129965713240077,_binary’4001newgamefbty720324875650074229’,4001,1897470,44,_binary’720324875650074229’,‘500.0000’,‘0.0000’,‘0.0000’,_binary’2022-08-28T13:34:12+08:00’,_binary’1900-01-01T00:00:00+08:00’,0,‘2022-08-28 13:36:34’,‘2022-08-28 13:36:34’),(4230129965881012277,_binary’4001newgamefbty720322723603023529’,4001,4181247,44,_binary’720322723603023529’,‘98.0000’,‘0.0000’,‘0.0000’,_binary’2022-08-28T13:30:01+08:00’,_binary’1900-01-01T00:00:00+08:00’,0,‘2022-08-28 13:36:34’,‘2022-08-28 13:36:34’),(4230129966049439752,_binary’4001newgamefbty720324187448676366’,4001,80020122,44,_binary’720324187448676366’,‘11.3900’,‘0.0000’,‘0.0000’,_binary’2022-08-28T13:32:52+08:00’,_binary’1900-01-01T00:00:00+08:00’,0,‘2022-08-28 13:36:34’,‘2022-08-28 13:36:34’),(4230129966216556590,_binary’4001newgamefbty720323333488379413’,4001,3011750,44,_binary’720323333488379413’,‘280.0000’,‘0.0000’,‘0.0000’,_binary’2022-08-28T13:31:12+08:00’,_binary’1900-01-01T00:00:00+08:00’,0,‘2022-08-28 13:36:34’,‘2022-08-28 13:36:34’),(4230129966385639482,_binary’4001newgamefbty720323015660798614’,4001,82425985,44,_binary’720323015660798614’,‘17.0000’,‘0.0000’,‘0.0000’,_binary’2022-08-28T13:30:35+08:00’,_binary’1900-01-01T00:00:00+08:00’,0,‘2022-08-28 13:36:34’,‘2022-08-28 13:36:34’),(4230129966720528408,_binary’4001newgamefbty720323886800967224’,4001,4081859,44,_binary’720323886800967224’,‘10.0000’,‘0.0000’,‘0.0000’,_binary’2022-08-28T13:32:17+08:00’,_binary’1900-01-01T00:00:00+08:00’,0,‘2022-08-28 13:36:34’,‘2022-08-28 13:36:34’),(4230129966726426649,_binary’4001newgamefbty720322994454402865’,4001,82314115,44,_binary’720322994454402865’,‘40.0000’,‘0.0000’,‘0.0000’,_binary’2022-08-28T13:30:33+08:00’,_binary’1900-01-01T00:00:00+08:00’,0,‘2022-08-28 13:36:34’,‘2022-08-28 13:36:34’): [tikv:8141]assertion failed: key: 7480000000000008605f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2b0e290e290e32ff0e310e2c0e2d0e2eff0e320e2b0e2a0e2eff0e2d0e2d0e2d0e29ff0e31000000000000f9, assertion: Exist, start_ts: 435595536483483650, existing start ts: 0, existing commit ts: 0”] [session="{\ “currDBName”: “”,\ “id”: 633048217743695731,\ “status”: 2,\ “strictMode”: true,\ “user”: {\ “Username”: “dba_tools”,\ “Hostname”: “10.10.11.108”,\ “CurrentUser”: false,\ “AuthUsername”: “dba_tools”,\ “AuthHostname”: “10.%”\ }\ }"] [2022/08/28 13:38:24.207 +08:00] [INFO] [conn.go:1149] [“command dispatched failed”] [conn=633048217743695731] [connInfo=“id:633048217743695731, addr:10.10.11.108:46833 status:10, collation:utf8mb4_general_ci, user:dba_tools”] [command=Query] [status=“inTxn:0, autocommit:1”] [sql=COMMIT] [txn_mode=OPTIMISTIC] [timestamp=435595536483483650] [err="[tikv:8141]assertion failed: key: 7480000000000008605f698000000000000001010e2d0e290e290e2aff0f640e8b10510ec1ff0e330f5b0e8b0eb9ff0e4a1002105e0e30ff0e2b0e290e290e32ff0e310e2c0e2d0e2eff0e320e2b0e2a0e2eff0e2d0e2d0e2d0e29ff0e31000000000000f9, assertion: Exist, start_ts: 435595536483483650, existing start ts: 0, existing commit ts: 0\ngithub.com/pingcap/errors.AddStack\ \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20211224045212-9687c2b0f87c/errors.go:174\ github.com/pingcap/errors.(*Error).GenWithStackByArgs\ \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20211224045212-9687c2b0f87c/normalize.go:164\ngithub.com/pingcap/tidb/session.(*session).handleAssertionFailure\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/session/session.go:679\ngithub.com/pingcap/tidb/session.(*session).doCommit\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/session/session.go:622\ngithub.com/pingcap/tidb/session.(*session).doCommitWithRetry\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/session/session.go:861\ngithub.com/pingcap/tidb/session.(*session).CommitTxn\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/session/session.go:927\ngithub.com/pingcap/tidb/session.autoCommitAfterStmt\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/session/tidb.go:280\ github.com/pingcap/tidb/session.finishStmt\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/session/tidb.go:246\ github.com/pingcap/tidb/session.runStmt\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/session/session.go:2080\ github.com/pingcap/tidb/session.(*session).ExecuteStmt\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/session/session.go:1938\ngithub.com/pingcap/tidb/server.(*TiDBContext).ExecuteStmt\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/server/driver_tidb.go:230\ngithub.com/pingcap/tidb/server.(*clientConn).handleStmt\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/server/conn.go:2022\ngithub.com/pingcap/tidb/server.(*clientConn).handleQuery\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/server/conn.go:1876\ngithub.com/pingcap/tidb/server.(*clientConn).dispatch\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/server/conn.go:1371\ngithub.com/pingcap/tidb/server.(*clientConn).Run\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/server/conn.go:1121\ngithub.com/pingcap/tidb/server.(*Server).onConn\ \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/server/server.go:559\ runtime.goexit\ \t/usr/local/go/src/runtime/asm_amd64.s:1571\ previous statement: REPLACE INTO activity_flow_clean.newgamefbty(id,md5,site_id,member_id,venue_id,venue_bill_no,bet_amount,net_amount,valid_bet_amount,bet_time,settle_time,flag,created_at,updated_at) VALUES (4230125943377014874,_binary’4001newgamefbty720322031375090287’,4001,82066823,44,_binary’720322031375090287’,‘99.0000’,’-22.8700’,‘22.8700’,_binary’2022-08-28T13:28:41+08:00’,_binary’2022-08-28T13:32:49+08:00’,1,‘2022-08-28 13:32:34’,‘2022-08-28 13:36:34’),(4230129965713240077,_binary’4001newgamefbty720324875650074229’,4001,1897470,44,_binary’720324875650074229’,‘500.0000’,‘0.0000’,‘0.0000’,_binary’2022-08-28T13:34:12+08:00’,_binary’1900-01-01T00:00:00+08:00’,0,‘2022-08-28 13:36:34’,‘2022-08-28 13:36:34’),(4230129965881012277,_binary’4001newgamefbty720322723603023529’,4001,4181247,44,_binary’720322723603023529’,‘98.0000’,‘0.0000’,‘0.0000’,_binary’2022-08-28T13:30:01+08:00’,_binary’1900-01-01T00:00:00+08:00’,0,‘2022-08-28 13:36:34’,‘2022-08-28 13:36:34’),(4230129966049439752,_binary’4001newgamefbty720324187448676366’,4001,80020122,44,_binary’720324187448676366’,‘11.3900’,‘0.0000’,‘0.0000’,_binary’2022-08-28T13:32:52+08:00’,_binary’1900-01-01T00:00:00+08:00’,0,‘2022-08-28 13:36:34’,‘2022-08-28 13:36:34’),(4230129966216556590,_binary’4001newgamefbty720323333488379413’,4001,3011750,44,_binary’720323333488379413’,‘280.0000’,‘0.0000’,‘0.0000’,_binary’2022-08-28T13:31:12+08:00’,_binary’1900-01-01T00:00:00+08:00’,0,‘2022-08-28 13:36:34’,‘2022-08-28 13:36:34’),(4230129966385639482,_binary’4001newgamefbty720323015660798614’,4001,82425985,44,_binary’720323015660798614’,‘17.0000’,‘0.0000’,‘0.0000’,_binary’2022-08-28T13:30:35+08:00’,_binary’1900-01-01T00:00:00+08:00’,0,‘2022-08-28 13:36:34’,‘2022-08-28 13:36:34’),(4230129966720528408,_binary’4001newgamefbty720323886800967224’,4001,4081859,44,_binary’720323886800967224’,‘10.0000’,‘0.0000’,‘0.0000’,_binary’2022-08-28T13:32:17+08:00’,_binary’1900-01-01T00:00:00+08:00’,0,‘2022-08-28 13:36:34’,‘2022-08-28 13:36:34’),(4230129966726426649,_binary’4001newgamefbty720322994454402865’,4001,82314115,44,_binary’720322994454402865’,‘40.0000’,‘0.0000’,‘0.0000’,_binary’2022-08-28T13:30:33+08:00’,_binary’1900-01-01T00:00:00+08:00’,0,‘2022-08-28 13:36:34’,‘2022-08-28 13:36:34’)"]

blaine · 2022 年8 月 28 日 06:52

经过最后的测试，我在目标端单独创建一个一样的表，再同步就可以数据入库。用备份还原就会报错，无法同步。这个问题以前我在4.0.9的版本中也遇到过。类似的问题。希望经过我的测试，可以给其他的朋友有帮助。最终是因为2边源端没有开启新的字符集模版，目标端开启了新的模版，因字符集的原因导致数据无法写入

system · 2022 年10 月 27 日 06:53

此话题已在最后回复的 60 天后被自动关闭。不再允许新回复。