DM 原来同步一个库，单表大概20多亿，binlog丢了导致断开，清掉下游dm_meta 中的数据，和表的数据，重新同步，结果瞬间跳过了全量阶段，s3上也没有dump的sql文件，直接跳到了sync阶段。
是我哪里没清理完吗？dm也没报错。
有这一步不应该出这个问题。
或者在start-task的时候加 --remove-meta试试。
跳过全量阶段，然后从你start开始的时间这个时间点开始同步之后的数据？？
问题还没确定，大概是我配置的问题。
全部的情况是这样：
总共13个表，有个别表比较大，能到20亿，故障后我担心全量同步时间太长，就改了配置，先全量同步一张表，再删别的表，再同步别的表，这样一个一个的恢复出来。
这个改了配置以后，任务一启动就变成增量阶段了，现在先用老配置启动了，dump时间长点就长点吧。
我把配置脱敏后贴下,主要改了2个部分，一个是 route 删掉了1-9，另一个是filter从 all dml 改成了 all，之前没同步ddl
帮忙看卡这个配置有没有什么明显错误，如果没有明显错误的话，后面我排查出来问题后再回复到这个帖子里面。：
全部的配置：
name: yewudb_all
shard-mode: "pessimistic"
task-mode: all
#ignore-checking-items: ["all"]
# timezone: "UTC"
validators:
global:
mode: full
worker-count: 1
row-error-delay: 30m
mydumpers:
global:
extra-args: "--consistency none"
threads: 8
loaders:
global:
dir: "s3://dmsync/yewudb-tidb?endpoint=https://s3.com&access_key=akakakak&secret_access_key=sksksksk&force_path_style=true"
pool-size: 32
syncers:
global:
worker-count: 16
batch: 200
compact: true
multiple-rows: true
safe-mode: true
target-database:
host: "tidb-ip"
port: 4000
user: "test"
password: "testpassword"
block-allow-list:
bw-rule-1:
do-dbs: ["order_*"]
do-tables:
- db-name: "order_*"
tbl-name: "*"
filters:
filter-rule-1:
schema-pattern: "order_*"
table-pattern: "*"
events: ["all dml"]
action: Do
routes:
route-rule-1:
schema-pattern: "order_*"
table-pattern: "trade_*"
target-schema: "dd_order"
target-table: "trade"
route-rule-2:
schema-pattern: "order_*"
table-pattern: "settle1_*"
target-schema: "dd_order"
target-table: "settle1"
route-rule-3:
schema-pattern: "order_*"
table-pattern: "settle2_*"
target-schema: "dd_order"
target-table: "settle2"
route-rule-4:
schema-pattern: "order_*"
table-pattern: "refund*"
target-schema: "dd_order"
target-table: "refund"
route-rule-5:
schema-pattern: "order_*"
table-pattern: "pop1*"
target-schema: "dd_order"
target-table: "pop1"
route-rule-6:
schema-pattern: "order_*"
table-pattern: "pop2[0-9]*"
target-schema: "dd_order"
target-table: "pop2"
route-rule-7:
schema-pattern: "order_*"
table-pattern: "order_*"
target-schema: "dd_order"
target-table: "order"
route-rule-8:
schema-pattern: "order_*"
table-pattern: "merchant1_*"
target-schema: "dd_order"
target-table: "merchant1"
route-rule-9:
schema-pattern: "order_*"
table-pattern: "merchant2_*"
target-schema: "dd_order"
target-table: "merchant2"
route-rule-10:
schema-pattern: "order_*"
table-pattern: "fund_*"
target-schema: "dd_order"
target-table: "fund"
route-rule-11:
schema-pattern: "order_*"
table-pattern: "clear"
target-schema: "dd_order"
target-table: "clear"
route-rule-12:
schema-pattern: "order_*"
table-pattern: "account_*"
target-schema: "dd_order"
target-table: "account"
mysql-instances:
- source-id: "orde-db-1202"
block-allow-list: "bw-rule-1"
filter-rules: ["filter-rule-1"]
route-rules: ["route-rule-1","route-rule-2","route-rule-3","route-rule-4","route-rule-5","route-rule-6","route-rule-7","route-rule-8","route-rule-9","route-rule-10","route-rule-11","route-rule-12"]
mydumper-config-name: "global"
loader-config-name: "global"
syncer-config-name: "global"
validator-config-name: "global"
- source-id: "orde-db-2102"
block-allow-list: "bw-rule-1"
filter-rules: ["filter-rule-1"]
route-rules: ["route-rule-1","route-rule-2","route-rule-3","route-rule-4","route-rule-5","route-rule-6","route-rule-7","route-rule-8","route-rule-9","route-rule-10","route-rule-11","route-rule-12"]
mydumper-config-name: "global"
loader-config-name: "global"
syncer-config-name: "global"
validator-config-name: "global"
- source-id: "orde-db-2402"
block-allow-list: "bw-rule-1"
filter-rules: ["filter-rule-1"]
route-rules: ["route-rule-1","route-rule-2","route-rule-3","route-rule-4","route-rule-5","route-rule-6","route-rule-7","route-rule-8","route-rule-9","route-rule-10","route-rule-11","route-rule-12"]
mydumper-config-name: "global"
loader-config-name: "global"
syncer-config-name: "global"
validator-config-name: "global"
- source-id: "orde-db-2802"
block-allow-list: "bw-rule-1"
filter-rules: ["filter-rule-1"]
route-rules: ["route-rule-1","route-rule-2","route-rule-3","route-rule-4","route-rule-5","route-rule-6","route-rule-7","route-rule-8","route-rule-9","route-rule-10","route-rule-11","route-rule-12"]
mydumper-config-name: "global"
loader-config-name: "global"
syncer-config-name: "global"
validator-config-name: "global"
- source-id: "orde-db-0802"
block-allow-list: "bw-rule-1"
filter-rules: ["filter-rule-1"]
route-rules: ["route-rule-1","route-rule-2","route-rule-3","route-rule-4","route-rule-5","route-rule-6","route-rule-7","route-rule-8","route-rule-9","route-rule-10","route-rule-11","route-rule-12"]
mydumper-config-name: "global"
loader-config-name: "global"
syncer-config-name: "global"
validator-config-name: "global"
- source-id: "orde-db-0202"
block-allow-list: "bw-rule-1"
filter-rules: ["filter-rule-1"]
route-rules: ["route-rule-1","route-rule-2","route-rule-3","route-rule-4","route-rule-5","route-rule-6","route-rule-7","route-rule-8","route-rule-9","route-rule-10","route-rule-11","route-rule-12"]
mydumper-config-name: "global"
loader-config-name: "global"
syncer-config-name: "global"
validator-config-name: "global"
- source-id: "orde-db-0602"
block-allow-list: "bw-rule-1"
filter-rules: ["filter-rule-1"]
route-rules: ["route-rule-1","route-rule-2","route-rule-3","route-rule-4","route-rule-5","route-rule-6","route-rule-7","route-rule-8","route-rule-9","route-rule-10","route-rule-11","route-rule-12"]
mydumper-config-name: "global"
loader-config-name: "global"
syncer-config-name: "global"
validator-config-name: "global"
- source-id: "orde-db-1602"
block-allow-list: "bw-rule-1"
filter-rules: ["filter-rule-1"]
route-rules: ["route-rule-1","route-rule-2","route-rule-3","route-rule-4","route-rule-5","route-rule-6","route-rule-7","route-rule-8","route-rule-9","route-rule-10","route-rule-11","route-rule-12"]
mydumper-config-name: "global"
loader-config-name: "global"
syncer-config-name: "global"
validator-config-name: "global"
修改后，同步部分表的配置：
name: yewudb_all
shard-mode: "pessimistic"
task-mode: all
#ignore-checking-items: ["all"]
# timezone: "UTC"
validators:
global:
mode: full
worker-count: 1
row-error-delay: 30m
mydumpers:
global:
extra-args: "--consistency none"
threads: 8
loaders:
global:
dir: "s3://dmsync/yewudb-tidb?endpoint=https://s3.com&access_key=akakakak&secret_access_key=sksksksk&force_path_style=true"
pool-size: 32
syncers:
global:
worker-count: 16
batch: 200
compact: true
multiple-rows: true
safe-mode: true
target-database:
host: "tidb-ip"
port: 4000
user: "test"
password: "testpassword"
block-allow-list:
bw-rule-1:
do-dbs: ["order_*"]
do-tables:
- db-name: "order_*"
tbl-name: "*"
filters:
filter-rule-1:
schema-pattern: "order_*"
table-pattern: "*"
events: ["all"]
action: Do
routes:
route-rule-10:
schema-pattern: "order_*"
table-pattern: "fund_*"
target-schema: "dd_order"
target-table: "fund"
route-rule-11:
schema-pattern: "order_*"
table-pattern: "clear"
target-schema: "dd_order"
target-table: "clear"
route-rule-12:
schema-pattern: "order_*"
table-pattern: "account_*"
target-schema: "dd_order"
target-table: "account"
mysql-instances:
- source-id: "orde-db-1202"
block-allow-list: "bw-rule-1"
filter-rules: ["filter-rule-1"]
route-rules: ["route-rule-10","route-rule-11","route-rule-12"]
mydumper-config-name: "global"
loader-config-name: "global"
syncer-config-name: "global"
validator-config-name: "global"
- source-id: "orde-db-2102"
block-allow-list: "bw-rule-1"
filter-rules: ["filter-rule-1"]
route-rules: ["route-rule-10","route-rule-11","route-rule-12"]
mydumper-config-name: "global"
loader-config-name: "global"
syncer-config-name: "global"
validator-config-name: "global"
- source-id: "orde-db-2402"
block-allow-list: "bw-rule-1"
filter-rules: ["filter-rule-1"]
route-rules: ["route-rule-10","route-rule-11","route-rule-12"]
mydumper-config-name: "global"
loader-config-name: "global"
syncer-config-name: "global"
validator-config-name: "global"
- source-id: "orde-db-2802"
block-allow-list: "bw-rule-1"
filter-rules: ["filter-rule-1"]
route-rules: ["route-rule-10","route-rule-11","route-rule-12"]
mydumper-config-name: "global"
loader-config-name: "global"
syncer-config-name: "global"
validator-config-name: "global"
- source-id: "orde-db-0802"
block-allow-list: "bw-rule-1"
filter-rules: ["filter-rule-1"]
route-rules: ["route-rule-10","route-rule-11","route-rule-12"]
mydumper-config-name: "global"
loader-config-name: "global"
syncer-config-name: "global"
validator-config-name: "global"
- source-id: "orde-db-0202"
block-allow-list: "bw-rule-1"
filter-rules: ["filter-rule-1"]
route-rules: ["route-rule-10","route-rule-11","route-rule-12"]
mydumper-config-name: "global"
loader-config-name: "global"
syncer-config-name: "global"
validator-config-name: "global"
- source-id: "orde-db-0602"
block-allow-list: "bw-rule-1"
filter-rules: ["filter-rule-1"]
route-rules: ["route-rule-10","route-rule-11","route-rule-12"]
mydumper-config-name: "global"
loader-config-name: "global"
syncer-config-name: "global"
validator-config-name: "global"
- source-id: "orde-db-1602"
block-allow-list: "bw-rule-1"
filter-rules: ["filter-rule-1"]
route-rules: ["route-rule-10","route-rule-11","route-rule-12"]
mydumper-config-name: "global"
loader-config-name: "global"
syncer-config-name: "global"
validator-config-name: "global"
可能是吧，也没仔细看是哪个时间点，我看s3上没有一点dump的文件，肯定不对。
这个感觉还是建议重新建立同步，把数据导出然后导入tidb，记录导出的binlog 重新开始
看不出有什么问题。我想如果dump，load阶段几乎没有，日志上也许能找到些什么。