tidb DM 在task-mode: all模式下增量数据没用同步到目标数据库

Hacker_IHDinkoC · 2020 年6 月 30 日 08:48

问题

Hacker_IHDinkoC · 2020 年6 月 30 日 08:49

test4-task.yaml (13.3 KB) dm-master.toml (1.0 KB) dm-worker.toml (905 字节)

Hacker_IHDinkoC · 2020 年6 月 30 日 08:53

这里，增量数据同步进度不一致。

Hacker_IHDinkoC · 2020 年6 月 30 日 09:02

用的版本是dm-v1.0.6-linux-amd64

来了老弟 · 2020 年6 月 30 日 09:03

你好

通过文档将 syncer 和 loader 标签配置一下

https://docs.pingcap.com/zh/tidb-data-migration/v1.0/task-configuration-file-full
关于 table 复制可以看下 black write list
https://docs.pingcap.com/zh/tidb-data-migration/v1.0/feature-overview#black--white-table-lists

PS：开帖请正确选择帖子标签和分类

Hacker_IHDinkoC · 2020 年6 月 30 日 09:25

我的任务文件 test4-task.yaml 是通过DM portal生成的。不应该出现黑白名单和路由配置错误

来了老弟 · 2020 年6 月 30 日 09:31

你好，清理现有 task 和 dm-mate 数据库。并使用这个 task 文件看下是否可以复制。该 task 新增了 loader 和 syncer 的配置项，在上面 task 附件中并没有看到类似配置。

name: test4
task-mode: all
is-sharding: false
target-database:
  host: 192.168.1.236
  port: 3316
  user: root
  password: Mx7+fMj9cxKJAAISB4keRkBUvOFt7Q==
mysql-instances:
- source-id: mysql-replica-01
  meta:
    binlog-name: mysql-bin.0000001
    binlog-pos: 4
  filter-rules:
  - mysql-replica-01.filter.19
  - mysql-replica-01.filter.10
  - mysql-replica-01.filter.13
  - mysql-replica-01.filter.15
  - mysql-replica-01.filter.1
  - mysql-replica-01.filter.5
  - mysql-replica-01.filter.17
  - mysql-replica-01.filter.9
  - mysql-replica-01.filter.12
  - mysql-replica-01.filter.2
  - mysql-replica-01.filter.4
  - mysql-replica-01.filter.6
  - mysql-replica-01.filter.11
  - mysql-replica-01.filter.14
  - mysql-replica-01.filter.16
  - mysql-replica-01.filter.18
  - mysql-replica-01.filter.3
  - mysql-replica-01.filter.7
  - mysql-replica-01.filter.8
  route-rules:
  - mysql-replica-01.route_rules.4
  - mysql-replica-01.route_rules.7
  - mysql-replica-01.route_rules.8
  - mysql-replica-01.route_rules.9
  - mysql-replica-01.route_rules.19
  - mysql-replica-01.route_rules.6
  - mysql-replica-01.route_rules.11
  - mysql-replica-01.route_rules.13
  - mysql-replica-01.route_rules.14
  - mysql-replica-01.route_rules.17
  - mysql-replica-01.route_rules.18
  - mysql-replica-01.route_rules.1
  - mysql-replica-01.route_rules.3
  - mysql-replica-01.route_rules.5
  - mysql-replica-01.route_rules.16
  - mysql-replica-01.route_rules.2
  - mysql-replica-01.route_rules.10
  - mysql-replica-01.route_rules.12
  - mysql-replica-01.route_rules.15
  black-white-list: mysql-replica-01.bw_list.1
  mydumper-config-name: mysql-replica-01.dump
  loader-config-name: "global"            # loader 配置名称
  syncer-config-name: "global"            # Syncer 配置名称
routes:
  mysql-replica-01.route_rules.1:
    schema-pattern: sms_user
    table-pattern: ""
    target-schema: sms_user
    target-table: ""
  mysql-replica-01.route_rules.2:
    schema-pattern: sms_user
    table-pattern: application_manage
    target-schema: sms_user
    target-table: application_manage
  mysql-replica-01.route_rules.3:
    schema-pattern: sms_user
    table-pattern: dic_city
    target-schema: sms_user
    target-table: dic_city
  mysql-replica-01.route_rules.4:
    schema-pattern: sms_user
    table-pattern: dic_country
    target-schema: sms_user
    target-table: dic_country
  mysql-replica-01.route_rules.5:
    schema-pattern: sms_user
    table-pattern: dic_device_model
    target-schema: sms_user
    target-table: dic_device_model
  mysql-replica-01.route_rules.6:
    schema-pattern: sms_user
    table-pattern: dic_device_purpose
    target-schema: sms_user
    target-table: dic_device_purpose
  mysql-replica-01.route_rules.7:
    schema-pattern: sms_user
    table-pattern: dic_device_type
    target-schema: sms_user
    target-table: dic_device_type
  mysql-replica-01.route_rules.8:
    schema-pattern: sms_user
    table-pattern: dic_line
    target-schema: sms_user
    target-table: dic_line
  mysql-replica-01.route_rules.9:
    schema-pattern: sms_user
    table-pattern: dic_organization
    target-schema: sms_user
    target-table: dic_organization
  mysql-replica-01.route_rules.10:
    schema-pattern: sms_user
    table-pattern: dic_province
    target-schema: sms_user
    target-table: dic_province
  mysql-replica-01.route_rules.11:
    schema-pattern: sms_user
    table-pattern: dic_station
    target-schema: sms_user
    target-table: dic_station
  mysql-replica-01.route_rules.12:
    schema-pattern: sms_user
    table-pattern: oauth_client_details
    target-schema: sms_user
    target-table: oauth_client_details
  mysql-replica-01.route_rules.13:
    schema-pattern: sms_user
    table-pattern: permission
    target-schema: sms_user
    target-table: permission
  mysql-replica-01.route_rules.14:
    schema-pattern: sms_user
    table-pattern: role
    target-schema: sms_user
    target-table: role
  mysql-replica-01.route_rules.15:
    schema-pattern: sms_user
    table-pattern: role_data_relate
    target-schema: sms_user
    target-table: role_data_relate
  mysql-replica-01.route_rules.16:
    schema-pattern: sms_user
    table-pattern: role_permission
    target-schema: sms_user
    target-table: role_permission
  mysql-replica-01.route_rules.17:
    schema-pattern: sms_user
    table-pattern: test_hky
    target-schema: sms_user
    target-table: test_hky
  mysql-replica-01.route_rules.18:
    schema-pattern: sms_user
    table-pattern: user
    target-schema: sms_user
    target-table: user
  mysql-replica-01.route_rules.19:
    schema-pattern: sms_user
    table-pattern: user_role
    target-schema: sms_user
    target-table: user_role
filters:
  mysql-replica-01.filter.1:
    schema-pattern: sms_user
    table-pattern: application_manage
    events:
    - all
    sql-pattern: []
    action: Ignore
  mysql-replica-01.filter.2:
    schema-pattern: sms_user
    table-pattern: dic_city
    events:
    - all
    sql-pattern: []
    action: Ignore
  mysql-replica-01.filter.3:
    schema-pattern: sms_user
    table-pattern: dic_country
    events:
    - all
    sql-pattern: []
    action: Ignore
  mysql-replica-01.filter.4:
    schema-pattern: sms_user
    table-pattern: dic_device_model
    events:
    - all
    sql-pattern: []
    action: Ignore
  mysql-replica-01.filter.5:
    schema-pattern: sms_user
    table-pattern: dic_device_purpose
    events:
    - all
    sql-pattern: []
    action: Ignore
  mysql-replica-01.filter.6:
    schema-pattern: sms_user
    table-pattern: dic_device_type
    events:
    - all
    sql-pattern: []
    action: Ignore
  mysql-replica-01.filter.7:
    schema-pattern: sms_user
    table-pattern: dic_line
    events:
    - all
    sql-pattern: []
    action: Ignore
  mysql-replica-01.filter.8:
    schema-pattern: sms_user
    table-pattern: dic_organization
    events:
    - all
    sql-pattern: []
    action: Ignore
  mysql-replica-01.filter.9:
    schema-pattern: sms_user
    table-pattern: dic_province
    events:
    - all
    sql-pattern: []
    action: Ignore
  mysql-replica-01.filter.10:
    schema-pattern: sms_user
    table-pattern: dic_station
    events:
    - all
    sql-pattern: []
    action: Ignore
  mysql-replica-01.filter.11:
    schema-pattern: sms_user
    table-pattern: oauth_client_details
    events:
    - all
    sql-pattern: []
    action: Ignore
  mysql-replica-01.filter.12:
    schema-pattern: sms_user
    table-pattern: permission
    events:
    - all
    sql-pattern: []
    action: Ignore
  mysql-replica-01.filter.13:
    schema-pattern: sms_user
    table-pattern: role
    events:
    - all
    sql-pattern: []
    action: Ignore
  mysql-replica-01.filter.14:
    schema-pattern: sms_user
    table-pattern: role_data_relate
    events:
    - all
    sql-pattern: []
    action: Ignore
  mysql-replica-01.filter.15:
    schema-pattern: sms_user
    table-pattern: role_permission
    events:
    - all
    sql-pattern: []
    action: Ignore
  mysql-replica-01.filter.16:
    schema-pattern: sms_user
    table-pattern: test_hky
    events:
    - all
    sql-pattern: []
    action: Ignore
  mysql-replica-01.filter.17:
    schema-pattern: sms_user
    table-pattern: user
    events:
    - all
    sql-pattern: []
    action: Ignore
  mysql-replica-01.filter.18:
    schema-pattern: sms_user
    table-pattern: user_role
    events:
    - all
    sql-pattern: []
    action: Ignore
  mysql-replica-01.filter.19:
    schema-pattern: sms_user
    table-pattern: ""
    events:
    - all
    sql-pattern: []
    action: Ignore
black-white-list:
  mysql-replica-01.bw_list.1:
    do-tables:
    - db-name: sms_user
      tbl-name: application_manage
    - db-name: sms_user
      tbl-name: dic_city
    - db-name: sms_user
      tbl-name: dic_country
    - db-name: sms_user
      tbl-name: dic_device_model
    - db-name: sms_user
      tbl-name: dic_device_purpose
    - db-name: sms_user
      tbl-name: dic_device_type
    - db-name: sms_user
      tbl-name: dic_line
    - db-name: sms_user
      tbl-name: dic_organization
    - db-name: sms_user
      tbl-name: dic_province
    - db-name: sms_user
      tbl-name: dic_station
    - db-name: sms_user
      tbl-name: oauth_client_details
    - db-name: sms_user
      tbl-name: permission
    - db-name: sms_user
      tbl-name: role
    - db-name: sms_user
      tbl-name: role_data_relate
    - db-name: sms_user
      tbl-name: role_permission
    - db-name: sms_user
      tbl-name: test_hky
    - db-name: sms_user
      tbl-name: user
    - db-name: sms_user
      tbl-name: user_role
    do-dbs: []
    ignore-tables:
    - db-name: sms_connect
      tbl-name: dic_bdms_cphm_device_mapping
    - db-name: sms_connect
      tbl-name: dic_client_address
    - db-name: sms_connect
      tbl-name: dic_connect_info
    - db-name: sms_connect
      tbl-name: dic_connect_system_config
    - db-name: sms_connect
      tbl-name: dic_sendback
    - db-name: sms_connect
      tbl-name: dic_system_subdeploy
    - db-name: sms_connect
      tbl-name: switch_curve_index
    - db-name: sms_connect
      tbl-name: switch_curve_tct
    - db-name: sms_connect
      tbl-name: tct_deploy_receive
    - db-name: sms_connect
      tbl-name: tct_power_data
    - db-name: sms_connect
      tbl-name: tct_power_deploy_history
    - db-name: sms_connect
      tbl-name: tct_switch_data
    - db-name: sms_connect
      tbl-name: tct_switch_data_old
    - db-name: sms_connect
      tbl-name: tct_switch_deploy_history
    - db-name: sms_connect
      tbl-name: upgrade_detail
    - db-name: sms_connect
      tbl-name: upgrade_task
    - db-name: sms_connect
      tbl-name: upgrade_value
    - db-name: sms_cphm
      tbl-name: alarm_dealing
    - db-name: sms_cphm
      tbl-name: alarm_history
    - db-name: sms_cphm
      tbl-name: alarm_type
    - db-name: sms_cphm
      tbl-name: auth_group
    - db-name: sms_cphm
      tbl-name: auth_group_permissions
    - db-name: sms_cphm
      tbl-name: auth_permission
    - db-name: sms_cphm
      tbl-name: auth_user
    - db-name: sms_cphm
      tbl-name: auth_user_groups
    - db-name: sms_cphm
      tbl-name: auth_user_user_permissions
    - db-name: sms_cphm
      tbl-name: cec_feature
    - db-name: sms_cphm
      tbl-name: data_bdms_mms
    - db-name: sms_cphm
      tbl-name: device_health
    - db-name: sms_cphm
      tbl-name: dic_bdms_cphm_device_mapping
    - db-name: sms_cphm
      tbl-name: dic_city
    - db-name: sms_cphm
      tbl-name: dic_collect_device
    - db-name: sms_cphm
      tbl-name: dic_cphm_system_config
    - db-name: sms_cphm
      tbl-name: dic_device
    - db-name: sms_cphm
      tbl-name: dic_device_model
    - db-name: sms_cphm
      tbl-name: dic_device_purpose
    - db-name: sms_cphm
      tbl-name: dic_device_type
    - db-name: sms_cphm
      tbl-name: dic_health_param
    - db-name: sms_cphm
      tbl-name: dic_line
    - db-name: sms_cphm
      tbl-name: dic_station
    - db-name: sms_cphm
      tbl-name: django_admin_log
    - db-name: sms_cphm
      tbl-name: django_content_type
    - db-name: sms_cphm
      tbl-name: django_migrations
    - db-name: sms_cphm
      tbl-name: django_session
    - db-name: sms_cphm
      tbl-name: health_standard
    - db-name: sms_cphm
      tbl-name: kzkf_feature
    - db-name: sms_cphm
      tbl-name: link_device_collect_device
    - db-name: sms_cphm
      tbl-name: log
    - db-name: sms_cphm
      tbl-name: message
    - db-name: sms_cphm
      tbl-name: pc_feature
    - db-name: sms_cphm
      tbl-name: power_data_type
    - db-name: sms_cphm
      tbl-name: power_dyp_value
    - db-name: sms_cphm
      tbl-name: power_env_value
    - db-name: sms_cphm
      tbl-name: power_switching_value
    - db-name: sms_cphm
      tbl-name: power_ups_value
    - db-name: sms_cphm
      tbl-name: power_wdw_value
    - db-name: sms_cphm
      tbl-name: power_xdc_value
    - db-name: sms_cphm
      tbl-name: score_record
    - db-name: sms_cphm
      tbl-name: standard_curve
    - db-name: sms_cphm
      tbl-name: standard_curve_json
    - db-name: sms_cphm
      tbl-name: switch_curve_index
    - db-name: sms_cphm
      tbl-name: switch_curve_tct
    - db-name: sms_cphm
      tbl-name: switch_electrical_param
    - db-name: sms_cphm
      tbl-name: switch_move_record
    - db-name: sms_cphm
      tbl-name: switch_relay_status
    - db-name: sms_cphm
      tbl-name: switch_represent_voltage
    - db-name: sms_cphm
      tbl-name: switch_switchvalue
    - db-name: sms_cphm
      tbl-name: t_position
    - db-name: sms_cphm
      tbl-name: t_privilege_treeview
    - db-name: sms_cphm
      tbl-name: t_role
    - db-name: sms_cphm
      tbl-name: t_user
    - db-name: sms_cphm
      tbl-name: temp4$
    - db-name: sms_cphm
      tbl-name: test_hky
    - db-name: sms_cphm
      tbl-name: user_operation_log
    - db-name: sms_cphm
      tbl-name: vp_feature
    - db-name: sms_cphm
      tbl-name: vpangle_feature
    - db-name: sms_cphm
      tbl-name: yjyai_base_fault_set
    - db-name: sms_cphm
      tbl-name: yjyai_comp_pred
    - db-name: sms_cphm
      tbl-name: yjyai_current_3x_feature
    - db-name: sms_cphm
      tbl-name: yjyai_time_seq_feature
    ignore-dbs: []
mydumpers:
  mysql-replica-01.dump:
    mydumper-path: bin/mydumper
    threads: 4
    chunk-filesize: 64
    skip-tz-utc: true
    extra-args: -T sms_user.application_manage,sms_user.dic_city,sms_user.dic_country,sms_user.dic_device_model,sms_user.dic_device_purpose,sms_user.dic_device_type,sms_user.dic_line,sms_user.dic_organization,sms_user.dic_province,sms_user.dic_station,sms_user.oauth_client_details,sms_user.permission,sms_user.role,sms_user.role_data_relate,sms_user.role_permission,sms_user.test_hky,sms_user.user,sms_user.user_role

loaders:                             
  global:                            
    pool-size: 16                    
    dir: "./dumped_data"             

syncers:                             
  global:                            
    worker-count: 16                 
    batch: 100                       
    enable-ansi-quotes: true