TiFlash添加副本，同步到一半出现报错无法启动

levy · 2021 年4 月 25 日 07:22

为提高效率，请提供以下信息，问题描述清晰能够更快得到解决：

【TiDB 版本】
v5.0.0

【问题描述】
同步一张表，突然就挂了，状态为 Disconnected，无法连接，无法启动与关闭

2021.04.25 15:11:51.928179 [ 1 ] Application: The configuration “path” is deprecated. Check [storage] section for new style.
2021.04.25 15:11:59.903483 [ 50 ] DB::RawCppPtr DB::PreHandleSnapshot(DB::EngineStoreServerWrap*, DB::BaseBuffView, uint64_t, DB::SSTViewVec, uint64_t, uint64_t): Code: 49, e.displayText() = DB::Exception: Raw TiDB PK: 8000000004EA1B28, Prewrite ts: 422348755474841734 can not found in default cf for key: 7480000000000029FFF15F728000000004FFEA1B280000000000FAFA23840B69CFFFEA, e.what() = DB::Exception, Stack trace:

bin/tiflash/tiflash(StackTrace::StackTrace()+0x15) [0x367e645]
bin/tiflash/tiflash(DB::Exception::Exception(std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&, int)+0x25) [0x36751d5]
bin/tiflash/tiflash(DB::RegionData::readDataByWriteIt(std::_Rb_tree_const_iterator<std::pair<std::pair<DB::RawTiDBPK, unsigned long> const, std::tuple<std::shared_ptr<DB::StringObject const>, std::shared_ptr<DB::StringObject const>, DB::RecordKVFormat::InnerDecodedWriteCFValue> > > const&, bool) const+0x50c) [0x73495ec]
bin/tiflash/tiflash(DB::Region::readDataByWriteIt(std::_Rb_tree_const_iterator<std::pair<std::pair<DB::RawTiDBPK, unsigned long> const, std::tuple<std::shared_ptr<DB::StringObject const>, std::shared_ptr<DB::StringObject const>, DB::RecordKVFormat::InnerDecodedWriteCFValue> > > const&, bool) const+0x17) [0x7328487]
bin/tiflash/tiflash(DB::ReadRegionCommitCache(std::shared_ptrDB::Region const&, bool)+0x1c1) [0x731d4b1]
bin/tiflash/tiflash(DB::GenRegionPreDecodeBlockData(std::shared_ptrDB::Region const&, DB::Context&)+0x30) [0x731dd80]
bin/tiflash/tiflash(DB::KVStore::preHandleSnapshot(std::shared_ptrDB::Region, DB::SSTViewVec, DB::TMTContext&)+0x573) [0x76fc1f3]
bin/tiflash/tiflash(DB::PreHandleSnapshot(DB::EngineStoreServerWrap*, DB::BaseBuffView, unsigned long, DB::SSTViewVec, unsigned long, unsigned long)+0xcf) [0x732776f]
/data/tidb-deploy/tiflash-9000/bin/tiflash/libtiflash_proxy.so(+0x11e0a6c) [0x7f4b138faa6c]
/data/tidb-deploy/tiflash-9000/bin/tiflash/libtiflash_proxy.so(+0x11c8481) [0x7f4b138e2481]
/data/tidb-deploy/tiflash-9000/bin/tiflash/libtiflash_proxy.so(+0x9fa940) [0x7f4b13114940]
/data/tidb-deploy/tiflash-9000/bin/tiflash/libtiflash_proxy.so(+0x1ca5480) [0x7f4b143bf480]
/data/tidb-deploy/tiflash-9000/bin/tiflash/libtiflash_proxy.so(+0x1c98f72) [0x7f4b143b2f72]
/data/tidb-deploy/tiflash-9000/bin/tiflash/libtiflash_proxy.so(+0x1c9aa13) [0x7f4b143b4a13]
/data/tidb-deploy/tiflash-9000/bin/tiflash/libtiflash_proxy.so(+0x1480a4f) [0x7f4b13b9aa4f]
/lib64/libpthread.so.0(+0x7ea4) [0x7f4b11edfea4]
/lib64/libc.so.6(clone+0x6c) [0x7f4b119068dc]

2021.04.25 15:12:16.425455 [ 1 ] Application: The configuration “path” is deprecated. Check [storage] section for new style.
2021.04.25 15:12:24.785147 [ 51 ] DB::RawCppPtr DB::PreHandleSnapshot(DB::EngineStoreServerWrap*, DB::BaseBuffView, uint64_t, DB::SSTViewVec, uint64_t, uint64_t): Code: 49, e.displayText() = DB::Exception: Raw TiDB PK: 8000000004EA1B28, Prewrite ts: 422348755474841734 can not found in default cf for key: 7480000000000029FFF15F728000000004FFEA1B280000000000FAFA23840B69CFFFEA, e.what() = DB::Exception, Stack trace:

bin/tiflash/tiflash(StackTrace::StackTrace()+0x15) [0x367e645]
bin/tiflash/tiflash(DB::Exception::Exception(std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&, int)+0x25) [0x36751d5]
bin/tiflash/tiflash(DB::RegionData::readDataByWriteIt(std::_Rb_tree_const_iterator<std::pair<std::pair<DB::RawTiDBPK, unsigned long> const, std::tuple<std::shared_ptr<DB::StringObject const>, std::shared_ptr<DB::StringObject const>, DB::RecordKVFormat::InnerDecodedWriteCFValue> > > const&, bool) const+0x50c) [0x73495ec]
bin/tiflash/tiflash(DB::Region::readDataByWriteIt(std::_Rb_tree_const_iterator<std::pair<std::pair<DB::RawTiDBPK, unsigned long> const, std::tuple<std::shared_ptr<DB::StringObject const>, std::shared_ptr<DB::StringObject const>, DB::RecordKVFormat::InnerDecodedWriteCFValue> > > const&, bool) const+0x17) [0x7328487]
bin/tiflash/tiflash(DB::ReadRegionCommitCache(std::shared_ptrDB::Region const&, bool)+0x1c1) [0x731d4b1]
bin/tiflash/tiflash(DB::GenRegionPreDecodeBlockData(std::shared_ptrDB::Region const&, DB::Context&)+0x30) [0x731dd80]
bin/tiflash/tiflash(DB::KVStore::preHandleSnapshot(std::shared_ptrDB::Region, DB::SSTViewVec, DB::TMTContext&)+0x573) [0x76fc1f3]
bin/tiflash/tiflash(DB::PreHandleSnapshot(DB::EngineStoreServerWrap*, DB::BaseBuffView, unsigned long, DB::SSTViewVec, unsigned long, unsigned long)+0xcf) [0x732776f]
/data/tidb-deploy/tiflash-9000/bin/tiflash/libtiflash_proxy.so(+0x11e0a6c) [0x7ff02e037a6c]
/data/tidb-deploy/tiflash-9000/bin/tiflash/libtiflash_proxy.so(+0x11c8481) [0x7ff02e01f481]
/data/tidb-deploy/tiflash-9000/bin/tiflash/libtiflash_proxy.so(+0x9fa940) [0x7ff02d851940]
/data/tidb-deploy/tiflash-9000/bin/tiflash/libtiflash_proxy.so(+0x1ca5480) [0x7ff02eafc480]
/data/tidb-deploy/tiflash-9000/bin/tiflash/libtiflash_proxy.so(+0x1c98f72) [0x7ff02eaeff72]
/data/tidb-deploy/tiflash-9000/bin/tiflash/libtiflash_proxy.so(+0x1c9aa13) [0x7ff02eaf1a13]
/data/tidb-deploy/tiflash-9000/bin/tiflash/libtiflash_proxy.so(+0x1480a4f) [0x7ff02e2d7a4f]
/lib64/libpthread.so.0(+0x7ea4) [0x7ff02c61cea4]
/lib64/libc.so.6(clone+0x6c) [0x7ff02c0438dc]

2021.04.25 15:12:41.422430 [ 1 ] Application: The configuration “path” is deprecated. Check [storage] section for new style.
2021.04.25 15:12:49.564486 [ 52 ] DB::RawCppPtr DB::PreHandleSnapshot(DB::EngineStoreServerWrap*, DB::BaseBuffView, uint64_t, DB::SSTViewVec, uint64_t, uint64_t): Code: 49, e.displayText() = DB::Exception: Raw TiDB PK: 8000000004EA1B28, Prewrite ts: 422348755474841734 can not found in default cf for key: 7480000000000029FFF15F728000000004FFEA1B280000000000FAFA23840B69CFFFEA, e.what() = DB::Exception, Stack trace:

bin/tiflash/tiflash(StackTrace::StackTrace()+0x15) [0x367e645]
bin/tiflash/tiflash(DB::Exception::Exception(std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&, int)+0x25) [0x36751d5]

若提问为性能优化、故障排查类问题，请下载脚本运行。终端输出的打印结果，请务必全选并复制粘贴上传。

这道题我不会 · 2021 年4 月 25 日 07:38

麻烦先提供下 tiflash 节点相关的完整日志和集群的拓扑结构。

levy · 2021 年4 月 25 日 07:42

删除副本后，状态为down，无法启动

tiflash_error.log (223.5 KB)

这道题我不会 · 2021 年4 月 25 日 07:48

麻烦将 tiflash.log 和 tiflash_cluster_manager.log 这两个日志也提供下，多谢。

levy · 2021 年4 月 25 日 08:02

tiflash日志有点大，我取末尾100w行

tiflash_error.log (223.5 KB)
tiflash_cluster_manager.log (429.5 KB)
tiflash.rar (2.1 MB)

这道题我不会 · 2021 年4 月 25 日 08:14

从日志中看确实在同步 table id 为 10737 的表时 tiflash 突然中断，你这边方便将导出下 tiflash 监控面板 TiFlash-Summary 、 TiFlash-Proxy-Summary 和 TiFlash-Proxy-Details 的数据吗？我们这边再分析下原因，导出方法参考下：
https://metricstool.pingcap.com/#backup-with-dev-tools

levy · 2021 年4 月 25 日 08:19

tidb-test-TiFlash-Proxy-Details_2021-04-25T08_18_24.335Z.json (651.2 KB)
tidb-test-TiFlash-Proxy-Summary_2021-04-25T08_17_47.539Z.json (369.3 KB)
tidb-test-TiFlash-Summary_2021-04-25T08_16_57.644Z.json (210.7 KB)

这道题我不会 · 2021 年4 月 25 日 08:35

监控页面没有展开，导致导出的数据不全，麻烦将监控面板里页面展开后重新导出一下。

levy · 2021 年4 月 25 日 08:46

tidb-test-TiFlash-Proxy-Summary_2021-04-25T08_41_13.732Z.json (524.9 KB) tidb-test-TiFlash-Proxy-Details_2021-04-25T08_40_05.871Z.json (4.5 MB) tidb-test-TiFlash-Summary_2021-04-25T08_44_17.486Z.json (2.7 MB)

这道题我不会 · 2021 年4 月 25 日 09:03

请问下在 tiflash 新增同步表（table id 为 10737 ）时，有无对集群做过其他的操作，比如通过 tidb-lightning 往集群中导入数据。

levy · 2021 年4 月 25 日 09:22

otter mysql -->tidb ，业务查询，没有其他操作

这道题我不会 · 2021 年4 月 25 日 09:29

请问下 MySQL 和 Tidb 之间是通过什么方式进行数据同步的？另外这个 TiDB 是对业务仅提供只读查询吗？

levy · 2021 年4 月 25 日 09:33

通过otter，tidb对外大部分时候是只读。另外那张表有虚拟列，这有影响吗？

这道题我不会 · 2021 年4 月 25 日 09:53

1.虚拟列这个功能是目前还是实验特性，你可以把这张表表结构提供下，我看下能否复现你的问题；
2.目前 tiflash 已经宕掉且只有一个节点，可以考虑先下线该节点再扩容上，然后重新同步下表数据。

levy · 2021 年4 月 26 日 02:15

CREATE TABLE inventory (
id char(32) NOT NULL DEFAULT ‘’ COMMENT ‘主键(UUID)’,
sku_no char(32) NOT NULL DEFAULT ‘’ COMMENT ‘sku编码’,
store_no char(18) NOT NULL DEFAULT ‘’ COMMENT ‘机构编码’,
store_name varchar(100) DEFAULT ‘’ COMMENT ‘机构名称’,
order_unit_no char(18) NOT NULL DEFAULT ‘’ COMMENT ‘订货单位’,
order_unit_name varchar(100) DEFAULT ‘’ COMMENT ‘订货单位名称’,
item_no char(18) NOT NULL DEFAULT ‘’ COMMENT ‘商品编码’,
item_code varchar(30) DEFAULT ‘’ COMMENT ‘商品编码’,
item_name varchar(200) DEFAULT ‘’ COMMENT ‘商品名称’,
color_no char(18) DEFAULT ‘’ COMMENT ‘颜色编码’,
color_name varchar(50) DEFAULT ‘’ COMMENT ‘颜色名称’,
brand_no char(18) DEFAULT ‘’ COMMENT ‘品牌编码’,
brand_name varchar(200) DEFAULT ‘’ COMMENT ‘品牌名称’,
category_no char(18) DEFAULT ‘’ COMMENT ‘类别编码’,
size_no varchar(10) DEFAULT ‘’ COMMENT ‘尺寸编码’,
size_name varchar(10) DEFAULT ‘’ COMMENT ‘尺码名称’,
size_kind char(2) NOT NULL DEFAULT ‘’ COMMENT ‘尺寸分类’,
cost decimal(12,2) DEFAULT ‘0.00’ COMMENT ‘含税单价’,
balance_qty int(11) DEFAULT ‘0’ COMMENT ‘实物库存（正品库存量）’,
factory_in_qty int(11) DEFAULT ‘0’ COMMENT ‘采购在途’,
factory_in_diff_qty int(11) DEFAULT ‘0’ COMMENT ‘采购在途差异量’,
transit_out_qty int(11) DEFAULT ‘0’ COMMENT ‘区内出库在途数量’,
out_diff_qty int(11) DEFAULT ‘0’ COMMENT ‘区内出库差异量’,
transit_in_qty int(11) DEFAULT ‘0’ COMMENT ‘区内入库在途数量’,
in_diff_qty int(11) DEFAULT ‘0’ COMMENT ‘区内入库差异在途’,
transit_out_account_qty int(11) DEFAULT ‘0’ COMMENT ‘区外出库在途数量’,
out_diff_account_qty int(11) DEFAULT ‘0’ COMMENT ‘区外出库差异量’,
transit_in_account_qty int(11) DEFAULT ‘0’ COMMENT ‘区外入库在途数量’,
in_diff_account_qty int(11) DEFAULT ‘0’ COMMENT ‘区外入库差异量’,
lock_qty int(11) DEFAULT ‘0’ COMMENT ‘冻结数量’,
occupied_qty int(11) DEFAULT ‘0’ COMMENT ‘预占数量’,
backup_qty int(11) DEFAULT ‘0’ COMMENT ‘备货数量’,
work_qty int(11) DEFAULT ‘0’ COMMENT ‘工服库存’,
unable_schedule_qty int(11) DEFAULT ‘0’ COMMENT ‘不可调度库存’,
guest_bad_qty int(11) DEFAULT ‘0’ COMMENT ‘客残数量’,
pre_destroy_qty int(11) DEFAULT ‘0’ COMMENT ‘待销毁库存变化量’,
original_bad_qty int(11) DEFAULT ‘0’ COMMENT ‘原残数量’,
bad_transit_qty int(11) DEFAULT ‘0’ COMMENT ‘次品在途量’,
bad_diff_qty int(11) DEFAULT ‘0’ COMMENT ‘次品在途差异’,
bad_transit_account_qty int(11) DEFAULT ‘0’ COMMENT ‘次品跨区入在途’,
bad_diff_account_qty int(11) DEFAULT ‘0’ COMMENT ‘次品跨区入在途差异’,
return_qty int(11) DEFAULT ‘0’ COMMENT ‘退厂在途数量（暂时不用）’,
borrow_qty int(11) DEFAULT ‘0’ COMMENT ‘借用数量（暂时不用）’,
update_time datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT ‘修改时间’,
sharding_flag char(20) DEFAULT ‘’ COMMENT ‘分库字段：本部+大区’,
stock_in_date datetime DEFAULT CURRENT_TIMESTAMP COMMENT ‘收货日期’,
sale_date datetime DEFAULT CURRENT_TIMESTAMP COMMENT ‘销售日期’,
lock_stock_qty int(11) DEFAULT ‘0’ COMMENT ‘锁库库存’,
un_share_qty int(16) DEFAULT ‘0’ COMMENT ‘不可共享库存’,
online_occupied_qty int(11) DEFAULT ‘0’ COMMENT ‘线上预占’,
online_available_qty int(11) GENERATED ALWAYS AS ((((((balance_qty - greatest(ifnull(lock_qty, 0), 0)) - greatest(ifnull(occupied_qty, 0), 0)) - greatest(ifnull(backup_qty, 0), 0)) - greatest(ifnull(un_share_qty, 0), 0)) - greatest(ifnull(work_qty, 0), 0)) + greatest(ifnull(online_occupied_qty, 0), 0)) VIRTUAL,
available_qty2 int(11) GENERATED ALWAYS AS (((((greatest(balance_qty, 0) - greatest(ifnull(lock_qty, 0), 0)) - greatest(ifnull(occupied_qty, 0), 0)) - greatest(ifnull(backup_qty, 0), 0)) - greatest(ifnull(un_share_qty, 0), 0)) - greatest(ifnull(work_qty, 0), 0)) VIRTUAL,
PRIMARY KEY (id) /*T![clustered_index] NONCLUSTERED */,
UNIQUE KEY idx_UNIQUE (order_unit_no,store_no,sku_no),
KEY idx_store_no (sharding_flag,item_no,order_unit_no,store_no,size_no),
KEY idx_store_item_no (store_no,order_unit_no,item_no,sku_no),
KEY idx_sku (item_no,size_no),
KEY idx_sku_no (sku_no),
KEY idx_online_available_qty (online_available_qty),
KEY idx_available_qty2 (available_qty2),
KEY idx_item_code (item_code),
KEY idx_stock_in_date (stock_in_date)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT=‘商品库存总账’

这道题我不会 · 2021 年4 月 26 日 05:56

请问下目前 tiflash 节点有执行缩容和扩容吗？这个问题初步怀疑是 v5.0.0 中一个已知问题，如果方便的话可以考虑先升级到 v5.0.1

levy · 2021 年4 月 26 日 08:36

谢谢，目前暂不同步此帐表。24刚出，我等一段时间再升级把

这道题我不会 · 2021 年4 月 26 日 11:38

好的，如果有其他问题可重新开贴提问。