今天凌晨又有一个TiFlash的节点挂掉，然后一直在尝试启动失败，这是新出现的报错的log：
[2022/06/24 02:32:28.445 +08:00] [ERROR] [DAGDriver.cpp:197] ["DAGDriver:DB Exception: Cannot read from file /data01/deploy/data/data/t_32925/log/page_1_0/page., errno: 29, strerror: Illegal seek
0x1d272d3 StackTrace::StackTrace() [tiflash+30569171]
dbms/src/Common/StackTrace.cpp:23
0x1d248d6 DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int) [tiflash+30558422]
dbms/src/Common/Exception.h:41
0x1d302ba DB::throwFromErrno(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, int) [tiflash+30606010]
dbms/src/Common/Exception.cpp:59
0x786632b void DB::PageUtil::readFile<std::__1::shared_ptr<DB::RandomAccessFile> >(std::__1::shared_ptr<DB::RandomAccessFile>&, long, char const*, unsigned long, std::__1::shared_ptr<DB::ReadLimiter> const&, bool) [tiflash+126247723]
dbms/src/Storages/Page/PageUtil.h:270
0x79f7e8c DB::PS::V2::PageFile::Reader::read(std::__1::vector<DB::PS::V2::PageFile::Reader::FieldReadInfo, std::__1::allocator<DB::PS::V2::PageFile::Reader::FieldReadInfo> >&, std::__1::shared_ptr<DB::ReadLimiter> const&) [tiflash+127893132]
dbms/src/Storages/Page/V2/PageFile.cpp:1039
0x7a0c089 DB::PS::V2::PageStorage::readImpl(unsigned long, std::__1::vector<std::__1::pair<unsigned long, std::__1::vector<unsigned long, std::__1::allocator<unsigned long> > >, std::__1::allocator<std::__1::pair<unsigned long, std::__1::vector<unsigned long, std::__1::allocator<unsigned long> > > > > const&, std::__1::shared_ptr<DB::ReadLimiter> const&, std::__1::shared_ptr<DB::PageStorageSnapshot>, bool) [tiflash+127975561]
dbms/src/Storages/Page/V2/PageStorage.cpp:783
0x7a8fbf0 DB::PageReaderImplNormal::read(std::__1::vector<std::__1::pair<unsigned long, std::__1::vector<unsigned long, std::__1::allocator<unsigned long> > >, std::__1::allocator<std::__1::pair<unsigned long, std::__1::vector<unsigned long, std::__1::allocator<unsigned long> > > > > const&) const [tiflash+128515056]
dbms/src/Storages/Page/PageStorage.cpp:113
0x7a8d3f2 DB::PageReader::read(std::__1::vector<std::__1::pair<unsigned long, std::__1::vector<unsigned long, std::__1::allocator<unsigned long> > >, std::__1::allocator<std::__1::pair<unsigned long, std::__1::vector<unsigned long, std::__1::allocator<unsigned long> > > > > const&) const [tiflash+128504818]
dbms/src/Storages/Page/PageStorage.cpp:415
0x7898948 DB::DM::ColumnFileTiny::readFromDisk(DB::PageReader const&, std::__1::vector<DB::DM::ColumnDefine, std::__1::allocator<DB::DM::ColumnDefine> > const&, unsigned long, unsigned long) const [tiflash+126454088]
dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTiny.cpp:79
0x7899124 DB::DM::ColumnFileTiny::fillColumns(DB::PageReader const&, std::__1::vector<DB::DM::ColumnDefine, std::__1::allocator<DB::DM::ColumnDefine> > const&, unsigned long, std::__1::vector<COWPtr<DB::IColumn>::immutable_ptr<DB::IColumn>, std::__1::allocator<COWPtr<DB::IColumn>::immutable_ptr<DB::IColumn> > >&) const [tiflash+126456100]
dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTiny.cpp:115
0x789a3f6 DB::DM::ColumnFileTinyReader::readRows(std::__1::vector<COWPtr<DB::IColumn>::mutable_ptr<DB::IColumn>, std::__1::allocator<COWPtr<DB::IColumn>::mutable_ptr<DB::IColumn> > >&, unsigned long, unsigned long, DB::DM::RowKeyRange const*) [tiflash+126460918]
dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTiny.cpp:237
0x788fa13 DB::DM::ColumnFileSetReader::readRows(std::__1::vector<COWPtr<DB::IColumn>::mutable_ptr<DB::IColumn>, std::__1::allocator<COWPtr<DB::IColumn>::mutable_ptr<DB::IColumn> > >&, unsigned long, unsigned long, DB::DM::RowKeyRange const*) [tiflash+126417427]
dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileSetReader.cpp:160
0x78c54ab DB::DM::DeltaValueReader::readRows(std::__1::vector<COWPtr<DB::IColumn>::mutable_ptr<DB::IColumn>, std::__1::allocator<COWPtr<DB::IColumn>::mutable_ptr<DB::IColumn> > >&, unsigned long, unsigned long, DB::DM::RowKeyRange const*) [tiflash+126637227]
dbms/src/Storages/DeltaMerge/Delta/Snapshot.cpp:117
0x77ebc05 void DB::DM::DeltaMergeBlockInputStream<DB::DM::DeltaValueReader, DB::DM::DTCompactedEntries<55ul, 20ul, 3ul>::Iterator, false>::next<false, false>(std::__1::vector<COWPtr<DB::IColumn>::mutable_ptr<DB::IColumn>, std::__1::allocator<COWPtr<DB::IColumn>::mutable_ptr<DB::IColumn> > >&, unsigned long&) [tiflash+125746181]
dbms/src/Storages/DeltaMerge/DeltaMerge.h:0
0x77ea997 DB::DM::DeltaMergeBlockInputStream<DB::DM::DeltaValueReader, DB::DM::DTCompactedEntries<55ul, 20ul, 3ul>::Iterator, false>::doRead() [tiflash+125741463]
dbms/src/Storages/DeltaMerge/DeltaMerge.h:242
0x77ea55c DB::DM::DeltaMergeBlockInputStream<DB::DM::DeltaValueReader, DB::DM::DTCompactedEntries<55ul, 20ul, 3ul>::Iterator, false>::read() [tiflash+125740380]
dbms/src/Storages/DeltaMerge/DeltaMerge.h:174
0x77ec9b9 DB::DM::DMRowKeyFilterBlockInputStream<true>::read() [tiflash+125749689]
dbms/src/Storages/DeltaMerge/RowKeyFilter.h:178
0x77e17d5 DB::DM::readNextBlock(std::__1::shared_ptr<DB::IBlockInputStream> const&) [tiflash+125704149]
dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h:253
0x7804405 DB::DM::DMVersionFilterBlockInputStream<0>::initNextBlock() [tiflash+125846533]
dbms/src/Storages/DeltaMerge/DMVersionFilterBlockInputStream.h:133
0x7803505 DB::DM::DMVersionFilterBlockInputStream<0>::read(DB::PODArray<unsigned char, 4096ul, Allocator<false>, 15ul, 16ul>*&, bool) [tiflash+125842693]
dbms/src/Storages/DeltaMerge/DMVersionFilterBlockInputStream.cpp:53
0x77b06d3 DB::DM::DMSegmentThreadInputStream::readImpl(DB::PODArray<unsigned char, 4096ul, Allocator<false>, 15ul, 16ul>*&, bool) [tiflash+125503187]
dbms/src/Storages/DeltaMerge/DMSegmentThreadInputStream.h:124
0x77b05c8 DB::DM::DMSegmentThreadInputStream::readImpl() [tiflash+125502920]
dbms/src/Storages/DeltaMerge/DMSegmentThreadInputStream.h:85
0x6688b8a DB::IProfilingBlockInputStream::read(DB::PODArray<unsigned char, 4096ul, Allocator<false>, 15ul, 16ul>*&, bool) [tiflash+107514762]
dbms/src/DataStreams/IProfilingBlockInputStream.cpp:75
0x66888c4 DB::IProfilingBlockInputStream::read() [tiflash+107514052]
dbms/src/DataStreams/IProfilingBlockInputStream.cpp:43
0x739e49b DB::ExpressionBlockInputStream::readImpl() [tiflash+121234587]
dbms/src/DataStreams/ExpressionBlockInputStream.cpp:50
0x1dc8c32 DB::IProfilingBlockInputStream::readImpl(DB::PODArray<unsigned char, 4096ul, Allocator<false>, 15ul, 16ul>*&, bool) [tiflash+31231026]
dbms/src/DataStreams/IProfilingBlockInputStream.h:232
0x6688a0e DB::IProfilingBlockInputStream::read(DB::PODArray<unsigned char, 4096ul, Allocator<false>, 15ul, 16ul>*&, bool) [tiflash+107514382]
dbms/src/DataStreams/IProfilingBlockInputStream.cpp:73
0x73a0ef0 DB::FilterBlockInputStream::readImpl() [tiflash+121245424]
dbms/src/DataStreams/FilterBlockInputStream.cpp:91
0x6688b8a DB::IProfilingBlockInputStream::read(DB::PODArray<unsigned char, 4096ul, Allocator<false>, 15ul, 16ul>*&, bool) [tiflash+107514762]
dbms/src/DataStreams/IProfilingBlockInputStream.cpp:75
0x66888c4 DB::IProfilingBlockInputStream::read() [tiflash+107514052]
dbms/src/DataStreams/IProfilingBlockInputStream.cpp:43
0x739e49b DB::ExpressionBlockInputStream::readImpl() [tiflash+121234587]
dbms/src/DataStreams/ExpressionBlockInputStream.cpp:50
0x6688b8a DB::IProfilingBlockInputStream::read(DB::PODArray<unsigned char, 4096ul, Allocator<false>, 15ul, 16ul>*&, bool) [tiflash+107514762]
dbms/src/DataStreams/IProfilingBlockInputStream.cpp:75"] [thread_id=47105]
[2022/06/24 05:46:25.680 +08:00] [ERROR] [Exception.cpp:85] ["DB::DM::DeltaMergeStore::DeltaMergeStore(DB::Context &, bool, const DB::String &, const DB::String &, DB::TableID, const DB::DM::ColumnDefines &, const DB::DM::ColumnDefine &, bool, size_t, const DB::DM::DeltaMergeStore::Settings &):Code: 49, e.displayText() = DB::Exception: PageFile binary version not match {MergingReader of PageFile_7_0, type: Formal, sequence no: 0, meta offset: 25098, data offset: 9991564} [unknown_version=0] [file=/data01/deploy/data/data/t_15303/meta/page_7_0/meta], e.what() = DB::Exception, Stack trace:
0x1d272d3 StackTrace::StackTrace() [tiflash+30569171]
dbms/src/Common/StackTrace.cpp:23
0x1d248d6 DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int) [tiflash+30558422]
dbms/src/Common/Exception.h:41
0x79f53a3 DB::PS::V2::PageFile::MetaMergingReader::moveNext(unsigned int*) [tiflash+127882147]
dbms/src/Storages/Page/V2/PageFile.cpp:585
0x7a06501 DB::PS::V2::PageStorage::restore() [tiflash+127952129]
dbms/src/Storages/Page/V2/PageStorage.cpp:289
0x781ec0f DB::DM::StoragePool::restore() [tiflash+125955087]
dbms/src/Storages/DeltaMerge/StoragePool.cpp:379
0x7788ca1 DB::DM::DeltaMergeStore::DeltaMergeStore(DB::Context&, bool, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, long, std::__1::vector<DB::DM::ColumnDefine, std::__1::allocator<DB::DM::ColumnDefine> > const&, DB::DM::ColumnDefine const&, bool, unsigned long, DB::DM::DeltaMergeStore::Settings const&) [tiflash+125340833]
dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp:243
0x771283d DB::StorageDeltaMerge::getAndMaybeInitStore() [tiflash+124856381]
dbms/src/Storages/StorageDeltaMerge.cpp:1545
0x77187fe DB::StorageDeltaMerge::getSchemaSnapshotAndBlockForDecoding(DB::TableDoubleLockHolder<false> const&, bool) [tiflash+124880894]
dbms/src/Storages/StorageDeltaMerge.cpp:910
0x7916b6d DB::writeRegionDataToStorage(DB::Context&, DB::RegionPtrWithBlock const&, std::__1::vector<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject<false> const> >, std::__1::allocator<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject<false> const> > > >&, Poco::Logger*)::$_0::operator()(bool) const [tiflash+126970733]
dbms/src/Storages/Transaction/PartitionStreams.cpp:123
0x7912f32 DB::writeRegionDataToStorage(DB::Context&, DB::RegionPtrWithBlock const&, std::__1::vector<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject<false> const> >, std::__1::allocator<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject<false> const> > > >&, Poco::Logger*) [tiflash+126955314]
dbms/src/Storages/Transaction/PartitionStreams.cpp:169
0x7912c9c DB::RegionTable::writeBlockByRegion(DB::Context&, DB::RegionPtrWithBlock const&, std::__1::vector<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject<false> const> >, std::__1::allocator<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject<false> const> > > >&, Poco::Logger*, bool) [tiflash+126954652]
dbms/src/Storages/Transaction/PartitionStreams.cpp:348
0x79351ff DB::Region::handleWriteRaftCmd(DB::WriteCmdsView const&, unsigned long, unsigned long, DB::TMTContext&) [tiflash+127095295]
dbms/src/Storages/Transaction/Region.cpp:712
0x7901910 DB::KVStore::handleWriteRaftCmd(DB::WriteCmdsView const&, unsigned long, unsigned long, unsigned long, DB::TMTContext&) [tiflash+126884112]
dbms/src/Storages/Transaction/KVStore.cpp:285
0x791b822 HandleWriteRaftCmd [tiflash+126990370]
dbms/src/Storages/Transaction/ProxyFFI.cpp:92
0x7f82f6f0a6e0 raftstore::store::fsm::apply::ApplyDelegate$LT$EK$GT$::process_raft_cmd::h4c8bba241d5ee4d4 [libtiflash_proxy.so+28567264]
0x7f82f6f07d6c raftstore::store::fsm::apply::ApplyDelegate$LT$EK$GT$::handle_raft_committed_entries::h478c4bad834ddec7 [libtiflash_proxy.so+28556652]
0x7f82f6f27f91 raftstore::store::fsm::apply::ApplyFsm$LT$EK$GT$::handle_apply::he4c0b1137ae7cad9 [libtiflash_proxy.so+28688273]
0x7f82f6f2abfe raftstore::store::fsm::apply::ApplyFsm$LT$EK$GT$::handle_tasks::h5b138e5889c69e66 [libtiflash_proxy.so+28699646]
0x7f82f6f2d510 _$LT$raftstore..store..fsm..apply..ApplyPoller$LT$EK$GT$$u20$as$u20$batch_system..batch..PollHandler$LT$raftstore..store..fsm..apply..ApplyFsm$LT$EK$GT$$C$raftstore..store..fsm..apply..ControlFsm$GT$$GT$::handle_normal::he20c24833a37424f [libtiflash_proxy.so+28710160]
0x7f82f6ba0609 batch_system::batch::Poller$LT$N$C$C$C$Handler$GT$::poll::hec57674a22fdcc30 [libtiflash_proxy.so+24987145]
0x7f82f7249cd8 std::sys_common::backtrace::__rust_begin_short_backtrace::h189c0b2868fa537c [libtiflash_proxy.so+31972568]
0x7f82f6a81ec1 core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h7c53021c905641b6 [libtiflash_proxy.so+23813825]
0x7f82f745ffca std::sys::unix::thread::Thread::new::thread_start::hd39c5f08bdcda277 [libtiflash_proxy.so+34160586]
0x7f82f4aa9ea5 start_thread [libpthread.so.0+32421]
0x7f82f45bc9fd __clone [libc.so.6+1042941]"] [thread_id=92]
[2022/06/24 05:46:59.759 +08:00] [ERROR] [Exception.cpp:85] ["DB::DM::DeltaMergeStore::DeltaMergeStore(DB::Context &, bool, const DB::String &, const DB::String &, DB::TableID, const DB::DM::ColumnDefines &, const DB::DM::ColumnDefine &, bool, size_t, const DB::DM::DeltaMergeStore::Settings &):Code: 40, e.displayText() = DB::Exception: Page [9] checksum not match, broken file: /data01/deploy/data/data/t_17615/meta/page_26_0/page, expected: 83cb09c59276f157, but: c38df517068eea44, e.what() = DB::Exception, Stack trace:
0x1d272d3 StackTrace::StackTrace() [tiflash+30569171]
dbms/src/Common/StackTrace.cpp:23
0x1d248d6 DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int) [tiflash+30558422]
dbms/src/Common/Exception.h:41
0x79f725a DB::PS::V2::PageFile::Reader::read(std::__1::vector<std::__1::pair<unsigned long, DB::PageEntry>, std::__1::allocator<std::__1::pair<unsigned long, DB::PageEntry> > >&, std::__1::shared_ptr<DB::ReadLimiter> const&, bool) [tiflash+127890010]
dbms/src/Storages/Page/V2/PageFile.cpp:916
0x7a0a78b DB::PS::V2::PageStorage::readImpl(unsigned long, unsigned long, std::__1::shared_ptr<DB::ReadLimiter> const&, std::__1::shared_ptr<DB::PageStorageSnapshot>, bool) [tiflash+127969163]
dbms/src/Storages/Page/V2/PageStorage.cpp:644
0x7a8f8fd DB::PageReaderImplNormal::read(unsigned long) const [tiflash+128514301]
dbms/src/Storages/Page/PageStorage.cpp:97
0x7a8d392 DB::PageReader::read(unsigned long) const [tiflash+128504722]
dbms/src/Storages/Page/PageStorage.cpp:400
0x78b05f9 DB::DM::ColumnFilePersistedSet::restore(DB::DM::DMContext&, DB::DM::RowKeyRange const&, unsigned long) [tiflash+126551545]
dbms/src/Storages/DeltaMerge/Delta/ColumnFilePersistedSet.cpp:110
0x78aa43a DB::DM::DeltaValueSpace::restore(DB::DM::DMContext&, DB::DM::RowKeyRange const&, unsigned long) [tiflash+126526522]
dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.cpp:59
0x77c34c0 DB::DM::Segment::restoreSegment(DB::DM::DMContext&, unsigned long) [tiflash+125580480]
dbms/src/Storages/DeltaMerge/Segment.cpp:274
0x7788d0f DB::DM::DeltaMergeStore::DeltaMergeStore(DB::Context&, bool, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, long, std::__1::vector<DB::DM::ColumnDefine, std::__1::allocator<DB::DM::ColumnDefine> > const&, DB::DM::ColumnDefine const&, bool, unsigned long, DB::DM::DeltaMergeStore::Settings const&) [tiflash+125340943]
dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp:270
0x771283d DB::StorageDeltaMerge::getAndMaybeInitStore() [tiflash+124856381]
dbms/src/Storages/StorageDeltaMerge.cpp:1545
0x77187fe DB::StorageDeltaMerge::getSchemaSnapshotAndBlockForDecoding(DB::TableDoubleLockHolder<false> const&, bool) [tiflash+124880894]
dbms/src/Storages/StorageDeltaMerge.cpp:910
0x7916b6d DB::writeRegionDataToStorage(DB::Context&, DB::RegionPtrWithBlock const&, std::__1::vector<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject<false> const> >, std::__1::allocator<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject<false> const> > > >&, Poco::Logger*)::$_0::operator()(bool) const [tiflash+126970733]
dbms/src/Storages/Transaction/PartitionStreams.cpp:123
0x7912f32 DB::writeRegionDataToStorage(DB::Context&, DB::RegionPtrWithBlock const&, std::__1::vector<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject<false> const> >, std::__1::allocator<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject<false> const> > > >&, Poco::Logger*) [tiflash+126955314]
dbms/src/Storages/Transaction/PartitionStreams.cpp:169
0x7912c9c DB::RegionTable::writeBlockByRegion(DB::Context&, DB::RegionPtrWithBlock const&, std::__1::vector<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject<false> const> >, std::__1::allocator<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject<false> const> > > >&, Poco::Logger*, bool) [tiflash+126954652]
dbms/src/Storages/Transaction/PartitionStreams.cpp:348
0x79351ff DB::Region::handleWriteRaftCmd(DB::WriteCmdsView const&, unsigned long, unsigned long, DB::TMTContext&) [tiflash+127095295]
dbms/src/Storages/Transaction/Region.cpp:712
0x7901910 DB::KVStore::handleWriteRaftCmd(DB::WriteCmdsView const&, unsigned long, unsigned long, unsigned long, DB::TMTContext&) [tiflash+126884112]
dbms/src/Storages/Transaction/KVStore.cpp:285
0x791b822 HandleWriteRaftCmd [tiflash+126990370]
dbms/src/Storages/Transaction/ProxyFFI.cpp:92
0x7f4277f366e0 raftstore::store::fsm::apply::ApplyDelegate$LT$EK$GT$::process_raft_cmd::h4c8bba241d5ee4d4 [libtiflash_proxy.so+28567264]
0x7f4277f33d6c raftstore::store::fsm::apply::ApplyDelegate$LT$EK$GT$::handle_raft_committed_entries::h478c4bad834ddec7 [libtiflash_proxy.so+28556652]
0x7f4277f53f91 raftstore::store::fsm::apply::ApplyFsm$LT$EK$GT$::handle_apply::he4c0b1137ae7cad9 [libtiflash_proxy.so+28688273]
0x7f4277f56bfe raftstore::store::fsm::apply::ApplyFsm$LT$EK$GT$::handle_tasks::h5b138e5889c69e66 [libtiflash_proxy.so+28699646]
0x7f4277f59510 _$LT$raftstore..store..fsm..apply..ApplyPoller$LT$EK$GT$$u20$as$u20$batch_system..batch..PollHandler$LT$raftstore..store..fsm..apply..ApplyFsm$LT$EK$GT$$C$raftstore..store..fsm..apply..ControlFsm$GT$$GT$::handle_normal::he20c24833a37424f [libtiflash_proxy.so+28710160]
0x7f4277bcc609 batch_system::batch::Poller$LT$N$C$C$C$Handler$GT$::poll::hec57674a22fdcc30 [libtiflash_proxy.so+24987145]
0x7f4278275cd8 std::sys_common::backtrace::__rust_begin_short_backtrace::h189c0b2868fa537c [libtiflash_proxy.so+31972568]
0x7f4277aadec1 core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h7c53021c905641b6 [libtiflash_proxy.so+23813825]
0x7f427848bfca std::sys::unix::thread::Thread::new::thread_start::hd39c5f08bdcda277 [libtiflash_proxy.so+34160586]
0x7f4275ad5ea5 start_thread [libpthread.so.0+32421]
0x7f42755e89fd __clone [libc.so.6+1042941]"] [thread_id=187]
[2022/06/24 05:51:41.033 +08:00] [ERROR] [Exception.cpp:85] ["Application:Storage inited fail, [table_id=15259]: Code: 49, e.displayText() = DB::Exception: PageFile binary version not match {MergingReader of PageFile_29_0, type: Formal, sequence no: 0, meta offset: 2095127, data offset: 35667226} [unknown_version=0] [file=/data01/deploy/data/data/t_15259/log/page_29_0/meta], e.what() = DB::Exception, Stack trace:
0x1d272d3 StackTrace::StackTrace() [tiflash+30569171]
dbms/src/Common/StackTrace.cpp:23
0x1d248d6 DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int) [tiflash+30558422]
dbms/src/Common/Exception.h:41
0x79f53a3 DB::PS::V2::PageFile::MetaMergingReader::moveNext(unsigned int*) [tiflash+127882147]
dbms/src/Storages/Page/V2/PageFile.cpp:585
0x7a06501 DB::PS::V2::PageStorage::restore() [tiflash+127952129]
dbms/src/Storages/Page/V2/PageStorage.cpp:289
0x781ebf9 DB::DM::StoragePool::restore() [tiflash+125955065]
dbms/src/Storages/DeltaMerge/StoragePool.cpp:377
0x7788ca1 DB::DM::DeltaMergeStore::DeltaMergeStore(DB::Context&, bool, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, long, std::__1::vector<DB::DM::ColumnDefine, std::__1::allocator<DB::DM::ColumnDefine> > const&, DB::DM::ColumnDefine const&, bool, unsigned long, DB::DM::DeltaMergeStore::Settings const&) [tiflash+125340833]
dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp:243
0x771283d DB::StorageDeltaMerge::getAndMaybeInitStore() [tiflash+124856381]
dbms/src/Storages/StorageDeltaMerge.cpp:1545
0x771fa2a DB::StorageDeltaMerge::initStoreIfDataDirExist() [tiflash+124910122]
dbms/src/Storages/StorageDeltaMerge.cpp:1577
0x1d564cb DB::initStores(DB::Context&, Poco::Logger*, bool)::$_14::operator()() const [tiflash+30762187]
dbms/src/Server/Server.cpp:487
0x1d5e5f1 void* std::__1::__thread_proxy<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, DB::initStores(DB::Context&, Poco::Logger*, bool)::$_14> >(void*) [tiflash+30795249]
/usr/local/bin/../include/c++/v1/thread:291
0x7fd6b3d07ea5 start_thread [libpthread.so.0+32421]
0x7fd6b381a9fd __clone [libc.so.6+1042941]"] [thread_id=34]
[2022/06/24 05:51:41.033 +08:00] [ERROR] [Exception.cpp:85] ["DB::EngineStoreApplyRes DB::HandleWriteRaftCmd(const DB::EngineStoreServerWrap *, DB::WriteCmdsView, DB::RaftCmdHeader):Code: 49, e.displayText() = DB::Exception: PageFile binary version not match {MergingReader of PageFile_7_0, type: Formal, sequence no: 0, meta offset: 25098, data offset: 9991564} [unknown_version=0] [file=/data01/deploy/data/data/t_15303/meta/page_7_0/meta], e.what() = DB::Exception, Stack trace:
0x1d272d3 StackTrace::StackTrace() [tiflash+30569171]
dbms/src/Common/StackTrace.cpp:23
0x1d248d6 DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int) [tiflash+30558422]
dbms/src/Common/Exception.h:41
0x79f53a3 DB::PS::V2::PageFile::MetaMergingReader::moveNext(unsigned int*) [tiflash+127882147]
dbms/src/Storages/Page/V2/PageFile.cpp:585
0x7a06501 DB::PS::V2::PageStorage::restore() [tiflash+127952129]
dbms/src/Storages/Page/V2/PageStorage.cpp:289
0x781ec0f DB::DM::StoragePool::restore() [tiflash+125955087]
dbms/src/Storages/DeltaMerge/StoragePool.cpp:379
0x7788ca1 DB::DM::DeltaMergeStore::DeltaMergeStore(DB::Context&, bool, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, long, std::__1::vector<DB::DM::ColumnDefine, std::__1::allocator<DB::DM::ColumnDefine> > const&, DB::DM::ColumnDefine const&, bool, unsigned long, DB::DM::DeltaMergeStore::Settings const&) [tiflash+125340833]
dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp:243
0x771283d DB::StorageDeltaMerge::getAndMaybeInitStore() [tiflash+124856381]
dbms/src/Storages/StorageDeltaMerge.cpp:1545
0x77187fe DB::StorageDeltaMerge::getSchemaSnapshotAndBlockForDecoding(DB::TableDoubleLockHolder<false> const&, bool) [tiflash+124880894]
dbms/src/Storages/StorageDeltaMerge.cpp:910
0x7916b6d DB::writeRegionDataToStorage(DB::Context&, DB::RegionPtrWithBlock const&, std::__1::vector<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject<false> const> >, std::__1::allocator<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject<false> const> > > >&, Poco::Logger*)::$_0::operator()(bool) const [tiflash+126970733]
dbms/src/Storages/Transaction/PartitionStreams.cpp:123
0x7912f32 DB::writeRegionDataToStorage(DB::Context&, DB::RegionPtrWithBlock const&, std::__1::vector<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject<false> const> >, std::__1::allocator<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject<false> const> > > >&, Poco::Logger*) [tiflash+126955314]
dbms/src/Storages/Transaction/PartitionStreams.cpp:169
0x7912c9c DB::RegionTable::writeBlockByRegion(DB::Context&, DB::RegionPtrWithBlock const&, std::__1::vector<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject<false> const> >, std::__1::allocator<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject<false> const> > > >&, Poco::Logger*, bool) [tiflash+126954652]
dbms/src/Storages/Transaction/PartitionStreams.cpp:348
0x79351ff DB::Region::handleWriteRaftCmd(DB::WriteCmdsView const&, unsigned long, unsigned long, DB::TMTContext&) [tiflash+127095295]
dbms/src/Storages/Transaction/Region.cpp:712
0x7901910 DB::KVStore::handleWriteRaftCmd(DB::WriteCmdsView const&, unsigned long, unsigned long, unsigned long, DB::TMTContext&) [tiflash+126884112]
dbms/src/Storages/Transaction/KVStore.cpp:285
0x791b822 HandleWriteRaftCmd [tiflash+126990370]
dbms/src/Storages/Transaction/ProxyFFI.cpp:92
0x7fd6b61686e0 raftstore::store::fsm::apply::ApplyDelegate$LT$EK$GT$::process_raft_cmd::h4c8bba241d5ee4d4 [libtiflash_proxy.so+28567264]
0x7fd6b6165d6c raftstore::store::fsm::apply::ApplyDelegate$LT$EK$GT$::handle_raft_committed_entries::h478c4bad834ddec7 [libtiflash_proxy.so+28556652]
0x7fd6b6185f91 raftstore::store::fsm::apply::ApplyFsm$LT$EK$GT$::handle_apply::he4c0b1137ae7cad9 [libtiflash_proxy.so+28688273]
0x7fd6b6188bfe raftstore::store::fsm::apply::ApplyFsm$LT$EK$GT$::handle_tasks::h5b138e5889c69e66 [libtiflash_proxy.so+28699646]
0x7fd6b618b510 _$LT$raftstore..store..fsm..apply..ApplyPoller$LT$EK$GT$$u20$as$u20$batch_system..batch..PollHandler$LT$raftstore..store..fsm..apply..ApplyFsm$LT$EK$GT$$C$raftstore..store..fsm..apply..ControlFsm$GT$$GT$::handle_normal::he20c24833a37424f [libtiflash_proxy.so+28710160]
0x7fd6b5dfe609 batch_system::batch::Poller$LT$N$C$C$C$Handler$GT$::poll::hec57674a22fdcc30 [libtiflash_proxy.so+24987145]
0x7fd6b64a7cd8 std::sys_common::backtrace::__rust_begin_short_backtrace::h189c0b2868fa537c [libtiflash_proxy.so+31972568]
0x7fd6b5cdfec1 core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h7c53021c905641b6 [libtiflash_proxy.so+23813825]
0x7fd6b66bdfca std::sys::unix::thread::Thread::new::thread_start::hd39c5f08bdcda277 [libtiflash_proxy.so+34160586]
0x7fd6b3d07ea5 start_thread [libpthread.so.0+32421]
0x7fd6b381a9fd __clone [libc.so.6+1042941]"] [thread_id=158]