【 TiDB 使用环境`】生产环境
【 TiDB 版本】tidb:v5.1.1 ,tispark :v2.5.0 , spark: 3.0.1
【遇到的问题】tispark读取tikv报错
【问题现象及影响】
tispark日志:
22/07/06 15:12:14 ERROR DAGIterator: Process region tasks failed, remain 0 tasks not executed due to
com.pingcap.tikv.exception.GrpcException: retry is exhausted.
at com.pingcap.tikv.util.ConcreteBackOffer.doBackOffWithMaxSleep(ConcreteBackOffer.java:148)
at com.pingcap.tikv.util.ConcreteBackOffer.doBackOff(ConcreteBackOffer.java:119)
at com.pingcap.tikv.region.RegionStoreClient.handleCopResponse(RegionStoreClient.java:703)
at com.pingcap.tikv.region.RegionStoreClient.coprocess(RegionStoreClient.java:675)
at com.pingcap.tikv.operation.iterator.DAGIterator.process(DAGIterator.java:219)
at com.pingcap.tikv.operation.iterator.DAGIterator.lambda$submitTasks$1(DAGIterator.java:90)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: com.pingcap.tikv.exception.GrpcException: message: “EpochNotMatch current epoch of region 275292521 is conf_ver: 437 version: 2285, but you sent conf_ver: 437 version: 2282”
epoch_not_match {
current_regions {
id: 275292521
start_key: “t\200\000\000\000\000\000\000\377\027_r\200\000\000\000\000\377\025\271o\000\000\000\000\000\372”
end_key: “t\200\000\000\000\000\000\000\377\031_i\200\000\000\000\000\377\000\000\001\003\200\000\000\000\377\000\000\025!\003\200\000\000\377\000\000\000\000\000\003\200\000\377\000\000\000\000\000\006\003\200\377\000\000\000\000\000\000N\000\376”
region_epoch {
conf_ver: 437
version: 2285
}
peers {
id: 275292522
store_id: 274433474
}
peers {
id: 275292524
store_id: 16
}
peers {
id: 275293104
store_id: 1
}
}
current_regions {
id: 275294317
end_key: “t\200\000\000\000\000\000\000\377\027_r\200\000\000\000\000\377\025\271o\000\000\000\000\000\372”
region_epoch {
conf_ver: 437
version: 2285
}
peers {
id: 275294318
store_id: 274433474
}
peers {
id: 275294319
store_id: 16
}
peers {
id: 275294320
store_id: 1
}
}
}
at com.pingcap.tikv.region.RegionStoreClient.handleCopResponse(RegionStoreClient.java:704)
… 9 more
tikv日志:
[endpoint.rs:632] [error-response] [err=“Region error (will back off and retry) message: "EpochNotMatch current epoch of region 275292521 is conf_ver: 437 version: 2285, but you sent conf_ver: 437 version: 2282" epoch_not_match { current_regions { id: 275292521 start_key: 7480000000000000FF175F728000000000FF15B96F0000000000FA end_key: 7480000000000000FF195F698000000000FF0000010380000000FF0000152103800000FF0000000000038000FF0000000000060380FF0000000000004E00FE region_epoch { conf_ver: 437 version: 2285 } peers { id: 275292522 store_id: 274433474 } peers { id: 275292524 store_id: 16 } peers { id: 275293104 store_id: 1 } } current_regions { id: 275294317 end_key: 7480000000000000FF175F728000000000FF15B96F0000000000FA region_epoch { conf_ver: 437 version: 2285 } peers { id: 275294318 store_id: 274433474 } peers { id: 275294319 store_id: 16 } peers { id: 275294320 store_id: 1 } } }”]