【 TiDB 使用环境】生产环境
【 TiDB 版本】v7.5.1
【遇到的问题：br执行全备有时成功有时失败】
【集群配置】
br工具报错打印日志如下
完整错误日志：
err_log_file (3.0 MB)
部分错误提示：
[2024/05/10 03:00:20.789 +08:00] [WARN] [push.go:81] [“skip store”] [range-sn=1] [store-id=5] [error=“the store last heartbeat is too far, at 31m13.240519664s: [BR:KV:ErrKVStorage]tikv storage occur I/O error”]
[2024/05/10 03:04:31.571 +08:00] [ERROR] [client.go:1022] [“find region failed”] [range-sn=568] [error=“rpc error: code = Canceled desc = context canceled”] [errorVerbose=“rpc error: code = Canceled desc = context canceled\ngithub.com/tikv/pd/client.(*client).respForErr\n\t/go/pkg/mod/github.com/tikv/pd/client@v0.0.0-20240210135946-3488a653ddd9/client.go:1602\ngithub.com/tikv/pd/client.(*client).GetRegion\n\t/go/pkg/mod/github.com/tikv/pd/client@v0.0.0-20240210135946-3488a653ddd9/client.go:947\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).findTargetPeer\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:1020\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).handleFineGrained\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:1229\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).fineGrainedBackup.func2\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:1108\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650”] [region=null] [stack=“github.com/pingcap/tidb/br/pkg/backup.(*Client).findTargetPeer\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:1022\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).handleFineGrained\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:1229\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).fineGrainedBackup.func2\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:1108”]
[2024/05/10 03:04:32.575 +08:00] [ERROR] [client.go:1051] [“can not find a valid target peer”] [range-sn=567] [key=7480000000000002FF1F5F698000000000FF0000030135313437FF30393932FF333331FF3834353633FF3200FF000000000000F803FF8000000000000000FF0380000000018C26FFE400000000000000F8] [stack=“github.com/pingcap/tidb/br/pkg/backup.(*Client).findTargetPeer\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:1051\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).handleFineGrained\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:1229\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).fineGrainedBackup.func2\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:1108”]
[2024/05/10 03:04:32.608 +08:00] [ERROR] [main.go:60] [“br failed”] [error=“can not find a valid leader for key t\ufffd\u0000\u0000\u0000\u0000\u0000\u0002\ufffd\u001f_r\ufffd\u0000\u0000\u0000\u0000\ufffd\ufffd\u001et\u0000\u0000\u0000\u0000\u0000\ufffd: [BR:Backup:ErrBackupNoLeader]backup no leader”] [errorVerbose=“[BR:Backup:ErrBackupNoLeader]backup no leader\ncan not find a valid leader for key t\ufffd\u0000\u0000\u0000\u0000\u0000\u0002\ufffd\u001f_r\ufffd\u0000\u0000\u0000\u0000\ufffd\ufffd\u001et\u0000\u0000\u0000\u0000\u0000\ufffd\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).findTargetPeer\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:1053\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).handleFineGrained\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:1229\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).fineGrainedBackup.func2\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:1108\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650”] [stack=“main.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:60\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:267”]
