br backup full 报错

🪐 TiDB 技术问题
1

[unit-name=“range start:74800000000000012c5f69800000000000000100 end:74800000000000012c5f698000000000000001fb”] [error=“rpc error: code = Canceled desc = context canceled”] [errorVerbose=“rpc error: code = Canceled desc = context canceled\ngithub.com/tikv/pd/client.(*client).respForErr\n\t/go/pkg/mod/github.com/tikv/pd/client@v0.0.0-20240210135946-3488a653ddd9/client.go:1602\ngithub.com/tikv/pd/client.(*client).GetAllStores\n\t/go/pkg/mod/github.com/tikv/pd/client@v0.0.0-20240210135946-3488a653ddd9/client.go:1193\ngithub.com/pingcap/tidb/br/pkg/conn/util.GetAllTiKVStores\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/conn/util/util.go:48\ngithub.com/pingcap/tidb/br/pkg/conn.GetAllTiKVStoresWithRetry.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/conn/conn.go:83\ngithub.com/pingcap/tidb/br/pkg/utils.WithRetry.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/utils/retry.go:217\ngithub.com/pingcap/tidb/br/pkg/utils.WithRetryV2[…]\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/utils/retry.go:235\ngithub.com/pingcap/tidb/br/pkg/utils.WithRetry\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/utils/retry.go:216\ngithub.com/pingcap/tidb/br/pkg/conn.GetAllTiKVStoresWithRetry\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/conn/conn.go:80\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).BackupRange\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:917\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).BackupRanges.func2\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:876\ngithub.com/pingcap/tidb/br/pkg/utils.(*WorkerPool).ApplyOnErrorGroup.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/utils/worker.go:76\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650”]
Error: can not find a valid leader for key t���_r�����0�: [BR:Backup:ErrBackupNoLeader]backup no leader

2024-08-24 22:54:43 (UTC+08:00)

TiKV 10.12.80.118:20160

[service.rs:130] [“backup canceled”] [error=RemoteStopped] [thread_id=0x5]

2024-08-24 22:54:43 (UTC+08:00)

TiKV 10.12.80.118:20160

[service.rs:130] [“backup canceled”] [error=RemoteStopped] [thread_id=0x5]

2024-08-24 22:54:43 (UTC+08:00)

TiKV 10.12.80.118:20160

[service.rs:130] [“backup canceled”] [error=RemoteStopped] [thread_id=0x5]

2024-08-24 22:54:43 (UTC+08:00)

TiKV 10.12.80.118:20160

[service.rs:130] [“backup canceled”] [error=RemoteStopped] [thread_id=0x5]

2024-08-24 22:54:44 (UTC+08:00)

TiKV 10.12.80.118:20160

[endpoint.rs:284] [“backup failed to send response”] [err_code=KV:Unknown] [err=“TrySendError { kind: Disconnected }”] [end_key=7480000000000000F65F698000000000000001040000018D5B7316060383D28EBB81830001] [start_key=7480000000000000F65F698000000000000001040000018D5B58A7480383D2881FD202D000] [region=“id: 3672297 start_key: 7480000000000000FFF65F698000000000FF000001040000018DFF5B58A7480383D288FF1FD202D000000000FC end_key: 7480000000000000FFF65F698000000000FF000001040000018DFF5B7316060383D28EFFBB81830001000000FC region_epoch { conf_ver: 17 version: 12878 } peers { id: 3672298 store_id: 4 } peers { id: 3672299 store_id: 1122825 } peers { id: 3672300 store_id: 1122827 }”] [thread_id=0x5]

2 
使用tiup cluster display <cluster-name>检查TiKV集群的状态。
确认所有的TiKV节点都是正常的，并且没有节点因为错误而宕机。
3

从报错信息堆栈看是BR从PD读取store信息失败了，看下BR到PD网络是否正常，还有–pd参数是否设置正确

4

解决了 这个报错有点迷惑人，原因是aws的ec2读取s3的权限不足。谢谢大佬

6

报错提示跟原因差距有点悬殊，这种得靠经验来判断了

7

这报错和实际原因 差了十万八千里呀。