【TiDB 使用环境】生产环境
【TiDB 版本】v6.1.0
【操作系统】
【部署方式】机器部署
【集群数据量】
【集群节点数】
【问题复现路径】
br备份成功了，现在恢复数据失败，如何解决：
错误日志:
[2025/04/17 05:53:00.310 -08:00] [INFO] [collector.go:69] [“DataBase Restore failed summary”] [total-ranges=0] [ranges-succeed=0] [ranges-failed=0]
Error: load backupmeta failed: failed to read s3 file, file info: input.bucket=‘mlglobalbackup’, input.key=‘tidb_iap_bak/db_iapserver_misc/2025-04-02/backupmeta’: InvalidObjectState: Operation is not valid for the object’s storage class
status code: 403, request id: f8f9a58c-6584-4280-b751-116a5ce881f2, host id:
[2025/04/17 05:53:00.304 -08:00] [INFO] [manager.go:227] [“failed to campaign”] [“owner info”=“[ddl] /tidb/ddl/fg/owner ownerManager 6667aa52-bb0f-478d-a473-6dbfc984c1cf”] [error=“context canceled”]
[2025/04/17 05:53:00.304 -08:00] [INFO] [manager.go:208] [“break campaign loop, context is done”] [“owner info”=“[ddl] /tidb/ddl/fg/owner ownerManager 6667aa52-bb0f-478d-a473-6dbfc984c1cf”]
[2025/04/17 05:53:00.305 -08:00] [INFO] [manager.go:252] [“revoke session”] [“owner info”=“[ddl] /tidb/ddl/fg/owner ownerManager 6667aa52-bb0f-478d-a473-6dbfc984c1cf”]
[2025/04/17 05:53:00.307 -08:00] [INFO] [ddl_worker.go:159] [“[ddl] DDL worker closed”] [worker=“worker 1, tp general”] [“take time”=175ns]
[2025/04/17 05:53:00.307 -08:00] [INFO] [ddl_worker.go:159] [“[ddl] DDL worker closed”] [worker=“worker 2, tp add index”] [“take time”=54ns]
[2025/04/17 05:53:00.307 -08:00] [INFO] [delete_range.go:130] [“[ddl] closing delRange”]
[2025/04/17 05:53:00.307 -08:00] [INFO] [session_pool.go:90] [“[ddl] closing sessionPool”]
[2025/04/17 05:53:00.307 -08:00] [INFO] [ddl.go:465] [“[ddl] DDL closed”] [ID=6667aa52-bb0f-478d-a473-6dbfc984c1cf] [“take time”=4.16906ms]
[2025/04/17 05:53:00.307 -08:00] [INFO] [ddl.go:363] [“[ddl] stop DDL”] [ID=6667aa52-bb0f-478d-a473-6dbfc984c1cf]
[2025/04/17 05:53:00.309 -08:00] [INFO] [domain.go:594] [“loadSchemaInLoop exited.”]
[2025/04/17 05:53:00.309 -08:00] [INFO] [domain.go:515] [“infoSyncerKeeper exited.”]
[2025/04/17 05:53:00.309 -08:00] [INFO] [domain.go:1758] [“serverIDKeeper exited.”]
[2025/04/17 05:53:00.309 -08:00] [INFO] [manager.go:227] [“failed to campaign”] [“owner info”=“[stats] /tidb/stats/owner ownerManager 6667aa52-bb0f-478d-a473-6dbfc984c1cf”] [error=“lost watcher waiting for delete”]
[2025/04/17 05:53:00.309 -08:00] [INFO] [manager.go:227] [“failed to campaign”] [“owner info”=“[stats] /tidb/stats/owner ownerManager 6667aa52-bb0f-478d-a473-6dbfc984c1cf”] [error=“lost watcher waiting for delete”]
[2025/04/17 05:53:00.309 -08:00] [INFO] [domain.go:540] [“globalConfigSyncerKeeper exited.”]
[2025/04/17 05:53:00.309 -08:00] [INFO] [manager.go:199] [“etcd session is done, creates a new one”] [“owner info”=“[stats] /tidb/stats/owner ownerManager 6667aa52-bb0f-478d-a473-6dbfc984c1cf”]
[2025/04/17 05:53:00.309 -08:00] [INFO] [domain.go:486] [“topNSlowQueryLoop exited.”]
[2025/04/17 05:53:00.309 -08:00] [INFO] [domain.go:721] [“domain closed”] [“take time”=6.843689ms]
[2025/04/17 05:53:00.310 -08:00] [INFO] [client.go:768] [“[pd] stop fetching the pending tso requests due to context canceled”] [dc-location=global]
[2025/04/17 05:53:00.310 -08:00] [INFO] [client.go:706] [“[pd] exit tso dispatcher”] [dc-location=global]
[2025/04/17 05:53:00.310 -08:00] [INFO] [client.go:768] [“[pd] stop fetching the pending tso requests due to context canceled”] [dc-location=global]
[2025/04/17 05:53:00.310 -08:00] [INFO] [client.go:706] [“[pd] exit tso dispatcher”] [dc-location=global]
[2025/04/17 05:53:00.310 -08:00] [INFO] [collector.go:204] [“units canceled”] [cancel-unit=0]
[2025/04/17 05:53:00.310 -08:00] [INFO] [collector.go:70] [“DataBase Restore failed summary”] [total-ranges=0] [ranges-succeed=0] [ranges-failed=0]
[2025/04/17 05:53:00.310 -08:00] [ERROR] [restore.go:40] [“failed to restore”] [error="load backupmeta failed: failed to read s3 file, file info: input.bucket=‘mlglobalbackup’, input.key=‘tidb_iap_bak/db_iapserver_misc/2025-04-02/backupmeta’: InvalidObjectState: Operation is not valid for the object’s storage class\n\tstatus code: 403, request id: f8f9a58c-6584-4280-b751-116a5ce881f2, host id: "] [errorVerbose=“InvalidObjectState: Operation is not valid for the object’s storage class\n\tstatus code: 403, request id: f8f9a58c-6584-4280-b751-116a5ce881f2, host id: \nfailed to read s3 file, file info: input.bucket=‘mlglobalbackup’, input.key=‘tidb_iap_bak/db_iapserver_misc/2025-04-02/backupmeta’\ngithub.com/pingcap/tidb/br/pkg/storage.(*S3Storage).ReadFile\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/storage/s3.go:419\ngithub.com/pingcap/tidb/br/pkg/task.ReadBackupMeta\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/task/common.go:627\ngithub.com/pingcap/tidb/br/pkg/task.RunRestore\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/task/restore.go:432\nmain.runRestoreCommand\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/restore.go:39\nmain.newDBRestoreCommand.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/restore.go:118\ngithub.com/spf13/cobra.(*Command).execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.4.0/command.go:856\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/go/pkg/mod/github.com/spf13/cobra@v1.4.0/command.go:974\ngithub.com/spf13/cobra.(*Command).Execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.4.0/command.go:902\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:57\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1571\nload backupmeta failed”] [stack=“main.runRestoreCommand\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/restore.go:40\nmain.newDBRestoreCommand.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/restore.go:118\ngithub.com/spf13/cobra.(*Command).execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.4.0/command.go:856\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/go/pkg/mod/github.com/spf13/cobra@v1.4.0/command.go:974\ngithub.com/spf13/cobra.(*Command).Execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.4.0/command.go:902\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:57\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250”]
[2025/04/17 05:53:00.310 -08:00] [ERROR] [main.go:59] [“br failed”] [error="load backupmeta failed: failed to read s3 file, file info: input.bucket=‘mlglobalbackup’, input.key=‘tidb_iap_bak/db_iapserver_misc/2025-04-02/backupmeta’: InvalidObjectState: Operation is not valid for the object’s storage class\n\tstatus code: 403, request id: f8f9a58c-6584-4280-b751-116a5ce881f2, host id: "] [errorVerbose=“InvalidObjectState: Operation is not valid for the object’s storage class\n\tstatus code: 403, request id: f8f9a58c-6584-4280-b751-116a5ce881f2, host id: \nfailed to read s3 file, file info: input.bucket=‘mlglobalbackup’, input.key=‘tidb_iap_bak/db_iapserver_misc/2025-04-02/backupmeta’\ngithub.com/pingcap/tidb/br/pkg/storage.(*S3Storage).ReadFile\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/storage/s3.go:419\ngithub.com/pingcap/tidb/br/pkg/task.ReadBackupMeta\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/task/common.go:627\ngithub.com/pingcap/tidb/br/pkg/task.RunRestore\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/task/restore.go:432\nmain.runRestoreCommand\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/restore.go:39\nmain.newDBRestoreCommand.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/restore.go:118\ngithub.com/spf13/cobra.(*Command).execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.4.0/command.go:856\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/go/pkg/mod/github.com/spf13/cobra@v1.4.0/command.go:974\ngithub.com/spf13/cobra.(*Command).Execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.4.0/command.go:902\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:57\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1571\nload backupmeta failed”] [stack=“main.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:59\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250”]