br备份失败!

【 TiDB 使用环境】生产环境 /测试/ Poc
【 TiDB 版本】5.0.1
【复现路径】备份失败了
【遇到的问题:问题现象及影响】
【资源配置】进入到 TiDB Dashboard -集群信息 (Cluster Info) -主机(Hosts) 截图此页面
【附件:截图/日志/监控】
日志
[2023/05/12 15:21:23.397 +08:00] [INFO] [info.go:49] [“Welcome to Backup & Restore (BR)”] [release-version=v5.4.2] [git-hash=d6be9105e6c40e6f82b8ee38e88c02d87664195a] [git-branch=heads/refs/tags/v5.4.2] [go-version=go1.16.4] [utc-build-time=“2022-06-27 06:42:41”] [race-enabled=false]
[2023/05/12 15:21:23.398 +08:00] [INFO] [common.go:634] [arguments] [__command=“br backup full”] [backupts=“2023-04-22 18:15:03”] [log-file=/tmp/backup.log] [pd=“[192.168.30.141:2379]”] [ratelimit=128] [s3.endpoint=http://192.168.9.102:19997] [storage=s3://tidbbackup/bak_20220105]
[2023/05/12 15:21:23.398 +08:00] [INFO] [conn.go:244] [“new mgr”] [pdAddrs=192.168.30.141:2379]
[2023/05/12 15:21:23.399 +08:00] [INFO] [client.go:352] [“[pd] create pd client with endpoints”] [pd-address=“[192.168.30.141:2379]”]
[2023/05/12 15:21:23.402 +08:00] [INFO] [base_client.go:331] [“[pd] update member urls”] [old-urls=“[http://192.168.30.141:2379]”] [new-urls=“[http://192.168.30.139:2379,http://192.168.30.141:2379]”]
[2023/05/12 15:21:23.402 +08:00] [INFO] [base_client.go:349] [“[pd] switch leader”] [new-leader=http://192.168.30.141:2379] [old-leader=]
[2023/05/12 15:21:23.402 +08:00] [INFO] [base_client.go:104] [“[pd] init cluster id”] [cluster-id=7125498871871762343]
[2023/05/12 15:21:23.402 +08:00] [INFO] [client.go:648] [“[pd] tso dispatcher created”] [dc-location=global]
[2023/05/12 15:21:23.403 +08:00] [INFO] [conn.go:219] [“checked alive KV stores”] [aliveStores=3] [totalStores=3]
[2023/05/12 15:21:23.403 +08:00] [INFO] [client.go:352] [“[pd] create pd client with endpoints”] [pd-address=“[192.168.30.141:2379]”]
[2023/05/12 15:21:23.405 +08:00] [INFO] [base_client.go:331] [“[pd] update member urls”] [old-urls=“[http://192.168.30.141:2379]”] [new-urls=“[http://192.168.30.139:2379,http://192.168.30.141:2379]”]
[2023/05/12 15:21:23.405 +08:00] [INFO] [base_client.go:349] [“[pd] switch leader”] [new-leader=http://192.168.30.141:2379] [old-leader=]
[2023/05/12 15:21:23.405 +08:00] [INFO] [base_client.go:104] [“[pd] init cluster id”] [cluster-id=7125498871871762343]
[2023/05/12 15:21:23.406 +08:00] [INFO] [client.go:648] [“[pd] tso dispatcher created”] [dc-location=global]
[2023/05/12 15:21:23.406 +08:00] [INFO] [tidb.go:72] [“new domain”] [store=tikv-7125498871871762343] [“ddl lease”=1s] [“stats lease”=-1ns] [“index usage sync lease”=0s]
[2023/05/12 15:21:23.456 +08:00] [INFO] [domain.go:169] [“full load InfoSchema success”] [currentSchemaVersion=0] [neededSchemaVersion=83] [“start time”=21.409494ms]
[2023/05/12 15:21:23.460 +08:00] [INFO] [domain.go:432] [“full load and reset schema validator”]
[2023/05/12 15:21:23.460 +08:00] [INFO] [ddl.go:347] [“[ddl] start DDL”] [ID=600afbaf-4a13-4025-a63c-30690db5b44f] [runWorker=false]
[2023/05/12 15:21:23.463 +08:00] [INFO] [backup.go:261] [“get newCollationEnable for check during restore”] [newCollationEnable=False]
[2023/05/12 15:21:23.463 +08:00] [INFO] [client.go:93] [“new backup client”]
[2023/05/12 15:21:23.521 +08:00] [ERROR] [client.go:752] [“[pd] fetch pending tso requests error”] [dc-location=global] [error=“[PD:client:ErrClientGetTSO]context canceled: context canceled”] [stack=“github.com/tikv/pd/client.(*client).handleDispatcher\n\t/go/pkg/mod/github.com/tikv/pd@v1.1.0-beta.0.20211118054146-02848d2660ee/client/client.go:752”]
[2023/05/12 15:21:23.521 +08:00] [INFO] [client.go:666] [“[pd] exit tso dispatcher”] [dc-location=global]
[2023/05/12 15:21:23.522 +08:00] [ERROR] [client.go:752] [“[pd] fetch pending tso requests error”] [dc-location=global] [error=“[PD:client:ErrClientGetTSO]context canceled: context canceled”] [stack=“github.com/tikv/pd/client.(*client).handleDispatcher\n\t/go/pkg/mod/github.com/tikv/pd@v1.1.0-beta.0.20211118054146-02848d2660ee/client/client.go:752”]
[2023/05/12 15:21:23.522 +08:00] [INFO] [client.go:666] [“[pd] exit tso dispatcher”] [dc-location=global]
[2023/05/12 15:21:23.522 +08:00] [INFO] [collector.go:202] [“units canceled”] [cancel-unit=0]
[2023/05/12 15:21:23.522 +08:00] [INFO] [collector.go:68] [“Full backup failed summary”] [total-ranges=0] [ranges-succeed=0] [ranges-failed=0]
[2023/05/12 15:21:23.522 +08:00] [ERROR] [backup.go:40] [“failed to backup”] [error=“GC safepoint 441417809853939712 exceed TS 440967758610432000: [BR:Backup:ErrBackupGCSafepointExceeded]backup GC safepoint exceeded”] [errorVerbose=“[BR:Backup:ErrBackupGCSafepointExceeded]backup GC safepoint exceeded\nGC safepoint 441417809853939712 exceed TS 440967758610432000\ngithub.com/pingcap/tidb/br/pkg/utils.CheckGCSafePoint\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/utils/safe_point.go:71\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).GetTS\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:133\ngithub.com/pingcap/tidb/br/pkg/task.RunBackup\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/task/backup.go:280\nmain.runBackupCommand\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:39\nmain.newFullBackupCommand.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:108\ngithub.com/spf13/cobra.(*Command).execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:852\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:960\ngithub.com/spf13/cobra.(*Command).Execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:897\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:56\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:225\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1371”] [stack=“main.runBackupCommand\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:40\nmain.newFullBackupCommand.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:108\ngithub.com/spf13/cobra.(*Command).execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:852\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:960\ngithub.com/spf13/cobra.(*Command).Execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:897\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:56\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:225”]
[2023/05/12 15:21:23.522 +08:00] [ERROR] [main.go:58] [“br failed”] [error=“GC safepoint 441417809853939712 exceed TS 440967758610432000: [BR:Backup:ErrBackupGCSafepointExceeded]backup GC safepoint exceeded”] [errorVerbose=“[BR:Backup:ErrBackupGCSafepointExceeded]backup GC safepoint exceeded\nGC safepoint 441417809853939712 exceed TS 440967758610432000\ngithub.com/pingcap/tidb/br/pkg/utils.CheckGCSafePoint\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/utils/safe_point.go:71\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).GetTS\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:133\ngithub.com/pingcap/tidb/br/pkg/task.RunBackup\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/task/backup.go:280\nmain.runBackupCommand\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:39\nmain.newFullBackupCommand.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:108\ngithub.com/spf13/cobra.(*Command).execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:852\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:960\ngithub.com/spf13/cobra.(*Command).Execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:897\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:56\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:225\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1371”] [stack=“main.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:58\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:225”]

GC设置的多久?

怎么看 :joy:

show global variables like '%gc%';

image

参考这个看看,是不是OOM了?

GC配置有问题么有

您这边的GC配置是默认的配置,另外BR应该会自动调整GC,不是GC的问题。
也可以尝试把GC调大,再试一下。