【 TiDB 使用环境】生产环境
【 TiDB 版本】
tidb版本为v6.5.0，在金山云的kce中部署的
【复现路径】
通过配置be工具备份全量集群数据到ks3存储
【遇到的问题：问题现象及影响】
1、v5.4.1、v6.5.0、v6.5.1版本的br工具无法备份6.5.0版本的tidb集群到ks3上
2、tidb-operator1.3版本、1.4版本都试过了全部报错
【资源配置】
var version string
var ns string
func init() {
flag.StringVar(&ns, “ns”, “eeae42ce-c439-442b-863b-0119929a076c”, “name space”)
flag.StringVar(&version, “v”, “v5.4.1”, “br version,support v6.5.1 and v5.4.1 default v5.4.1”)
}
func newConfig() *v1alpha1.Backup {
image := fmt.Sprintf(“hub.kce.ksyun.com/nosql/tidb/br:%s”, version)
sendCredToTikv := true
storageClassName := “openebs-hostpath”
suffix := time.Now().Unix()
return &v1alpha1.Backup{
TypeMeta: metav1.TypeMeta{
Kind: “Backup”,
APIVersion: “pingcap.com/v1alpha1”,
},
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf(“liyuntang-%d”, suffix),
},
Spec: v1alpha1.BackupSpec{
//ServiceAccount: "tidb-backup-manager",
BR: &v1alpha1.BRConfig{
Cluster: "basic",
ClusterNamespace: ns,
SendCredToTikv: &sendCredToTikv,
},
ToolImage: image,
ImagePullSecrets: []corev1.LocalObjectReference{{Name: "nosql-image"}},
StorageProvider: v1alpha1.StorageProvider{
S3: &v1alpha1.S3StorageProvider{
Provider: "other",
Region: "BEIJING",
Bucket: "tidb-cket-bj",
Endpoint: "http://kal.ksyun.com",
SecretName: "kt",
Prefix: fmt.Sprintf("200074/%s/manual/%d", ns, suffix),
},
},
StorageClassName: &storageClassName,
Type: v1alpha1.BackupTypeFull,
CleanPolicy: v1alpha1.CleanPolicyTypeDelete,
},
}
}
【附件：截图/日志/监控】
[root@vm10-32-1-164 tmp]# kubectl logs backup-2-1-liyuntang-1679477820-qvc24 -n $ns
Create rclone.conf file.
/tidb-backup-manager backup --namespace=eeae42ce-c439-442b-863b-0119929a076c --backupName=2-1-liyuntang-1679477820 --tikvVersion=v6.5.0
I0322 17:37:11.709234 9 backup.go:72] start to process backup eeae42ce-c439-442b-863b-0119929a076c/2-1-liyuntang-1679477820
I0322 17:37:11.725730 9 backup_status_updater.go:86] Backup: [eeae42ce-c439-442b-863b-0119929a076c/2-1-liyuntang-1679477820] updated successfully
E0322 17:37:11.730469 9 backup_status_updater.go:89] Failed to update backup [eeae42ce-c439-442b-863b-0119929a076c/2-1-liyuntang-1679477820], error: Operation cannot be fulfilled on backups.pingcap.com “2-1-liyuntang-1679477820”: the object has been modified; please apply your changes to the latest version and try again
I0322 17:37:11.749029 9 backup_status_updater.go:86] Backup: [eeae42ce-c439-442b-863b-0119929a076c/2-1-liyuntang-1679477820] updated successfully
I0322 17:37:11.749097 9 backup.go:69] Running br command with args: [backup full --pd=basic-pd.eeae42ce-c439-442b-863b-0119929a076c:2379 --storage=s3://tidb-backup-bucket-bj/2000102274/eeae42ce-c439-442b-863b-0119929a076c/manual/2-1-liyuntang-1679477820 --s3.region=BEIJING --s3.provider=other --s3.endpoint=http://ks3-cn-beijing-internal.ksyun.com]
I0322 17:37:11.839176 9 backup.go:93] [2023/03/22 17:37:11.838 +08:00] [INFO] [info.go:49] [“Welcome to Backup & Restore (BR)”] [release-version=v6.5.0] [git-hash=706c3fa3c526cdba5b3e9f066b1a568fb96c56e3] [git-branch=heads/refs/tags/v6.5.0] [go-version=go1.19.3] [utc-build-time=“2022-12-27 03:41:53”] [race-enabled=false]
I0322 17:37:11.839245 9 backup.go:93] [2023/03/22 17:37:11.839 +08:00] [INFO] [common.go:718] [arguments] [__command=“br backup full”] [pd=“[basic-pd.eeae42ce-c439-442b-863b-0119929a076c:2379]”] [s3.endpoint=http://ks3-cn-beijing-internal.ksyun.com] [s3.provider=other] [s3.region=BEIJING] [storage=s3://tidb-backup-bucket-bj/2000102274/eeae42ce-c439-442b-863b-0119929a076c/manual/2-1-liyuntang-1679477820]
I0322 17:37:11.839311 9 backup.go:93] [2023/03/22 17:37:11.839 +08:00] [INFO] [backup.go:188] [“use checkpoint’s default GC TTL”] [“GC TTL”=4320]
I0322 17:37:11.839465 9 backup.go:93] [2023/03/22 17:37:11.839 +08:00] [INFO] [conn.go:147] [“new mgr”] [pdAddrs=basic-pd.eeae42ce-c439-442b-863b-0119929a076c:2379]
I0322 17:37:11.855475 9 backup.go:93] [2023/03/22 17:37:11.855 +08:00] [INFO] [client.go:405] [“[pd] create pd client with endpoints”] [pd-address=“[basic-pd.eeae42ce-c439-442b-863b-0119929a076c:2379]”]
I0322 17:37:11.873635 9 backup.go:93] [2023/03/22 17:37:11.873 +08:00] [INFO] [base_client.go:360] [“[pd] update member urls”] [old-urls=“[http://basic-pd.eeae42ce-c439-442b-863b-0119929a076c:2379]”] [new-urls=“[http://basic-pd-0.basic-pd-peer.eeae42ce-c439-442b-863b-0119929a076c.svc:2379,http://basic-pd-1.basic-pd-peer.eeae42ce-c439-442b-863b-0119929a076c.svc:2379,http://basic-pd-2.basic-pd-peer.eeae42ce-c439-442b-863b-0119929a076c.svc:2379]”]
I0322 17:37:11.874295 9 backup.go:93] [2023/03/22 17:37:11.873 +08:00] [INFO] [base_client.go:378] [“[pd] switch leader”] [new-leader=http://basic-pd-0.basic-pd-peer.eeae42ce-c439-442b-863b-0119929a076c.svc:2379] [old-leader=]
I0322 17:37:11.874353 9 backup.go:93] [2023/03/22 17:37:11.873 +08:00] [INFO] [base_client.go:105] [“[pd] init cluster id”] [cluster-id=7205466925320758990]
I0322 17:37:11.874368 9 backup.go:93] [2023/03/22 17:37:11.873 +08:00] [INFO] [client.go:698] [“[pd] tso dispatcher created”] [dc-location=global]
I0322 17:37:11.893793 9 backup.go:93] [2023/03/22 17:37:11.893 +08:00] [INFO] [conn.go:121] [“checked alive KV stores”] [aliveStores=4] [totalStores=4]
I0322 17:37:11.893830 9 backup.go:93] [2023/03/22 17:37:11.893 +08:00] [INFO] [client.go:405] [“[pd] create pd client with endpoints”] [pd-address=“[basic-pd.eeae42ce-c439-442b-863b-0119929a076c:2379]”]
I0322 17:37:11.909555 9 backup.go:93] [2023/03/22 17:37:11.909 +08:00] [INFO] [base_client.go:360] [“[pd] update member urls”] [old-urls=“[http://basic-pd.eeae42ce-c439-442b-863b-0119929a076c:2379]”] [new-urls=“[http://basic-pd-0.basic-pd-peer.eeae42ce-c439-442b-863b-0119929a076c.svc:2379,http://basic-pd-1.basic-pd-peer.eeae42ce-c439-442b-863b-0119929a076c.svc:2379,http://basic-pd-2.basic-pd-peer.eeae42ce-c439-442b-863b-0119929a076c.svc:2379]”]
I0322 17:37:11.909707 9 backup.go:93] [2023/03/22 17:37:11.909 +08:00] [INFO] [base_client.go:378] [“[pd] switch leader”] [new-leader=http://basic-pd-0.basic-pd-peer.eeae42ce-c439-442b-863b-0119929a076c.svc:2379] [old-leader=]
I0322 17:37:11.909726 9 backup.go:93] [2023/03/22 17:37:11.909 +08:00] [INFO] [base_client.go:105] [“[pd] init cluster id”] [cluster-id=7205466925320758990]
I0322 17:37:11.909796 9 backup.go:93] [2023/03/22 17:37:11.909 +08:00] [INFO] [client.go:698] [“[pd] tso dispatcher created”] [dc-location=global]
I0322 17:37:11.926315 9 backup.go:93] [2023/03/22 17:37:11.926 +08:00] [INFO] [tidb.go:77] [“new domain”] [store=tikv-7205466925320758990] [“ddl lease”=1s] [“stats lease”=-1ns] [“index usage sync lease”=0s]
I0322 17:37:11.959845 9 backup.go:93] [2023/03/22 17:37:11.959 +08:00] [INFO] [domain.go:2280] [acquireServerID] [serverID=1157728] [“lease id”=250787077b919dfd]
I0322 17:37:11.966885 9 backup.go:93] [2023/03/22 17:37:11.966 +08:00] [WARN] [info.go:245] [“init TiFlashReplicaManager”] [“pd addrs”=“[basic-pd-1.basic-pd-peer.eeae42ce-c439-442b-863b-0119929a076c.svc:2379,basic-pd-0.basic-pd-peer.eeae42ce-c439-442b-863b-0119929a076c.svc:2379,basic-pd-2.basic-pd-peer.eeae42ce-c439-442b-863b-0119929a076c.svc:2379]”]
I0322 17:37:12.131522 9 backup.go:93] [2023/03/22 17:37:12.131 +08:00] [INFO] [domain.go:220] [“full load InfoSchema success”] [currentSchemaVersion=0] [neededSchemaVersion=138] [“start time”=118.796307ms]
I0322 17:37:12.135279 9 backup.go:93] [2023/03/22 17:37:12.135 +08:00] [INFO] [domain.go:487] [“full load and reset schema validator”]
I0322 17:37:12.135340 9 backup.go:93] [2023/03/22 17:37:12.135 +08:00] [INFO] [ddl.go:701] [“[ddl] start DDL”] [ID=97618410-f614-4975-a3b4-512021ba6fd9] [runWorker=false]
I0322 17:37:12.135385 9 backup.go:93] [2023/03/22 17:37:12.135 +08:00] [INFO] [ddl.go:647] [“[ddl] start delRangeManager OK”] [“is a emulator”=false]
I0322 17:37:12.135514 9 backup.go:93] [2023/03/22 17:37:12.135 +08:00] [WARN] [env.go:52] [“[ddl-ingest] initialize environment failed”] [“storage limitation”=“only support TiKV storage”] [“current storage”=unistore] [“lightning is initialized”=false]
I0322 17:37:12.135618 9 backup.go:93] [2023/03/22 17:37:12.135 +08:00] [INFO] [ddl_worker.go:171] [“[ddl] start DDL worker”] [worker=“worker 1, tp general”]
I0322 17:37:12.135702 9 backup.go:93] [2023/03/22 17:37:12.135 +08:00] [INFO] [ddl_worker.go:171] [“[ddl] start DDL worker”] [worker=“worker 2, tp add index”]
I0322 17:37:12.135956 9 backup.go:93] [2023/03/22 17:37:12.135 +08:00] [INFO] [env.go:108] [“[ddl-ingest] the lightning sorted dir”] [“data path:”=/tmp/tidb/tmp_ddl-4000]
I0322 17:37:12.136048 9 backup.go:93] [2023/03/22 17:37:12.135 +08:00] [INFO] [env.go:75] [“[ddl-ingest] init global lightning backend environment finished”] [“memory limitation”=2147483648] [“sort path disk quota”=107374182400] [“max open file number”=1048576] [“lightning is initialized”=true]
I0322 17:37:12.136125 9 backup.go:93] [2023/03/22 17:37:12.136 +08:00] [INFO] [owner_daemon.go:70] [“begin advancer daemon”] [daemon-id=LogBackup::Advancer]
I0322 17:37:12.136144 9 backup.go:93] [2023/03/22 17:37:12.136 +08:00] [INFO] [manager.go:151] [“start campaign owner”] [ownerInfo=“[log-backup] /tidb/br-stream/owner”]
I0322 17:37:12.139580 9 backup.go:93] [2023/03/22 17:37:12.139 +08:00] [INFO] [owner_daemon.go:77] [“begin running daemon”] [id=01614d1c-3150-457f-b756-1279fe339fe3] [daemon-id=LogBackup::Advancer]
I0322 17:37:12.152950 9 backup.go:93] [2023/03/22 17:37:12.152 +08:00] [INFO] [backup.go:369] [“get new_collations_enabled_on_first_bootstrap config from system table”] [new_collation_enabled=True]
I0322 17:37:13.142840 9 backup.go:93] [2023/03/22 17:37:13.142 +08:00] [INFO] [ddl_workerpool.go:82] [“[ddl] closing workerPool”]
I0322 17:37:13.142869 9 backup.go:93] [2023/03/22 17:37:13.142 +08:00] [INFO] [ddl_workerpool.go:82] [“[ddl] closing workerPool”]
I0322 17:37:13.142886 9 backup.go:93] [2023/03/22 17:37:13.142 +08:00] [INFO] [ddl_worker.go:165] [“[ddl] DDL worker closed”] [worker=“worker 1, tp general”] [“take time”=1.481µs]
I0322 17:37:13.142902 9 backup.go:93] [2023/03/22 17:37:13.142 +08:00] [INFO] [ddl_worker.go:165] [“[ddl] DDL worker closed”] [worker=“worker 2, tp add index”] [“take time”=341ns]
I0322 17:37:13.142915 9 backup.go:93] [2023/03/22 17:37:13.142 +08:00] [INFO] [delete_range.go:148] [“[ddl] closing delRange”]
I0322 17:37:13.142925 9 backup.go:93] [2023/03/22 17:37:13.142 +08:00] [INFO] [session_pool.go:94] [“[ddl] closing sessionPool”]
I0322 17:37:13.142938 9 backup.go:93] [2023/03/22 17:37:13.142 +08:00] [INFO] [ddl.go:815] [“[ddl] DDL closed”] [ID=97618410-f614-4975-a3b4-512021ba6fd9] [“take time”=989.963581ms]
I0322 17:37:13.142952 9 backup.go:93] [2023/03/22 17:37:13.142 +08:00] [INFO] [ddl.go:639] [“[ddl] stop DDL”] [ID=97618410-f614-4975-a3b4-512021ba6fd9]
I0322 17:37:13.149836 9 backup.go:93] [2023/03/22 17:37:13.149 +08:00] [INFO] [domain.go:590] [“globalConfigSyncerKeeper exited.”]
I0322 17:37:13.149870 9 backup.go:93] [2023/03/22 17:37:13.149 +08:00] [INFO] [manager.go:247] [“failed to campaign”] [“owner info”=“[log-backup] /tidb/br-stream/owner ownerManager 01614d1c-3150-457f-b756-1279fe339fe3”] [error=“lost watcher waiting for delete”]
I0322 17:37:13.149887 9 backup.go:93] [2023/03/22 17:37:13.149 +08:00] [INFO] [domain.go:565] [“infoSyncerKeeper exited.”]
I0322 17:37:13.149904 9 backup.go:93] [2023/03/22 17:37:13.149 +08:00] [INFO] [domain.go:2307] [“serverIDKeeper exited.”]
I0322 17:37:13.150051 9 backup.go:93] [2023/03/22 17:37:13.149 +08:00] [INFO] [manager.go:247] [“failed to campaign”] [“owner info”=“[log-backup] /tidb/br-stream/owner ownerManager 01614d1c-3150-457f-b756-1279fe339fe3”] [error=“context canceled”]
I0322 17:37:13.150075 9 backup.go:93] [2023/03/22 17:37:13.149 +08:00] [INFO] [owner_daemon.go:81] [“daemon loop exits”] [id=01614d1c-3150-457f-b756-1279fe339fe3] [daemon-id=LogBackup::Advancer]
I0322 17:37:13.150092 9 backup.go:93] [2023/03/22 17:37:13.150 +08:00] [INFO] [manager.go:219] [“etcd session is done, creates a new one”] [“owner info”=“[log-backup] /tidb/br-stream/owner ownerManager 01614d1c-3150-457f-b756-1279fe339fe3”]
I0322 17:37:13.150107 9 backup.go:93] [2023/03/22 17:37:13.149 +08:00] [INFO] [domain.go:1104] [“closestReplicaReadCheckLoop exited.”]
I0322 17:37:13.150122 9 backup.go:93] [2023/03/22 17:37:13.150 +08:00] [WARN] [domain.go:2393] [“fail to wait until the ttl job manager stop”] [error=“context canceled”]
I0322 17:37:13.150144 9 backup.go:93] [2023/03/22 17:37:13.150 +08:00] [INFO] [domain.go:536] [“topNSlowQueryLoop exited.”]
I0322 17:37:13.150295 9 backup.go:93] [2023/03/22 17:37:13.150 +08:00] [INFO] [manager.go:223] [“break campaign loop, NewSession failed”] [“owner info”=“[log-backup] /tidb/br-stream/owner ownerManager 01614d1c-3150-457f-b756-1279fe339fe3”] [error=“context canceled”] [errorVerbose=“context canceled\ngithub.com/pingcap/errors.AddStack\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20220729040631-518f63d66278/errors.go:174\ngithub.com/pingcap/errors.Trace\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20220729040631-518f63d66278/juju_adaptor.go:15\ngithub.com/pingcap/tidb/util.contextDone\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/util/etcd.go:90\ngithub.com/pingcap/tidb/util.NewSession\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/util/etcd.go:50\ngithub.com/pingcap/tidb/owner.(*ownerManager).campaignLoop\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/owner/manager.go:221\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594”]
I0322 17:37:13.151085 9 backup.go:93] [2023/03/22 17:37:13.151 +08:00] [INFO] [domain.go:748] [“loadSchemaInLoop exited.”]
I0322 17:37:13.151117 9 backup.go:93] [2023/03/22 17:37:13.151 +08:00] [INFO] [domain.go:876] [“domain closed”] [“take time”=998.145782ms]
I0322 17:37:13.151130 9 backup.go:93] [2023/03/22 17:37:13.151 +08:00] [INFO] [glue.go:141] [“one shot domain closed”]
I0322 17:37:13.151142 9 backup.go:93] [2023/03/22 17:37:13.151 +08:00] [INFO] [glue.go:128] [“one shot session closed”]
I0322 17:37:13.151159 9 backup.go:93] [2023/03/22 17:37:13.151 +08:00] [INFO] [client.go:105] [“new backup client”]
I0322 17:37:13.180008 9 backup.go:93] [2023/03/22 17:37:13.179 +08:00] [INFO] [client.go:779] [“[pd] stop fetching the pending tso requests due to context canceled”] [dc-location=global]
I0322 17:37:13.180032 9 backup.go:93] [2023/03/22 17:37:13.179 +08:00] [INFO] [client.go:716] [“[pd] exit tso dispatcher”] [dc-location=global]
I0322 17:37:13.180364 9 backup.go:93] [2023/03/22 17:37:13.180 +08:00] [INFO] [client.go:779] [“[pd] stop fetching the pending tso requests due to context canceled”] [dc-location=global]
I0322 17:37:13.180382 9 backup.go:93] [2023/03/22 17:37:13.180 +08:00] [INFO] [client.go:716] [“[pd] exit tso dispatcher”] [dc-location=global]
I0322 17:37:13.180423 9 backup.go:93] [2023/03/22 17:37:13.180 +08:00] [INFO] [collector.go:220] [“units canceled”] [cancel-unit=0]
I0322 17:37:13.180444 9 backup.go:93] [2023/03/22 17:37:13.180 +08:00] [INFO] [collector.go:221] [“Full Backup failed summary”] [total-ranges=0] [ranges-succeed=0] [ranges-failed=0]
I0322 17:37:13.180691 9 backup.go:93] [2023/03/22 17:37:13.180 +08:00] [ERROR] [backup.go:49] [“failed to backup”] [error=“s3 bucket and region are not matched, bucket=tidb-backup-bucket-bj, input region=BEIJING, real region=us-east-1”] [errorVerbose=“s3 bucket and region are not matched, bucket=tidb-backup-bucket-bj, input region=BEIJING, real region=us-east-1\ngithub.com/pingcap/errors.AddStack\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20220729040631-518f63d66278/errors.go:174\ngithub.com/pingcap/errors.Trace\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20220729040631-518f63d66278/juju_adaptor.go:15\ngithub.com/pingcap/tidb/br/pkg/storage.NewS3Storage\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/storage/s3.go:377\ngithub.com/pingcap/tidb/br/pkg/storage.New\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/storage/storage.go:176\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).SetStorage\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:385\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).SetStorageAndCheckNotInUse\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:222\ngithub.com/pingcap/tidb/br/pkg/task.RunBackup\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/task/backup.go:387\nmain.runBackupCommand\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:48\nmain.newFullBackupCommand.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:117\ngithub.com/spf13/cobra.(*Command).execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:916\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:1044\ngithub.com/spf13/cobra.(*Command).Execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:968\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:57\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594”] [stack=“main.runBackupCommand\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:49\nmain.newFullBackupCommand.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:117\ngithub.com/spf13/cobra.(*Command).execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:916\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:1044\ngithub.com/spf13/cobra.(*Command).Execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:968\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:57\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250”]
I0322 17:37:13.180854 9 backup.go:93] [2023/03/22 17:37:13.180 +08:00] [ERROR] [main.go:59] [“br failed”] [error=“s3 bucket and region are not matched, bucket=tidb-backup-bucket-bj, input region=BEIJING, real region=us-east-1”] [errorVerbose=“s3 bucket and region are not matched, bucket=tidb-backup-bucket-bj, input region=BEIJING, real region=us-east-1\ngithub.com/pingcap/errors.AddStack\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20220729040631-518f63d66278/errors.go:174\ngithub.com/pingcap/errors.Trace\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20220729040631-518f63d66278/juju_adaptor.go:15\ngithub.com/pingcap/tidb/br/pkg/storage.NewS3Storage\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/storage/s3.go:377\ngithub.com/pingcap/tidb/br/pkg/storage.New\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/storage/storage.go:176\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).SetStorage\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:385\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).SetStorageAndCheckNotInUse\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:222\ngithub.com/pingcap/tidb/br/pkg/task.RunBackup\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/task/backup.go:387\nmain.runBackupCommand\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:48\nmain.newFullBackupCommand.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:117\ngithub.com/spf13/cobra.(*Command).execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:916\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:1044\ngithub.com/spf13/cobra.(*Command).Execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:968\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:57\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594”] [stack=“main.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:59\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250”]
I0322 17:37:13.188333 9 backup.go:93]
I0322 17:37:13.188365 9 backup.go:100] Error: s3 bucket and region are not matched, bucket=tidb-backup-bucket-bj, input region=BEIJING, real region=us-east-1
E0322 17:37:13.188681 9 manager.go:293] backup cluster eeae42ce-c439-442b-863b-0119929a076c/2-1-liyuntang-1679477820 data failed, err: cluster eeae42ce-c439-442b-863b-0119929a076c/2-1-liyuntang-1679477820, wait pipe message failed, errMsg [2023/03/22 17:37:13.180 +08:00] [ERROR] [backup.go:49] [“failed to backup”] [error=“s3 bucket and region are not matched, bucket=tidb-backup-bucket-bj, input region=BEIJING, real region=us-east-1”] [errorVerbose=“s3 bucket and region are not matched, bucket=tidb-backup-bucket-bj, input region=BEIJING, real region=us-east-1\ngithub.com/pingcap/errors.AddStack\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20220729040631-518f63d66278/errors.go:174\ngithub.com/pingcap/errors.Trace\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20220729040631-518f63d66278/juju_adaptor.go:15\ngithub.com/pingcap/tidb/br/pkg/storage.NewS3Storage\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/storage/s3.go:377\ngithub.com/pingcap/tidb/br/pkg/storage.New\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/storage/storage.go:176\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).SetStorage\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:385\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).SetStorageAndCheckNotInUse\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:222\ngithub.com/pingcap/tidb/br/pkg/task.RunBackup\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/task/backup.go:387\nmain.runBackupCommand\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:48\nmain.newFullBackupCommand.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:117\ngithub.com/spf13/cobra.(*Command).execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:916\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:1044\ngithub.com/spf13/cobra.(*Command).Execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:968\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:57\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594”] [stack=“main.runBackupCommand\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:49\nmain.newFullBackupCommand.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:117\ngithub.com/spf13/cobra.(*Command).execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:916\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:1044\ngithub.com/spf13/cobra.(*Command).Execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:968\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:57\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250”]
[2023/03/22 17:37:13.180 +08:00] [ERROR] [main.go:59] [“br failed”] [error=“s3 bucket and region are not matched, bucket=tidb-backup-bucket-bj, input region=BEIJING, real region=us-east-1”] [errorVerbose=“s3 bucket and region are not matched, bucket=tidb-backup-bucket-bj, input region=BEIJING, real region=us-east-1\ngithub.com/pingcap/errors.AddStack\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20220729040631-518f63d66278/errors.go:174\ngithub.com/pingcap/errors.Trace\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20220729040631-518f63d66278/juju_adaptor.go:15\ngithub.com/pingcap/tidb/br/pkg/storage.NewS3Storage\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/storage/s3.go:377\ngithub.com/pingcap/tidb/br/pkg/storage.New\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/storage/storage.go:176\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).SetStorage\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:385\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).SetStorageAndCheckNotInUse\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:222\ngithub.com/pingcap/tidb/br/pkg/task.RunBackup\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/task/backup.go:387\nmain.runBackupCommand\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:48\nmain.newFullBackupCommand.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:117\ngithub.com/spf13/cobra.(*Command).execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:916\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:1044\ngithub.com/spf13/cobra.(*Command).Execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:968\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:57\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594”] [stack=“main.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:59\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250”]
Error: s3 bucket and region are not matched, bucket=tidb-backup-bucket-bj, input region=BEIJING, real region=us-east-1
, err: exit status 1
E0322 17:37:13.193642 9 backup_status_updater.go:89] Failed to update backup [eeae42ce-c439-442b-863b-0119929a076c/2-1-liyuntang-1679477820], error: Operation cannot be fulfilled on backups.pingcap.com “2-1-liyuntang-1679477820”: the object has been modified; please apply your changes to the latest version and try again
I0322 17:37:13.213332 9 backup_status_updater.go:86] Backup: [eeae42ce-c439-442b-863b-0119929a076c/2-1-liyuntang-1679477820] updated successfully
error: cluster eeae42ce-c439-442b-863b-0119929a076c/2-1-liyuntang-1679477820, wait pipe message failed, errMsg [2023/03/22 17:37:13.180 +08:00] [ERROR] [backup.go:49] [“failed to backup”] [error=“s3 bucket and region are not matched, bucket=tidb-backup-bucket-bj, input region=BEIJING, real region=us-east-1”] [errorVerbose=“s3 bucket and region are not matched, bucket=tidb-backup-bucket-bj, input region=BEIJING, real region=us-east-1\ngithub.com/pingcap/errors.AddStack\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20220729040631-518f63d66278/errors.go:174\ngithub.com/pingcap/errors.Trace\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20220729040631-518f63d66278/juju_adaptor.go:15\ngithub.com/pingcap/tidb/br/pkg/storage.NewS3Storage\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/storage/s3.go:377\ngithub.com/pingcap/tidb/br/pkg/storage.New\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/storage/storage.go:176\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).SetStorage\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:385\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).SetStorageAndCheckNotInUse\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:222\ngithub.com/pingcap/tidb/br/pkg/task.RunBackup\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/task/backup.go:387\nmain.runBackupCommand\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:48\nmain.newFullBackupCommand.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:117\ngithub.com/spf13/cobra.(*Command).execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:916\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:1044\ngithub.com/spf13/cobra.(*Command).Execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:968\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:57\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594”] [stack=“main.runBackupCommand\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:49\nmain.newFullBackupCommand.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:117\ngithub.com/spf13/cobra.(*Command).execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:916\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:1044\ngithub.com/spf13/cobra.(*Command).Execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:968\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:57\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250”]
[2023/03/22 17:37:13.180 +08:00] [ERROR] [main.go:59] [“br failed”] [error=“s3 bucket and region are not matched, bucket=tidb-backup-bucket-bj, input region=BEIJING, real region=us-east-1”] [errorVerbose=“s3 bucket and region are not matched, bucket=tidb-backup-bucket-bj, input region=BEIJING, real region=us-east-1\ngithub.com/pingcap/errors.AddStack\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20220729040631-518f63d66278/errors.go:174\ngithub.com/pingcap/errors.Trace\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20220729040631-518f63d66278/juju_adaptor.go:15\ngithub.com/pingcap/tidb/br/pkg/storage.NewS3Storage\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/storage/s3.go:377\ngithub.com/pingcap/tidb/br/pkg/storage.New\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/storage/storage.go:176\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).SetStorage\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:385\ngithub.com/pingcap/tidb/br/pkg/backup.(*Client).SetStorageAndCheckNotInUse\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/backup/client.go:222\ngithub.com/pingcap/tidb/br/pkg/task.RunBackup\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/task/backup.go:387\nmain.runBackupCommand\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:48\nmain.newFullBackupCommand.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/backup.go:117\ngithub.com/spf13/cobra.(*Command).execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:916\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:1044\ngithub.com/spf13/cobra.(*Command).Execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:968\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:57\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594”] [stack=“main.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/cmd/br/main.go:59\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250”]
Error: s3 bucket and region are not matched, bucket=tidb-backup-bucket-bj, input region=BEIJING, real region=us-east-1
, err: exit status 1
