因为我用的是7.5.4,在不考虑更改tikv本身region的大小下,观察源码发现共有两种解法
方法一:
更改 MetaFileSize 小于 100 M,重新编译br,进行备份,这个已经测试成功
方法二:
观察源码发现,v7.5.4,硬编码了分块上传的块大小为 5M,但是这个 5M 只有在 option.Concurrency > 1 的情况才起作用,而我的备份设置了ratelimit,ratelimit 与 option.Concurrency 是互斥的,只要设置了ratelimit就是单线程备份,所以这个5M的分块无法限制,第二种方法就是去掉ratelimit,这种方法今天会进行验证
var hardcodedS3ChunkSize = 5 * 1024 * 1024
// Create creates multi upload request.
func (rs *S3Storage) Create(ctx context.Context, name string, option *WriterOption) (ExternalFileWriter, error) {
var uploader ExternalFileWriter
var err error
if option == nil || option.Concurrency <= 1 {
uploader, err = rs.createUploader(ctx, name)
if err != nil {
return nil, err
}
} else {
// NewUploaderWithClient:https://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3manager/#NewUploaderWithClient
up := s3manager.NewUploaderWithClient(rs.svc, func(u *s3manager.Uploader) {
u.Concurrency = option.Concurrency
u.BufferProvider = s3manager.NewBufferedReadSeekerWriteToPool(option.Concurrency * hardcodedS3ChunkSize)
})
