diff --git a/CHANGELOG.md b/CHANGELOG.md index c2a778363..e7cc465fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ Important Changes in 0.X.Y `--limit-download` flags. https://github.com/restic/restic/issues/1216 https://github.com/restic/restic/pull/1336 + https://github.com/restic/restic/pull/1358 * Failed backend requests are now automatically retried. https://github.com/restic/restic/pull/1353 diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index ca0164527..1141497a1 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -214,10 +214,37 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err debug.Log("InsertObject(%v, %v)", be.bucketName, objName) + // Set chunk size to zero to disable resumable uploads. + // + // With a non-zero chunk size (the default is + // googleapi.DefaultUploadChunkSize, 8MB), Insert will buffer data from + // rd in chunks of this size so it can upload these chunks in + // individual requests. + // + // This chunking allows the library to automatically handle network + // interruptions and re-upload only the last chunk rather than the full + // file. + // + // Unfortunately, this buffering doesn't play nicely with + // --limit-upload, which applies a rate limit to rd. This rate limit + // ends up only limiting the read from rd into the buffer rather than + // the network traffic itself. This results in poor network rate limit + // behavior, where individual chunks are written to the network at full + // bandwidth for several seconds, followed by several seconds of no + // network traffic as the next chunk is read through the rate limiter. + // + // By disabling chunking, rd is passed further down the request stack, + // where there is less (but some) buffering, which ultimately results + // in better rate limiting behavior. + // + // restic typically writes small blobs (4MB-30MB), so the resumable + // uploads are not providing significant benefit anyways. + cs := googleapi.ChunkSize(0) + info, err := be.service.Objects.Insert(be.bucketName, &storage.Object{ Name: objName, - }).Media(rd).Do() + }).Media(rd, cs).Do() be.sem.ReleaseToken()