backends: add sanity check for the uploaded file size

Bugs in the error handling while uploading a file to the backend could
cause incomplete files, e.g. https://github.com/golang/go/issues/42400
which could affect the local backend.

Proactively add sanity checks which will treat an upload as failed if
the reported upload size does not match the actual file size.

internal/backend/azure/azure.go

@@ -172,6 +172,7 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
 	// read the data, in 100 MiB chunks
 	buf := make([]byte, 100*1024*1024)
 	var blocks []storage.Block
+	uploadedBytes := 0
 
 	for {
 		n, err := io.ReadFull(rd, buf)
@@ -188,6 +189,7 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
 		}
 
 		buf = buf[:n]
+		uploadedBytes += n
 
 		// upload it as a new "block", use the base64 hash for the ID
 		h := restic.Hash(buf)
@@ -204,6 +206,11 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
 		})
 	}
 
+	// sanity check
+	if uploadedBytes != int(rd.Length()) {
+		return errors.Errorf("wrote %d bytes instead of the expected %d bytes", uploadedBytes, rd.Length())
+	}
+
 	debug.Log("uploaded %d parts: %v", len(blocks), blocks)
 	err = file.PutBlockList(blocks, nil)
 	debug.Log("PutBlockList returned %v", err)

internal/backend/b2/b2.go

@@ -209,6 +209,10 @@ func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.Rewind
 		return errors.Wrap(err, "Copy")
 	}
 
+	// sanity check
+	if n != rd.Length() {
+		return errors.Errorf("wrote %d bytes instead of the expected %d bytes", n, rd.Length())
+	}
 	return errors.Wrap(w.Close(), "Close")
 }
 

internal/backend/gs/gs.go

@@ -245,6 +245,10 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
 	}
 
 	debug.Log("%v -> %v bytes", objName, wbytes)
+	// sanity check
+	if wbytes != rd.Length() {
+		return errors.Errorf("wrote %d bytes instead of the expected %d bytes", wbytes, rd.Length())
+	}
 	return nil
 }
 

internal/backend/local/local.go

@@ -118,11 +118,16 @@ func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReade
 	}
 
 	// save data, then sync
-	_, err = io.Copy(f, rd)
+	wbytes, err := io.Copy(f, rd)
 	if err != nil {
 		_ = f.Close()
 		return errors.Wrap(err, "Write")
 	}
+	// sanity check
+	if wbytes != rd.Length() {
+		_ = f.Close()
+		return errors.Errorf("wrote %d bytes instead of the expected %d bytes", wbytes, rd.Length())
+	}
 
 	if err = f.Sync(); err != nil {
 		pathErr, ok := err.(*os.PathError)

internal/backend/mem/mem_backend.go

@@ -84,6 +84,11 @@ func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re
 	be.data[h] = buf
 	debug.Log("saved %v bytes at %v", len(buf), h)
 
+	// sanity check
+	if int64(len(buf)) != rd.Length() {
+		return errors.Errorf("wrote %d bytes instead of the expected %d bytes", len(buf), rd.Length())
+	}
+
 	return ctx.Err()
 }
 

internal/backend/s3/s3.go

@@ -272,9 +272,14 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
 	opts.ContentType = "application/octet-stream"
 
 	debug.Log("PutObject(%v, %v, %v)", be.cfg.Bucket, objName, rd.Length())
-	n, err := be.client.PutObject(ctx, be.cfg.Bucket, objName, ioutil.NopCloser(rd), int64(rd.Length()), opts)
+	info, err := be.client.PutObject(ctx, be.cfg.Bucket, objName, ioutil.NopCloser(rd), int64(rd.Length()), opts)
 
-	debug.Log("%v -> %v bytes, err %#v: %v", objName, n, err, err)
+	debug.Log("%v -> %v bytes, err %#v: %v", objName, info.Size, err, err)
+
+	// sanity check
+	if err != nil && info.Size != rd.Length() {
+		return errors.Errorf("wrote %d bytes instead of the expected %d bytes", info.Size, rd.Length())
+	}
 
 	return errors.Wrap(err, "client.PutObject")
 }

internal/backend/sftp/sftp.go

@@ -288,12 +288,18 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader
 	}
 
 	// save data, make sure to use the optimized sftp upload method
-	_, err = f.ReadFrom(rd)
+	wbytes, err := f.ReadFrom(rd)
 	if err != nil {
 		_ = f.Close()
 		return errors.Wrap(err, "Write")
 	}
 
+	// sanity check
+	if wbytes != rd.Length() {
+		_ = f.Close()
+		return errors.Errorf("wrote %d bytes instead of the expected %d bytes", wbytes, rd.Length())
+	}
+
 	err = f.Close()
 	if err != nil {
 		return errors.Wrap(err, "Close")