From 7048cc3e58f7db03a2c7ac6a107297b2474454b2 Mon Sep 17 00:00:00 2001 From: Pauline Middelink Date: Tue, 13 Aug 2019 18:27:20 +0200 Subject: [PATCH 01/12] Add copy functionality Add a copy command to copy snapshots between repositories. It allows the user to specify a destination repository, password, password-file, password-command or key-hint to supply the necessary details to open the destination repository. You need to supply a list of snapshots to copy, snapshots which already exist in the destination repository will be skipped. Note, when using the network this becomes rather slow, as it needs to read the blocks, decrypt them using the source key, then encrypt them again using the destination key before finally writing them out to the destination repository. --- changelog/unreleased/issue-323 | 13 +++ cmd/restic/cmd_copy.go | 203 +++++++++++++++++++++++++++++++++ cmd/restic/global.go | 4 +- cmd/restic/main.go | 2 +- 4 files changed, 219 insertions(+), 3 deletions(-) create mode 100644 changelog/unreleased/issue-323 create mode 100644 cmd/restic/cmd_copy.go diff --git a/changelog/unreleased/issue-323 b/changelog/unreleased/issue-323 new file mode 100644 index 000000000..6b3b56b31 --- /dev/null +++ b/changelog/unreleased/issue-323 @@ -0,0 +1,13 @@ +Enhancement: Add command for copying snapshots between repositories + +We've added a copy command, allowing you to copy snapshots from one +repository to another. + +Note that this process will have to read (download) and write (upload) the +entire snapshot(s) due to the different encryption keys used on the source +and destination repository. Also, the transferred files are not re-chunked, +which may break deduplication between files already stored in the +destination repo and files copied there using this command. + +https://github.com/restic/restic/issues/323 +https://github.com/restic/restic/pull/2606 diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go new file mode 100644 index 000000000..30545e27b --- /dev/null +++ b/cmd/restic/cmd_copy.go @@ -0,0 +1,203 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" + + "github.com/spf13/cobra" +) + +var cmdCopy = &cobra.Command{ + Use: "copy [flags] [snapshotID ...]", + Short: "Copy snapshots from one repository to another", + Long: ` +The "copy" command copies one or more snapshots from one repository to another +repository. Note that this will have to read (download) and write (upload) the +entire snapshot(s) due to the different encryption keys on the source and +destination, and that transferred files are not re-chunked, which may break +their deduplication. +`, + RunE: func(cmd *cobra.Command, args []string) error { + return runCopy(copyOptions, globalOptions, args) + }, +} + +// CopyOptions bundles all options for the copy command. +type CopyOptions struct { + Repo string + PasswordFile string + PasswordCommand string + KeyHint string + Hosts []string + Tags restic.TagLists + Paths []string +} + +var copyOptions CopyOptions + +func init() { + cmdRoot.AddCommand(cmdCopy) + + f := cmdCopy.Flags() + f.StringVarP(©Options.Repo, "repo2", "", os.Getenv("RESTIC_REPOSITORY2"), "destination repository to copy snapshots to (default: $RESTIC_REPOSITORY2)") + f.StringVarP(©Options.PasswordFile, "password-file2", "", os.Getenv("RESTIC_PASSWORD_FILE2"), "read the destination repository password from a file (default: $RESTIC_PASSWORD_FILE2)") + f.StringVarP(©Options.KeyHint, "key-hint2", "", os.Getenv("RESTIC_KEY_HINT2"), "key ID of key to try decrypting the destination repository first (default: $RESTIC_KEY_HINT2)") + f.StringVarP(©Options.PasswordCommand, "password-command2", "", os.Getenv("RESTIC_PASSWORD_COMMAND2"), "specify a shell command to obtain a password for the destination repository (default: $RESTIC_PASSWORD_COMMAND2)") + + f.StringArrayVarP(©Options.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)") + f.Var(©Options.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given") + f.StringArrayVar(©Options.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given") +} + +func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { + if opts.Repo == "" { + return errors.Fatal("Please specify a destination repository location (--repo2)") + } + var err error + dstGopts := gopts + dstGopts.Repo = opts.Repo + dstGopts.PasswordFile = opts.PasswordFile + dstGopts.PasswordCommand = opts.PasswordCommand + dstGopts.KeyHint = opts.KeyHint + dstGopts.password, err = resolvePassword(dstGopts, "RESTIC_PASSWORD2") + if err != nil { + return err + } + dstGopts.password, err = ReadPassword(dstGopts, "enter password for destination repository: ") + if err != nil { + return err + } + + ctx, cancel := context.WithCancel(gopts.ctx) + defer cancel() + + srcRepo, err := OpenRepository(gopts) + if err != nil { + return err + } + + dstRepo, err := OpenRepository(dstGopts) + if err != nil { + return err + } + + srcLock, err := lockRepo(srcRepo) + defer unlockRepo(srcLock) + if err != nil { + return err + } + + dstLock, err := lockRepo(dstRepo) + defer unlockRepo(dstLock) + if err != nil { + return err + } + + debug.Log("Loading source index") + if err := srcRepo.LoadIndex(ctx); err != nil { + return err + } + + debug.Log("Loading destination index") + if err := dstRepo.LoadIndex(ctx); err != nil { + return err + } + + for sn := range FindFilteredSnapshots(ctx, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) { + Verbosef("snapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) + Verbosef(" copy started, this may take a while...\n") + + if err := copyTree(ctx, srcRepo, dstRepo, *sn.Tree); err != nil { + return err + } + debug.Log("tree copied") + + if err = dstRepo.Flush(ctx); err != nil { + return err + } + debug.Log("flushed packs") + + err = dstRepo.SaveIndex(ctx) + if err != nil { + debug.Log("error saving index: %v", err) + return err + } + debug.Log("saved index") + + // save snapshot + sn.Parent = nil // Parent does not have relevance in the new repo. + sn.Original = nil // Original does not have relevance in the new repo. + newID, err := dstRepo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn) + if err != nil { + return err + } + Verbosef("snapshot %s saved\n", newID.Str()) + } + return nil +} + +func copyTree(ctx context.Context, srcRepo, dstRepo restic.Repository, treeID restic.ID) error { + tree, err := srcRepo.LoadTree(ctx, treeID) + if err != nil { + return fmt.Errorf("LoadTree(%v) returned error %v", treeID.Str(), err) + } + // Do we already have this tree blob? + if !dstRepo.Index().Has(treeID, restic.TreeBlob) { + newTreeID, err := dstRepo.SaveTree(ctx, tree) + if err != nil { + return fmt.Errorf("SaveTree(%v) returned error %v", treeID.Str(), err) + } + // Assurance only. + if newTreeID != treeID { + return fmt.Errorf("SaveTree(%v) returned unexpected id %s", treeID.Str(), newTreeID.Str()) + } + } + + // TODO: keep only one (big) buffer around. + // TODO: parellize this stuff, likely only needed inside a tree. + + for _, entry := range tree.Nodes { + // If it is a directory, recurse + if entry.Type == "dir" && entry.Subtree != nil { + if err := copyTree(ctx, srcRepo, dstRepo, *entry.Subtree); err != nil { + return err + } + } + // Copy the blobs for this file. + for _, blobID := range entry.Content { + // Do we already have this data blob? + if dstRepo.Index().Has(blobID, restic.DataBlob) { + continue + } + debug.Log("Copying blob %s\n", blobID.Str()) + size, found := srcRepo.LookupBlobSize(blobID, restic.DataBlob) + if !found { + return fmt.Errorf("LookupBlobSize(%v) failed", blobID) + } + buf := restic.NewBlobBuffer(int(size)) + n, err := srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, buf) + if err != nil { + return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err) + } + if n != len(buf) { + return fmt.Errorf("wrong number of bytes read, want %d, got %d", len(buf), n) + } + + newBlobID, err := dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID) + if err != nil { + return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err) + } + // Assurance only. + if newBlobID != blobID { + return fmt.Errorf("SaveBlob(%v) returned unexpected id %s", blobID.Str(), newBlobID.Str()) + } + } + } + + return nil +} diff --git a/cmd/restic/global.go b/cmd/restic/global.go index d3b264b3d..bc34b5b98 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -274,7 +274,7 @@ func Exitf(exitcode int, format string, args ...interface{}) { } // resolvePassword determines the password to be used for opening the repository. -func resolvePassword(opts GlobalOptions) (string, error) { +func resolvePassword(opts GlobalOptions, envStr string) (string, error) { if opts.PasswordFile != "" && opts.PasswordCommand != "" { return "", errors.Fatalf("Password file and command are mutually exclusive options") } @@ -299,7 +299,7 @@ func resolvePassword(opts GlobalOptions) (string, error) { return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile") } - if pwd := os.Getenv("RESTIC_PASSWORD"); pwd != "" { + if pwd := os.Getenv(envStr); pwd != "" { return pwd, nil } diff --git a/cmd/restic/main.go b/cmd/restic/main.go index 2f553c9cc..2e75575b3 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -54,7 +54,7 @@ directories in an encrypted repository stored on different backends. if c.Name() == "version" { return nil } - pwd, err := resolvePassword(globalOptions) + pwd, err := resolvePassword(globalOptions, "RESTIC_PASSWORD") if err != nil { fmt.Fprintf(os.Stderr, "Resolving password failed: %v\n", err) Exit(1) From 4508d406efd555b93441eac1a2f5a5e6231ffd53 Mon Sep 17 00:00:00 2001 From: greatroar <@> Date: Fri, 19 Jun 2020 12:15:37 +0200 Subject: [PATCH 02/12] copy: Remove separate SaveIndex in restic copy Flush does this now. --- cmd/restic/cmd_copy.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 30545e27b..10424c938 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -120,14 +120,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { if err = dstRepo.Flush(ctx); err != nil { return err } - debug.Log("flushed packs") - - err = dstRepo.SaveIndex(ctx) - if err != nil { - debug.Log("error saving index: %v", err) - return err - } - debug.Log("saved index") + debug.Log("flushed packs and saved index") // save snapshot sn.Parent = nil // Parent does not have relevance in the new repo. From 908b23fda007a266e48366a642b9aec9b7fdff5f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 22 Aug 2020 12:14:42 +0200 Subject: [PATCH 03/12] copy: Update for modernized repository interface --- cmd/restic/cmd_copy.go | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 10424c938..cbc29a1b3 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -168,27 +168,15 @@ func copyTree(ctx context.Context, srcRepo, dstRepo restic.Repository, treeID re continue } debug.Log("Copying blob %s\n", blobID.Str()) - size, found := srcRepo.LookupBlobSize(blobID, restic.DataBlob) - if !found { - return fmt.Errorf("LookupBlobSize(%v) failed", blobID) - } - buf := restic.NewBlobBuffer(int(size)) - n, err := srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, buf) + buf, err := srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, nil) if err != nil { return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err) } - if n != len(buf) { - return fmt.Errorf("wrong number of bytes read, want %d, got %d", len(buf), n) - } - newBlobID, err := dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID) + _, _, err = dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID, false) if err != nil { return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err) } - // Assurance only. - if newBlobID != blobID { - return fmt.Errorf("SaveBlob(%v) returned unexpected id %s", blobID.Str(), newBlobID.Str()) - } } } From b0a8c4ad6c5b4ec4b619d1a1af8380669aacb22c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 22 Aug 2020 12:22:00 +0200 Subject: [PATCH 04/12] copy: Only process each tree once This speeds up copying multiple overlapping snapshots from one repository to another, as we only have to copy the changed parts of later snapshots. --- cmd/restic/cmd_copy.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index cbc29a1b3..9fbc19028 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -108,11 +108,13 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { return err } + visitedTrees := restic.NewIDSet() + for sn := range FindFilteredSnapshots(ctx, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) { Verbosef("snapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) Verbosef(" copy started, this may take a while...\n") - if err := copyTree(ctx, srcRepo, dstRepo, *sn.Tree); err != nil { + if err := copyTree(ctx, srcRepo, dstRepo, *sn.Tree, visitedTrees); err != nil { return err } debug.Log("tree copied") @@ -134,11 +136,18 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { return nil } -func copyTree(ctx context.Context, srcRepo, dstRepo restic.Repository, treeID restic.ID) error { +func copyTree(ctx context.Context, srcRepo, dstRepo restic.Repository, treeID restic.ID, visitedTrees restic.IDSet) error { + // We have already processed this tree + if visitedTrees.Has(treeID) { + return nil + } + tree, err := srcRepo.LoadTree(ctx, treeID) if err != nil { return fmt.Errorf("LoadTree(%v) returned error %v", treeID.Str(), err) } + visitedTrees.Insert(treeID) + // Do we already have this tree blob? if !dstRepo.Index().Has(treeID, restic.TreeBlob) { newTreeID, err := dstRepo.SaveTree(ctx, tree) @@ -157,7 +166,7 @@ func copyTree(ctx context.Context, srcRepo, dstRepo restic.Repository, treeID re for _, entry := range tree.Nodes { // If it is a directory, recurse if entry.Type == "dir" && entry.Subtree != nil { - if err := copyTree(ctx, srcRepo, dstRepo, *entry.Subtree); err != nil { + if err := copyTree(ctx, srcRepo, dstRepo, *entry.Subtree, visitedTrees); err != nil { return err } } From ec9a53b7e8df6ac22007f9df28fc2fbf41019bcd Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 22 Aug 2020 16:09:08 +0200 Subject: [PATCH 05/12] copy: Mark and skip previously copied snapshots Use the `Original` field of the copied snapshot to store a persistent snapshot ID. This can either be the ID of the source snapshot if `Original` was not yet set or the previous value stored in the `Original` field. In order to still copy snapshots modified using the tags command the source snapshot is compared to all snapshots in the destination repository which have the same persistent ID. Snapshots are only considered equal if all fields except `Original` and `Parent` match. That way modified snapshots are still copied while avoiding duplicate copies at the same time. --- cmd/restic/cmd_copy.go | 55 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 9fbc19028..7c2752792 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -109,9 +109,36 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { } visitedTrees := restic.NewIDSet() + dstSnapshotByOriginal := make(map[restic.ID][]*restic.Snapshot) + for sn := range FindFilteredSnapshots(ctx, dstRepo, opts.Hosts, opts.Tags, opts.Paths, nil) { + if sn.Original != nil && !sn.Original.IsNull() { + dstSnapshotByOriginal[*sn.Original] = append(dstSnapshotByOriginal[*sn.Original], sn) + } + // also consider identical snapshot copies + dstSnapshotByOriginal[*sn.ID()] = append(dstSnapshotByOriginal[*sn.ID()], sn) + } for sn := range FindFilteredSnapshots(ctx, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) { - Verbosef("snapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) + Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) + + // check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields + srcOriginal := *sn.ID() + if sn.Original != nil { + srcOriginal = *sn.Original + } + if originalSns, ok := dstSnapshotByOriginal[srcOriginal]; ok { + isCopy := false + for _, originalSn := range originalSns { + if similarSnapshots(originalSn, sn) { + Verbosef("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str()) + isCopy = true + break + } + } + if isCopy { + continue + } + } Verbosef(" copy started, this may take a while...\n") if err := copyTree(ctx, srcRepo, dstRepo, *sn.Tree, visitedTrees); err != nil { @@ -125,8 +152,11 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { debug.Log("flushed packs and saved index") // save snapshot - sn.Parent = nil // Parent does not have relevance in the new repo. - sn.Original = nil // Original does not have relevance in the new repo. + sn.Parent = nil // Parent does not have relevance in the new repo. + // Use Original as a persistent snapshot ID + if sn.Original == nil { + sn.Original = sn.ID() + } newID, err := dstRepo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn) if err != nil { return err @@ -136,6 +166,25 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { return nil } +func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool { + // everything except Parent and Original must match + if !sna.Time.Equal(snb.Time) || !sna.Tree.Equal(*snb.Tree) || sna.Hostname != snb.Hostname || + sna.Username != snb.Username || sna.UID != snb.UID || sna.GID != snb.GID || + len(sna.Paths) != len(snb.Paths) || len(sna.Excludes) != len(snb.Excludes) || + len(sna.Tags) != len(snb.Tags) { + return false + } + if !sna.HasPaths(snb.Paths) || !sna.HasTags(snb.Tags) { + return false + } + for i, a := range sna.Excludes { + if a != snb.Excludes[i] { + return false + } + } + return true +} + func copyTree(ctx context.Context, srcRepo, dstRepo restic.Repository, treeID restic.ID, visitedTrees restic.IDSet) error { // We have already processed this tree if visitedTrees.Has(treeID) { From 591a8c4cdf1e1ad931a15ac24531b5cb83286b0d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 22 Aug 2020 20:00:16 +0200 Subject: [PATCH 06/12] integration tests: Deduplicate backup test-data setup code --- cmd/restic/integration_test.go | 85 +++++++--------------------------- 1 file changed, 16 insertions(+), 69 deletions(-) diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 1267db0a0..f162fb634 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -260,22 +260,18 @@ func testRunPrune(t testing.TB, gopts GlobalOptions) { rtest.OK(t, runPrune(gopts)) } +func testSetupBackupData(t testing.TB, env *testEnvironment) string { + datafile := filepath.Join("testdata", "backup-data.tar.gz") + testRunInit(t, env.gopts) + rtest.SetupTarTestFixture(t, env.testdata, datafile) + return datafile +} + func TestBackup(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() - datafile := filepath.Join("testdata", "backup-data.tar.gz") - fd, err := os.Open(datafile) - if os.IsNotExist(errors.Cause(err)) { - t.Skipf("unable to find data file %q, skipping", datafile) - return - } - rtest.OK(t, err) - rtest.OK(t, fd.Close()) - - testRunInit(t, env.gopts) - - rtest.SetupTarTestFixture(t, env.testdata, datafile) + testSetupBackupData(t, env) opts := BackupOptions{} // first backup @@ -329,18 +325,7 @@ func TestBackupNonExistingFile(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() - datafile := filepath.Join("testdata", "backup-data.tar.gz") - fd, err := os.Open(datafile) - if os.IsNotExist(errors.Cause(err)) { - t.Skipf("unable to find data file %q, skipping", datafile) - return - } - rtest.OK(t, err) - rtest.OK(t, fd.Close()) - - rtest.SetupTarTestFixture(t, env.testdata, datafile) - - testRunInit(t, env.gopts) + testSetupBackupData(t, env) globalOptions.stderr = ioutil.Discard defer func() { globalOptions.stderr = os.Stderr @@ -503,11 +488,7 @@ func TestBackupErrors(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() - datafile := filepath.Join("testdata", "backup-data.tar.gz") - - rtest.SetupTarTestFixture(t, env.testdata, datafile) - - testRunInit(t, env.gopts) + testSetupBackupData(t, env) // Assume failure inaccessibleFile := filepath.Join(env.testdata, "0", "0", "9", "0") @@ -596,10 +577,7 @@ func TestBackupTags(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() - datafile := filepath.Join("testdata", "backup-data.tar.gz") - testRunInit(t, env.gopts) - rtest.SetupTarTestFixture(t, env.testdata, datafile) - + testSetupBackupData(t, env) opts := BackupOptions{} testRunBackup(t, "", []string{env.testdata}, opts, env.gopts) @@ -630,10 +608,7 @@ func TestTag(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() - datafile := filepath.Join("testdata", "backup-data.tar.gz") - testRunInit(t, env.gopts) - rtest.SetupTarTestFixture(t, env.testdata, datafile) - + testSetupBackupData(t, env) testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts) testRunCheck(t, env.gopts) newest, _ := testRunSnapshots(t, env.gopts) @@ -1033,10 +1008,7 @@ func TestFind(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() - datafile := filepath.Join("testdata", "backup-data.tar.gz") - testRunInit(t, env.gopts) - rtest.SetupTarTestFixture(t, env.testdata, datafile) - + datafile := testSetupBackupData(t, env) opts := BackupOptions{} testRunBackup(t, "", []string{env.testdata}, opts, env.gopts) @@ -1073,10 +1045,7 @@ func TestFindJSON(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() - datafile := filepath.Join("testdata", "backup-data.tar.gz") - testRunInit(t, env.gopts) - rtest.SetupTarTestFixture(t, env.testdata, datafile) - + datafile := testSetupBackupData(t, env) opts := BackupOptions{} testRunBackup(t, "", []string{env.testdata}, opts, env.gopts) @@ -1199,18 +1168,7 @@ func TestPrune(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() - datafile := filepath.Join("testdata", "backup-data.tar.gz") - fd, err := os.Open(datafile) - if os.IsNotExist(errors.Cause(err)) { - t.Skipf("unable to find data file %q, skipping", datafile) - return - } - rtest.OK(t, err) - rtest.OK(t, fd.Close()) - - testRunInit(t, env.gopts) - - rtest.SetupTarTestFixture(t, env.testdata, datafile) + testSetupBackupData(t, env) opts := BackupOptions{} testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts) @@ -1385,18 +1343,7 @@ func TestQuietBackup(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() - datafile := filepath.Join("testdata", "backup-data.tar.gz") - fd, err := os.Open(datafile) - if os.IsNotExist(errors.Cause(err)) { - t.Skipf("unable to find data file %q, skipping", datafile) - return - } - rtest.OK(t, err) - rtest.OK(t, fd.Close()) - - testRunInit(t, env.gopts) - - rtest.SetupTarTestFixture(t, env.testdata, datafile) + testSetupBackupData(t, env) opts := BackupOptions{} env.gopts.Quiet = false From 88ad58d6cdc1684b72409353961aaab80cf56af2 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 22 Aug 2020 20:39:03 +0200 Subject: [PATCH 07/12] integration tests: Redirect directory diff into intermediate buffer --- cmd/restic/integration_helpers_test.go | 33 +++++++------------ cmd/restic/integration_helpers_unix_test.go | 15 +++++---- .../integration_helpers_windows_test.go | 9 ++--- cmd/restic/integration_test.go | 12 +++---- 4 files changed, 31 insertions(+), 38 deletions(-) diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index 136ec6fe0..0069e27a9 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "fmt" "io/ioutil" @@ -75,14 +76,13 @@ func sameModTime(fi1, fi2 os.FileInfo) bool { return fi1.ModTime().Equal(fi2.ModTime()) } -// directoriesEqualContents checks if both directories contain exactly the same -// contents. -func directoriesEqualContents(dir1, dir2 string) bool { +// directoriesContentsDiff returns a diff between both directories. If these +// contain exactly the same contents, then the diff is an empty string. +func directoriesContentsDiff(dir1, dir2 string) string { + var out bytes.Buffer ch1 := walkDir(dir1) ch2 := walkDir(dir2) - changes := false - var a, b *dirEntry for { var ok bool @@ -106,36 +106,27 @@ func directoriesEqualContents(dir1, dir2 string) bool { } if ch1 == nil { - fmt.Printf("+%v\n", b.path) - changes = true + fmt.Fprintf(&out, "+%v\n", b.path) } else if ch2 == nil { - fmt.Printf("-%v\n", a.path) - changes = true - } else if !a.equals(b) { + fmt.Fprintf(&out, "-%v\n", a.path) + } else if !a.equals(&out, b) { if a.path < b.path { - fmt.Printf("-%v\n", a.path) - changes = true + fmt.Fprintf(&out, "-%v\n", a.path) a = nil continue } else if a.path > b.path { - fmt.Printf("+%v\n", b.path) - changes = true + fmt.Fprintf(&out, "+%v\n", b.path) b = nil continue } else { - fmt.Printf("%%%v\n", a.path) - changes = true + fmt.Fprintf(&out, "%%%v\n", a.path) } } a, b = nil, nil } - if changes { - return false - } - - return true + return out.String() } type dirStat struct { diff --git a/cmd/restic/integration_helpers_unix_test.go b/cmd/restic/integration_helpers_unix_test.go index 2a06db63d..1130d2638 100644 --- a/cmd/restic/integration_helpers_unix_test.go +++ b/cmd/restic/integration_helpers_unix_test.go @@ -4,25 +4,26 @@ package main import ( "fmt" + "io" "io/ioutil" "os" "path/filepath" "syscall" ) -func (e *dirEntry) equals(other *dirEntry) bool { +func (e *dirEntry) equals(out io.Writer, other *dirEntry) bool { if e.path != other.path { - fmt.Fprintf(os.Stderr, "%v: path does not match (%v != %v)\n", e.path, e.path, other.path) + fmt.Fprintf(out, "%v: path does not match (%v != %v)\n", e.path, e.path, other.path) return false } if e.fi.Mode() != other.fi.Mode() { - fmt.Fprintf(os.Stderr, "%v: mode does not match (%v != %v)\n", e.path, e.fi.Mode(), other.fi.Mode()) + fmt.Fprintf(out, "%v: mode does not match (%v != %v)\n", e.path, e.fi.Mode(), other.fi.Mode()) return false } if !sameModTime(e.fi, other.fi) { - fmt.Fprintf(os.Stderr, "%v: ModTime does not match (%v != %v)\n", e.path, e.fi.ModTime(), other.fi.ModTime()) + fmt.Fprintf(out, "%v: ModTime does not match (%v != %v)\n", e.path, e.fi.ModTime(), other.fi.ModTime()) return false } @@ -30,17 +31,17 @@ func (e *dirEntry) equals(other *dirEntry) bool { stat2, _ := other.fi.Sys().(*syscall.Stat_t) if stat.Uid != stat2.Uid { - fmt.Fprintf(os.Stderr, "%v: UID does not match (%v != %v)\n", e.path, stat.Uid, stat2.Uid) + fmt.Fprintf(out, "%v: UID does not match (%v != %v)\n", e.path, stat.Uid, stat2.Uid) return false } if stat.Gid != stat2.Gid { - fmt.Fprintf(os.Stderr, "%v: GID does not match (%v != %v)\n", e.path, stat.Gid, stat2.Gid) + fmt.Fprintf(out, "%v: GID does not match (%v != %v)\n", e.path, stat.Gid, stat2.Gid) return false } if stat.Nlink != stat2.Nlink { - fmt.Fprintf(os.Stderr, "%v: Number of links do not match (%v != %v)\n", e.path, stat.Nlink, stat2.Nlink) + fmt.Fprintf(out, "%v: Number of links do not match (%v != %v)\n", e.path, stat.Nlink, stat2.Nlink) return false } diff --git a/cmd/restic/integration_helpers_windows_test.go b/cmd/restic/integration_helpers_windows_test.go index 9e3fbac9b..85285efb7 100644 --- a/cmd/restic/integration_helpers_windows_test.go +++ b/cmd/restic/integration_helpers_windows_test.go @@ -4,23 +4,24 @@ package main import ( "fmt" + "io" "io/ioutil" "os" ) -func (e *dirEntry) equals(other *dirEntry) bool { +func (e *dirEntry) equals(out io.Writer, other *dirEntry) bool { if e.path != other.path { - fmt.Fprintf(os.Stderr, "%v: path does not match (%v != %v)\n", e.path, e.path, other.path) + fmt.Fprintf(out, "%v: path does not match (%v != %v)\n", e.path, e.path, other.path) return false } if e.fi.Mode() != other.fi.Mode() { - fmt.Fprintf(os.Stderr, "%v: mode does not match (%v != %v)\n", e.path, e.fi.Mode(), other.fi.Mode()) + fmt.Fprintf(out, "%v: mode does not match (%v != %v)\n", e.path, e.fi.Mode(), other.fi.Mode()) return false } if !sameModTime(e.fi, other.fi) { - fmt.Fprintf(os.Stderr, "%v: ModTime does not match (%v != %v)\n", e.path, e.fi.ModTime(), other.fi.ModTime()) + fmt.Fprintf(out, "%v: ModTime does not match (%v != %v)\n", e.path, e.fi.ModTime(), other.fi.ModTime()) return false } diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index f162fb634..43cf8ed41 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -314,8 +314,8 @@ func TestBackup(t *testing.T) { restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i)) t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir) testRunRestore(t, env.gopts, restoredir, snapshotIDs[0]) - rtest.Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, "testdata")), - "directories are not equal") + diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, "testdata")) + rtest.Assert(t, diff == "", "directories are not equal: %v", diff) } testRunCheck(t, env.gopts) @@ -856,8 +856,8 @@ func TestRestore(t *testing.T) { restoredir := filepath.Join(env.base, "restore") testRunRestoreLatest(t, env.gopts, restoredir, nil, nil) - rtest.Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, filepath.Base(env.testdata))), - "directories are not equal") + diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, filepath.Base(env.testdata))) + rtest.Assert(t, diff == "", "directories are not equal %v", diff) } func TestRestoreLatest(t *testing.T) { @@ -1276,8 +1276,8 @@ func TestHardLink(t *testing.T) { restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i)) t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir) testRunRestore(t, env.gopts, restoredir, snapshotIDs[0]) - rtest.Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, "testdata")), - "directories are not equal") + diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, "testdata")) + rtest.Assert(t, diff == "", "directories are not equal %v", diff) linkResults := createFileSetPerHardlink(filepath.Join(restoredir, "testdata")) rtest.Assert(t, linksEqual(linkTests, linkResults), From 15374d22e9ec7c78e772f13d2cbbff2e0f799291 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 22 Aug 2020 20:17:31 +0200 Subject: [PATCH 08/12] integration tests: Add basic tests for copy command --- cmd/restic/cmd_copy.go | 11 +++- cmd/restic/integration_test.go | 115 +++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 3 deletions(-) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 7c2752792..be2c67356 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -30,6 +30,7 @@ their deduplication. // CopyOptions bundles all options for the copy command. type CopyOptions struct { Repo string + password string PasswordFile string PasswordCommand string KeyHint string @@ -64,9 +65,13 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { dstGopts.PasswordFile = opts.PasswordFile dstGopts.PasswordCommand = opts.PasswordCommand dstGopts.KeyHint = opts.KeyHint - dstGopts.password, err = resolvePassword(dstGopts, "RESTIC_PASSWORD2") - if err != nil { - return err + if opts.password != "" { + dstGopts.password = opts.password + } else { + dstGopts.password, err = resolvePassword(dstGopts, "RESTIC_PASSWORD2") + if err != nil { + return err + } } dstGopts.password, err = ReadPassword(dstGopts, "enter password for destination repository: ") if err != nil { diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 43cf8ed41..36ce9e487 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -600,6 +600,121 @@ func TestBackupTags(t *testing.T) { "expected parent to be %v, got %v", parent.ID, newest.Parent) } +func testRunCopy(t testing.TB, srcGopts GlobalOptions, dstGopts GlobalOptions) { + copyOpts := CopyOptions{ + Repo: dstGopts.Repo, + password: dstGopts.password, + } + + rtest.OK(t, runCopy(copyOpts, srcGopts, nil)) +} + +func TestCopy(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + env2, cleanup2 := withTestEnvironment(t) + defer cleanup2() + + testSetupBackupData(t, env) + opts := BackupOptions{} + testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts) + testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "2")}, opts, env.gopts) + testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts) + testRunCheck(t, env.gopts) + + testRunInit(t, env2.gopts) + testRunCopy(t, env.gopts, env2.gopts) + + snapshotIDs := testRunList(t, "snapshots", env.gopts) + copiedSnapshotIDs := testRunList(t, "snapshots", env2.gopts) + + // Check that the copies size seems reasonable + rtest.Assert(t, len(snapshotIDs) == len(copiedSnapshotIDs), "expected %v snapshots, found %v", + len(snapshotIDs), len(copiedSnapshotIDs)) + stat := dirStats(env.repo) + stat2 := dirStats(env2.repo) + sizeDiff := int64(stat.size) - int64(stat2.size) + if sizeDiff < 0 { + sizeDiff = -sizeDiff + } + rtest.Assert(t, sizeDiff < int64(stat.size)/50, "expected less than 2%% size difference: %v vs. %v", + stat.size, stat2.size) + + // Check integrity of the copy + testRunCheck(t, env2.gopts) + + // Check that the copied snapshots have the same tree contents as the old ones (= identical tree hash) + origRestores := make(map[string]struct{}) + for i, snapshotID := range snapshotIDs { + restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i)) + origRestores[restoredir] = struct{}{} + testRunRestore(t, env.gopts, restoredir, snapshotID) + } + for i, snapshotID := range copiedSnapshotIDs { + restoredir := filepath.Join(env2.base, fmt.Sprintf("restore%d", i)) + testRunRestore(t, env2.gopts, restoredir, snapshotID) + foundMatch := false + for cmpdir := range origRestores { + diff := directoriesContentsDiff(restoredir, cmpdir) + if diff == "" { + delete(origRestores, cmpdir) + foundMatch = true + } + } + + rtest.Assert(t, foundMatch, "found no counterpart for snapshot %v", snapshotID) + } + + rtest.Assert(t, len(origRestores) == 0, "found not copied snapshots") +} + +func TestCopyIncremental(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + env2, cleanup2 := withTestEnvironment(t) + defer cleanup2() + + testSetupBackupData(t, env) + opts := BackupOptions{} + testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts) + testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "2")}, opts, env.gopts) + testRunCheck(t, env.gopts) + + testRunInit(t, env2.gopts) + testRunCopy(t, env.gopts, env2.gopts) + + snapshotIDs := testRunList(t, "snapshots", env.gopts) + copiedSnapshotIDs := testRunList(t, "snapshots", env2.gopts) + + // Check that the copies size seems reasonable + testRunCheck(t, env2.gopts) + rtest.Assert(t, len(snapshotIDs) == len(copiedSnapshotIDs), "expected %v snapshots, found %v", + len(snapshotIDs), len(copiedSnapshotIDs)) + + // check that no snapshots are copied, as there are no new ones + testRunCopy(t, env.gopts, env2.gopts) + testRunCheck(t, env2.gopts) + copiedSnapshotIDs = testRunList(t, "snapshots", env2.gopts) + rtest.Assert(t, len(snapshotIDs) == len(copiedSnapshotIDs), "still expected %v snapshots, found %v", + len(snapshotIDs), len(copiedSnapshotIDs)) + + // check that only new snapshots are copied + testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts) + testRunCopy(t, env.gopts, env2.gopts) + testRunCheck(t, env2.gopts) + snapshotIDs = testRunList(t, "snapshots", env.gopts) + copiedSnapshotIDs = testRunList(t, "snapshots", env2.gopts) + rtest.Assert(t, len(snapshotIDs) == len(copiedSnapshotIDs), "still expected %v snapshots, found %v", + len(snapshotIDs), len(copiedSnapshotIDs)) + + // also test the reverse direction + testRunCopy(t, env2.gopts, env.gopts) + testRunCheck(t, env.gopts) + snapshotIDs = testRunList(t, "snapshots", env.gopts) + rtest.Assert(t, len(snapshotIDs) == len(copiedSnapshotIDs), "still expected %v snapshots, found %v", + len(copiedSnapshotIDs), len(snapshotIDs)) +} + func testRunTag(t testing.TB, opts TagOptions, gopts GlobalOptions) { rtest.OK(t, runTag(opts, gopts, []string{})) } From 9a4796594a51a6318697fef44de5e13ac8a53d7c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 22 Aug 2020 20:45:22 +0200 Subject: [PATCH 09/12] integration tests: Fix checking of wrong snapshot --- cmd/restic/integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 36ce9e487..74661ca1c 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -313,7 +313,7 @@ func TestBackup(t *testing.T) { for i, snapshotID := range snapshotIDs { restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i)) t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir) - testRunRestore(t, env.gopts, restoredir, snapshotIDs[0]) + testRunRestore(t, env.gopts, restoredir, snapshotID) diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, "testdata")) rtest.Assert(t, diff == "", "directories are not equal: %v", diff) } @@ -1390,7 +1390,7 @@ func TestHardLink(t *testing.T) { for i, snapshotID := range snapshotIDs { restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i)) t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir) - testRunRestore(t, env.gopts, restoredir, snapshotIDs[0]) + testRunRestore(t, env.gopts, restoredir, snapshotID) diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, "testdata")) rtest.Assert(t, diff == "", "directories are not equal %v", diff) From 91e8d998cd1c95cb33addcfd700f85cec16b537d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 24 Aug 2020 23:15:16 +0200 Subject: [PATCH 10/12] Add documentation for copy command --- doc/045_working_with_repos.rst | 60 ++++++++++++++++++++++++++++++++++ doc/manual_rest.rst | 1 + 2 files changed, 61 insertions(+) diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index 74df2c3bc..58573e3d6 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -82,6 +82,66 @@ Furthermore you can group the output by the same filters (host, paths, tags): 1 snapshots +Copying snapshots between repositories +====================================== + +In case you want to transfer snapshots between two repositories, for +example from a local to a remote repository, you can use the ``copy`` command: + +.. code-block:: console + + $ restic -r /srv/restic-repo copy --repo2 /srv/restic-repo-copy + repository d6504c63 opened successfully, password is correct + repository 3dd0878c opened successfully, password is correct + + snapshot 410b18a2 of [/home/user/work] at 2020-06-09 23:15:57.305305 +0200 CEST) + copy started, this may take a while... + snapshot 7a746a07 saved + + snapshot 4e5d5487 of [/home/user/work] at 2020-05-01 22:44:07.012113 +0200 CEST) + skipping snapshot 4e5d5487, was already copied to snapshot 50eb62b7 + +The example command copies all snapshots from the source repository +``/srv/restic-repo`` to the destination repository ``/srv/restic-repo-copy``. +Snapshots which have previously been copied between repositories will +be skipped by later copy runs. + +.. note:: Note that this process will have to read (download) and write (upload) the + entire snapshot(s) due to the different encryption keys used in the source and + destination repository. Also, the transferred files are not re-chunked, which + may break deduplication between files already stored in the destination repo + and files copied there using this command. + +For the destination repository ``--repo2`` the password can be read from +a file ``--password-file2`` or from a command ``--password-command2``. +Alternatively the environment variables ``$RESTIC_PASSWORD_COMMAND2`` and +``$RESTIC_PASSWORD_FILE2`` can be used. It is also possible to directly +pass the password via ``$RESTIC_PASSWORD2``. The key which should be used +for decryption can be selected by passing its ID via the flag ``--key-hint2`` +or the environment variable ``$RESTIC_KEY_HINT2``. + +In case the source and destination repository use the same backend, then +configuration options and environment variables to configure the backend +apply to both repositories. For example it might not be possible to specify +different accounts for the source and destination repository. You can +avoid this limitation by using the rclone backend along with remotes which +are configured in rclone. + +The list of snapshots to copy can be filtered by host, path in the backup +and / or a comma-separated tag list: + +.. code-block:: console + + $ restic -r /srv/restic-repo copy --repo2 /srv/restic-repo-copy --host luigi --path /srv --tag foo,bar + +It is also possible to explicitly specify the list of snapshots to copy, in +which case only these instead of all snapshots will be copied: + +.. code-block:: console + + $ restic -r /srv/restic-repo copy --repo2 /srv/restic-repo-copy 410b18a2 4e5d5487 latest + + Checking integrity and consistency ================================== diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index e2fc51a6e..32e5f716a 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -20,6 +20,7 @@ Usage help is available: cache Operate on local cache directories cat Print internal objects to stdout check Check the repository for errors + copy Copy snapshots from one repository to another diff Show differences between two snapshots dump Print a backed-up file to stdout find Find a file, a directory or restic IDs From bbe8b73f032cc390e1c44b0f24ba4ef2f09ac75d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 24 Aug 2020 23:16:25 +0200 Subject: [PATCH 11/12] Update help text of backup command in docs --- doc/manual_rest.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index 32e5f716a..813cdfb8c 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -80,10 +80,9 @@ command: EXIT STATUS =========== - Exit status is 0 if the command was successful, and non-zero if there was any error. - - Note that some issues such as unreadable or deleted files during backup - currently doesn't result in a non-zero error exit status. + Exit status is 0 if the command was successful. + Exit status is 1 if there was a fatal error (no snapshot created). + Exit status is 3 if some source data could not be read (incomplete snapshot created). Usage: restic backup [flags] FILE/DIR [FILE/DIR] ... From 412623b848efce2f7a02c384dd8e0bec7b10b126 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 24 Aug 2020 23:03:49 +0200 Subject: [PATCH 12/12] copy: Reuse buffer for downloaded blobs --- cmd/restic/cmd_copy.go | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index be2c67356..8003b882c 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -113,7 +113,6 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { return err } - visitedTrees := restic.NewIDSet() dstSnapshotByOriginal := make(map[restic.ID][]*restic.Snapshot) for sn := range FindFilteredSnapshots(ctx, dstRepo, opts.Hosts, opts.Tags, opts.Paths, nil) { if sn.Original != nil && !sn.Original.IsNull() { @@ -123,6 +122,13 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { dstSnapshotByOriginal[*sn.ID()] = append(dstSnapshotByOriginal[*sn.ID()], sn) } + cloner := &treeCloner{ + srcRepo: srcRepo, + dstRepo: dstRepo, + visitedTrees: restic.NewIDSet(), + buf: nil, + } + for sn := range FindFilteredSnapshots(ctx, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) { Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) @@ -146,7 +152,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { } Verbosef(" copy started, this may take a while...\n") - if err := copyTree(ctx, srcRepo, dstRepo, *sn.Tree, visitedTrees); err != nil { + if err := cloner.copyTree(ctx, *sn.Tree); err != nil { return err } debug.Log("tree copied") @@ -190,21 +196,28 @@ func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool { return true } -func copyTree(ctx context.Context, srcRepo, dstRepo restic.Repository, treeID restic.ID, visitedTrees restic.IDSet) error { +type treeCloner struct { + srcRepo restic.Repository + dstRepo restic.Repository + visitedTrees restic.IDSet + buf []byte +} + +func (t *treeCloner) copyTree(ctx context.Context, treeID restic.ID) error { // We have already processed this tree - if visitedTrees.Has(treeID) { + if t.visitedTrees.Has(treeID) { return nil } - tree, err := srcRepo.LoadTree(ctx, treeID) + tree, err := t.srcRepo.LoadTree(ctx, treeID) if err != nil { return fmt.Errorf("LoadTree(%v) returned error %v", treeID.Str(), err) } - visitedTrees.Insert(treeID) + t.visitedTrees.Insert(treeID) // Do we already have this tree blob? - if !dstRepo.Index().Has(treeID, restic.TreeBlob) { - newTreeID, err := dstRepo.SaveTree(ctx, tree) + if !t.dstRepo.Index().Has(treeID, restic.TreeBlob) { + newTreeID, err := t.dstRepo.SaveTree(ctx, tree) if err != nil { return fmt.Errorf("SaveTree(%v) returned error %v", treeID.Str(), err) } @@ -214,29 +227,28 @@ func copyTree(ctx context.Context, srcRepo, dstRepo restic.Repository, treeID re } } - // TODO: keep only one (big) buffer around. // TODO: parellize this stuff, likely only needed inside a tree. for _, entry := range tree.Nodes { // If it is a directory, recurse if entry.Type == "dir" && entry.Subtree != nil { - if err := copyTree(ctx, srcRepo, dstRepo, *entry.Subtree, visitedTrees); err != nil { + if err := t.copyTree(ctx, *entry.Subtree); err != nil { return err } } // Copy the blobs for this file. for _, blobID := range entry.Content { // Do we already have this data blob? - if dstRepo.Index().Has(blobID, restic.DataBlob) { + if t.dstRepo.Index().Has(blobID, restic.DataBlob) { continue } debug.Log("Copying blob %s\n", blobID.Str()) - buf, err := srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, nil) + t.buf, err = t.srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, t.buf) if err != nil { return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err) } - _, _, err = dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID, false) + _, _, err = t.dstRepo.SaveBlob(ctx, restic.DataBlob, t.buf, blobID, false) if err != nil { return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err) }