From a9707a5728398747f2a11cdf24c1d98636a5b46c Mon Sep 17 00:00:00 2001 From: Pauline Middelink Date: Thu, 9 Mar 2017 16:06:28 +0100 Subject: [PATCH] Refactor output of `find` to allow for json and normal output. Rather complicated solution becaused I wanted to retain the streaming character of the output, which means for json I have to manually add headers and footers per snapshot scanned + a list around the whole set. As the json ouput is now partly handcrafted, add proper testing to catch unintentional changes to the output, making it non-json compliant. Closes #869 --- src/cmds/restic/cmd_find.go | 108 +++++++++++++++++++++++++--- src/cmds/restic/integration_test.go | 65 ++++++++++++++--- 2 files changed, 154 insertions(+), 19 deletions(-) diff --git a/src/cmds/restic/cmd_find.go b/src/cmds/restic/cmd_find.go index 23c39485d..5c1893268 100644 --- a/src/cmds/restic/cmd_find.go +++ b/src/cmds/restic/cmd_find.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "path/filepath" "strings" "time" @@ -84,7 +85,94 @@ func parseTime(str string) (time.Time, error) { return time.Time{}, errors.Fatalf("unable to parse time: %q", str) } -func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, prefix string, snapshotID *string) error { +type statefulOutput struct { + ListLong bool + JSON bool + inuse bool + newsn *restic.Snapshot + oldsn *restic.Snapshot + hits int +} + +func (s *statefulOutput) PrintJSON(prefix string, node *restic.Node) { + type findNode restic.Node + b, err := json.Marshal(struct { + // Add these attributes + Path string `json:"path,omitempty"` + Permissions string `json:"permissions,omitempty"` + + *findNode + + // Make the following attributes disappear + Name byte `json:"name,omitempty"` + Inode byte `json:"inode,omitempty"` + ExtendedAttributes byte `json:"extended_attributes,omitempty"` + Device byte `json:"device,omitempty"` + Content byte `json:"content,omitempty"` + Subtree byte `json:"subtree,omitempty"` + }{ + Path: filepath.Join(prefix, node.Name), + Permissions: node.Mode.String(), + findNode: (*findNode)(node), + }) + if err != nil { + Warnf("Marshall failed: %v\n", err) + return + } + if !s.inuse { + Printf("[") + s.inuse = true + } + if s.newsn != s.oldsn { + if s.oldsn != nil { + Printf("],\"hits\":%d,\"snapshot\":%q},", s.hits, s.oldsn.ID()) + } + Printf(`{"matches":[`) + s.oldsn = s.newsn + s.hits = 0 + } + if s.hits > 0 { + Printf(",") + } + Printf(string(b)) + s.hits++ +} + +func (s *statefulOutput) PrintNormal(prefix string, node *restic.Node) { + if s.newsn != s.oldsn { + if s.oldsn != nil { + Verbosef("\n") + } + s.oldsn = s.newsn + Verbosef("Found matching entries in snapshot %s\n", s.oldsn.ID()) + } + Printf(formatNode(prefix, node, s.ListLong) + "\n") +} + +func (s *statefulOutput) Print(prefix string, node *restic.Node) { + if s.JSON { + s.PrintJSON(prefix, node) + } else { + s.PrintNormal(prefix, node) + } +} + +func (s *statefulOutput) Finish() { + if s.JSON { + // do some finishing up + if s.oldsn != nil { + Printf("],\"hits\":%d,\"snapshot\":%q}", s.hits, s.oldsn.ID()) + } + if s.inuse { + Printf("]\n") + } else { + Printf("[]\n") + } + return + } +} + +func findInTree(repo *repository.Repository, pat *findPattern, id restic.ID, prefix string, state *statefulOutput) error { debug.Log("checking tree %v\n", id) tree, err := repo.LoadTree(id) @@ -117,17 +205,13 @@ func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, pref continue } - if snapshotID != nil { - Verbosef("Found matching entries in snapshot %s\n", *snapshotID) - snapshotID = nil - } - Printf(formatNode(prefix, node, findOptions.ListLong) + "\n") + state.Print(prefix, node) } else { debug.Log(" pattern does not match\n") } if node.Type == "dir" { - if err := findInTree(repo, pat, *node.Subtree, filepath.Join(prefix, node.Name), snapshotID); err != nil { + if err := findInTree(repo, pat, *node.Subtree, filepath.Join(prefix, node.Name), state); err != nil { return err } } @@ -136,11 +220,11 @@ func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, pref return nil } -func findInSnapshot(repo *repository.Repository, sn *restic.Snapshot, pat findPattern) error { +func findInSnapshot(repo *repository.Repository, sn *restic.Snapshot, pat findPattern, state *statefulOutput) error { debug.Log("searching in snapshot %s\n for entries within [%s %s]", sn.ID(), pat.oldest, pat.newest) - snapshotID := sn.ID().Str() - if err := findInTree(repo, pat, *sn.Tree, string(filepath.Separator), &snapshotID); err != nil { + state.newsn = sn + if err := findInTree(repo, &pat, *sn.Tree, string(filepath.Separator), state); err != nil { return err } return nil @@ -189,11 +273,13 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error { ctx, cancel := context.WithCancel(gopts.ctx) defer cancel() + state := statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON} for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) { - if err = findInSnapshot(repo, sn, pat); err != nil { + if err = findInSnapshot(repo, sn, pat, &state); err != nil { return err } } + state.Finish() return nil } diff --git a/src/cmds/restic/integration_test.go b/src/cmds/restic/integration_test.go index 0adf495a3..be1ce755e 100644 --- a/src/cmds/restic/integration_test.go +++ b/src/cmds/restic/integration_test.go @@ -149,18 +149,20 @@ func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { return strings.Split(string(buf.Bytes()), "\n") } -func testRunFind(t testing.TB, gopts GlobalOptions, pattern string) []string { +func testRunFind(t testing.TB, wantJSON bool, gopts GlobalOptions, pattern string) []byte { buf := bytes.NewBuffer(nil) globalOptions.stdout = buf + globalOptions.JSON = wantJSON defer func() { globalOptions.stdout = os.Stdout + globalOptions.JSON = false }() opts := FindOptions{} OK(t, runFind(opts, gopts, []string{pattern})) - return strings.Split(string(buf.Bytes()), "\n") + return buf.Bytes() } func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snapmap map[restic.ID]Snapshot) { @@ -1037,14 +1039,61 @@ func TestFind(t *testing.T) { testRunBackup(t, []string{env.testdata}, opts, gopts) testRunCheck(t, gopts) - results := testRunFind(t, gopts, "unexistingfile") - Assert(t, len(results) != 0, "unexisting file found in repo (%v)", datafile) + results := testRunFind(t, false, gopts, "unexistingfile") + Assert(t, len(results) == 0, "unexisting file found in repo (%v)", datafile) - results = testRunFind(t, gopts, "testfile") - Assert(t, len(results) != 1, "file not found in repo (%v)", datafile) + results = testRunFind(t, false, gopts, "testfile") + lines := strings.Split(string(results), "\n") + Assert(t, len(lines) == 2, "expected one file found in repo (%v)", datafile) - results = testRunFind(t, gopts, "test") - Assert(t, len(results) < 2, "less than two file found in repo (%v)", datafile) + results = testRunFind(t, false, gopts, "testfile*") + lines = strings.Split(string(results), "\n") + Assert(t, len(lines) == 4, "expected three files found in repo (%v)", datafile) + }) +} + +type testMatch struct { + Path string `json:"path,omitempty"` + Permissions string `json:"permissions,omitempty"` + Size uint64 `json:"size,omitempty"` + Date time.Time `json:"date,omitempty"` + UID uint32 `json:"uid,omitempty"` + GID uint32 `json:"gid,omitempty"` +} + +type testMatches struct { + Hits int `json:"hits,omitempty"` + SnapshotID string `json:"snapshot,omitempty"` + Matches []testMatch `json:"matches,omitempty"` +} + +func TestFindJSON(t *testing.T) { + withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) { + datafile := filepath.Join("testdata", "backup-data.tar.gz") + testRunInit(t, gopts) + SetupTarTestFixture(t, env.testdata, datafile) + + opts := BackupOptions{} + + testRunBackup(t, []string{env.testdata}, opts, gopts) + testRunCheck(t, gopts) + + results := testRunFind(t, true, gopts, "unexistingfile") + matches := []testMatches{} + OK(t, json.Unmarshal(results, &matches)) + Assert(t, len(matches) == 0, "expected no match in repo (%v)", datafile) + + results = testRunFind(t, true, gopts, "testfile") + OK(t, json.Unmarshal(results, &matches)) + Assert(t, len(matches) == 1, "expected a single snapshot in repo (%v)", datafile) + Assert(t, len(matches[0].Matches) == 1, "expected a single file to match (%v)", datafile) + Assert(t, matches[0].Hits == 1, "expected hits to show 1 match (%v)", datafile) + + results = testRunFind(t, true, gopts, "testfile*") + OK(t, json.Unmarshal(results, &matches)) + Assert(t, len(matches) == 1, "expected a single snapshot in repo (%v)", datafile) + Assert(t, len(matches[0].Matches) == 3, "expected 3 files to match (%v)", datafile) + Assert(t, matches[0].Hits == 3, "expected hits to show 3 matches (%v)", datafile) }) }