From cc8b690b527018ef57cc8a66b4aadf6db91d65e4 Mon Sep 17 00:00:00 2001 From: Simon Beck Date: Thu, 20 Dec 2018 16:39:48 +0100 Subject: [PATCH] Restore whole folder to sdtout as tar With this change it is possible to dump a folder to stdout as a tar. The It can be used just like the normal dump command: `./restic dump fa97e6e1 "/data/test/" > test.tar` Where `/data/test/` is a a folder instead of a file. --- build.go | 12 +-- changelog/unreleased/pull-2124 | 8 ++ cmd/restic/acl.go | 131 +++++++++++++++++++++++ cmd/restic/acl_test.go | 96 +++++++++++++++++ cmd/restic/cmd_dump.go | 183 +++++++++++++++++++++++++++------ doc/050_restore.rst | 10 ++ 6 files changed, 401 insertions(+), 39 deletions(-) create mode 100644 changelog/unreleased/pull-2124 create mode 100644 cmd/restic/acl.go create mode 100644 cmd/restic/acl_test.go diff --git a/build.go b/build.go index fdd6f943a..2e3e5d05f 100644 --- a/build.go +++ b/build.go @@ -60,12 +60,12 @@ import ( // config contains the configuration for the program to build. var config = Config{ - Name: "restic", // name of the program executable and directory - Namespace: "github.com/restic/restic", // subdir of GOPATH, e.g. "github.com/foo/bar" - Main: "./cmd/restic", // package name for the main package - DefaultBuildTags: []string{"selfupdate"}, // specify build tags which are always used - Tests: []string{"./..."}, // tests to run - MinVersion: GoVersion{Major: 1, Minor: 9, Patch: 0}, // minimum Go version supported + Name: "restic", // name of the program executable and directory + Namespace: "github.com/restic/restic", // subdir of GOPATH, e.g. "github.com/foo/bar" + Main: "./cmd/restic", // package name for the main package + DefaultBuildTags: []string{"selfupdate"}, // specify build tags which are always used + Tests: []string{"./..."}, // tests to run + MinVersion: GoVersion{Major: 1, Minor: 10, Patch: 0}, // minimum Go version supported } // Config configures the build. diff --git a/changelog/unreleased/pull-2124 b/changelog/unreleased/pull-2124 new file mode 100644 index 000000000..938110151 --- /dev/null +++ b/changelog/unreleased/pull-2124 @@ -0,0 +1,8 @@ +Enhancement: Ability to dump folders to tar via stdout + +We've added the ability to dump whole folders to stdout via the `dump` command. +Restic now requires at least Go 1.10 due to a limitation of the standard +library for Go <= 1.9. + +https://github.com/restic/restic/pull/2124 +https://github.com/restic/restic/issues/2123 diff --git a/cmd/restic/acl.go b/cmd/restic/acl.go new file mode 100644 index 000000000..562ea89e1 --- /dev/null +++ b/cmd/restic/acl.go @@ -0,0 +1,131 @@ +package main + +// Adapted from https://github.com/maxymania/go-system/blob/master/posix_acl/posix_acl.go + +import ( + "bytes" + "encoding/binary" + "fmt" +) + +const ( + aclUserOwner = 0x0001 + aclUser = 0x0002 + aclGroupOwner = 0x0004 + aclGroup = 0x0008 + aclMask = 0x0010 + aclOthers = 0x0020 +) + +type aclSID uint64 + +type aclElem struct { + Tag uint16 + Perm uint16 + ID uint32 +} + +type acl struct { + Version uint32 + List []aclElement +} + +type aclElement struct { + aclSID + Perm uint16 +} + +func (a *aclSID) setUID(uid uint32) { + *a = aclSID(uid) | (aclUser << 32) +} +func (a *aclSID) setGID(gid uint32) { + *a = aclSID(gid) | (aclGroup << 32) +} + +func (a *aclSID) setType(tp int) { + *a = aclSID(tp) << 32 +} + +func (a aclSID) getType() int { + return int(a >> 32) +} +func (a aclSID) getID() uint32 { + return uint32(a & 0xffffffff) +} +func (a aclSID) String() string { + switch a >> 32 { + case aclUserOwner: + return "user::" + case aclUser: + return fmt.Sprintf("user:%v:", a.getID()) + case aclGroupOwner: + return "group::" + case aclGroup: + return fmt.Sprintf("group:%v:", a.getID()) + case aclMask: + return "mask::" + case aclOthers: + return "other::" + } + return "?:" +} + +func (a aclElement) String() string { + str := "" + if (a.Perm & 4) != 0 { + str += "r" + } else { + str += "-" + } + if (a.Perm & 2) != 0 { + str += "w" + } else { + str += "-" + } + if (a.Perm & 1) != 0 { + str += "x" + } else { + str += "-" + } + return fmt.Sprintf("%v%v", a.aclSID, str) +} + +func (a *acl) decode(xattr []byte) { + var elem aclElement + ae := new(aclElem) + nr := bytes.NewReader(xattr) + e := binary.Read(nr, binary.LittleEndian, &a.Version) + if e != nil { + a.Version = 0 + return + } + if len(a.List) > 0 { + a.List = a.List[:0] + } + for binary.Read(nr, binary.LittleEndian, ae) == nil { + elem.aclSID = (aclSID(ae.Tag) << 32) | aclSID(ae.ID) + elem.Perm = ae.Perm + a.List = append(a.List, elem) + } +} + +func (a *acl) encode() []byte { + buf := new(bytes.Buffer) + ae := new(aclElem) + binary.Write(buf, binary.LittleEndian, &a.Version) + for _, elem := range a.List { + ae.Tag = uint16(elem.getType()) + ae.Perm = elem.Perm + ae.ID = elem.getID() + binary.Write(buf, binary.LittleEndian, ae) + } + return buf.Bytes() +} + +func (a *acl) String() string { + var finalacl string + for _, acl := range a.List { + finalacl += acl.String() + "\n" + } + return finalacl +} diff --git a/cmd/restic/acl_test.go b/cmd/restic/acl_test.go new file mode 100644 index 000000000..1e069d168 --- /dev/null +++ b/cmd/restic/acl_test.go @@ -0,0 +1,96 @@ +package main + +import ( + "reflect" + "testing" +) + +func Test_acl_decode(t *testing.T) { + type args struct { + xattr []byte + } + tests := []struct { + name string + args args + want string + }{ + { + name: "decode string", + args: args{ + xattr: []byte{2, 0, 0, 0, 1, 0, 6, 0, 255, 255, 255, 255, 2, 0, 7, 0, 0, 0, 0, 0, 2, 0, 7, 0, 254, 255, 0, 0, 4, 0, 7, 0, 255, 255, 255, 255, 16, 0, 7, 0, 255, 255, 255, 255, 32, 0, 4, 0, 255, 255, 255, 255}, + }, + want: "user::rw-\nuser:0:rwx\nuser:65534:rwx\ngroup::rwx\nmask::rwx\nother::r--\n", + }, + { + name: "decode fail", + args: args{ + xattr: []byte("abctest"), + }, + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := &acl{} + a.decode(tt.args.xattr) + if tt.want != a.String() { + t.Errorf("acl.decode() = %v, want: %v", a.String(), tt.want) + } + }) + } +} + +func Test_acl_encode(t *testing.T) { + tests := []struct { + name string + want []byte + args []aclElement + }{ + { + name: "encode values", + want: []byte{2, 0, 0, 0, 1, 0, 6, 0, 255, 255, 255, 255, 2, 0, 7, 0, 0, 0, 0, 0, 2, 0, 7, 0, 254, 255, 0, 0, 4, 0, 7, 0, 255, 255, 255, 255, 16, 0, 7, 0, 255, 255, 255, 255, 32, 0, 4, 0, 255, 255, 255, 255}, + args: []aclElement{ + { + aclSID: 8589934591, + Perm: 6, + }, + { + aclSID: 8589934592, + Perm: 7, + }, + { + aclSID: 8590000126, + Perm: 7, + }, + { + aclSID: 21474836479, + Perm: 7, + }, + { + aclSID: 73014444031, + Perm: 7, + }, + { + aclSID: 141733920767, + Perm: 4, + }, + }, + }, + { + name: "encode fail", + want: []byte{2, 0, 0, 0}, + args: []aclElement{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := &acl{ + Version: 2, + List: tt.args, + } + if got := a.encode(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("acl.encode() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index a2e4fbe4a..ba21c8ea6 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -1,15 +1,19 @@ package main import ( + "archive/tar" "context" "fmt" + "io" "os" "path" "path/filepath" + "strings" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/walker" "github.com/spf13/cobra" ) @@ -50,41 +54,18 @@ func init() { func splitPath(p string) []string { d, f := path.Split(p) - if d == "" || d == "/" { + if d == "" { return []string{f} } + if d == "/" { + return []string{d} + } s := splitPath(path.Clean(d)) return append(s, f) } -func dumpNode(ctx context.Context, repo restic.Repository, node *restic.Node) error { - var buf []byte - for _, id := range node.Content { - size, found := repo.LookupBlobSize(id, restic.DataBlob) - if !found { - return errors.Errorf("id %v not found in repository", id) - } +func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, pathToPrint string) error { - buf = buf[:cap(buf)] - if len(buf) < restic.CiphertextLength(int(size)) { - buf = restic.NewBlobBuffer(int(size)) - } - - n, err := repo.LoadBlob(ctx, restic.DataBlob, id, buf) - if err != nil { - return err - } - buf = buf[:n] - - _, err = os.Stdout.Write(buf) - if err != nil { - return errors.Wrap(err, "Write") - } - } - return nil -} - -func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string) error { if tree == nil { return fmt.Errorf("called with a nil tree") } @@ -97,16 +78,19 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor } item := filepath.Join(prefix, pathComponents[0]) for _, node := range tree.Nodes { - if node.Name == pathComponents[0] { + if node.Name == pathComponents[0] || pathComponents[0] == "/" { switch { case l == 1 && node.Type == "file": - return dumpNode(ctx, repo, node) + return getNodeData(ctx, os.Stdout, repo, node) case l > 1 && node.Type == "dir": subtree, err := repo.LoadTree(ctx, *node.Subtree) if err != nil { return errors.Wrapf(err, "cannot load subtree for %q", item) } - return printFromTree(ctx, subtree, repo, item, pathComponents[1:]) + return printFromTree(ctx, subtree, repo, item, pathComponents[1:], pathToPrint) + case node.Type == "dir": + node.Path = pathToPrint + return tarTree(ctx, repo, node, pathToPrint) case l > 1: return fmt.Errorf("%q should be a dir, but s a %q", item, node.Type) case node.Type != "file": @@ -129,7 +113,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error { debug.Log("dump file %q from %q", pathToPrint, snapshotIDString) - splittedPath := splitPath(pathToPrint) + splittedPath := splitPath(path.Clean(pathToPrint)) repo, err := OpenRepository(gopts) if err != nil { @@ -173,10 +157,143 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error { Exitf(2, "loading tree for snapshot %q failed: %v", snapshotIDString, err) } - err = printFromTree(ctx, tree, repo, "", splittedPath) + err = printFromTree(ctx, tree, repo, "", splittedPath, pathToPrint) if err != nil { Exitf(2, "cannot dump file: %v", err) } return nil } + +func getNodeData(ctx context.Context, output io.Writer, repo restic.Repository, node *restic.Node) error { + var buf []byte + for _, id := range node.Content { + + size, found := repo.LookupBlobSize(id, restic.DataBlob) + if !found { + return errors.Errorf("id %v not found in repository", id) + } + + buf = buf[:cap(buf)] + if len(buf) < restic.CiphertextLength(int(size)) { + buf = restic.NewBlobBuffer(int(size)) + } + + n, err := repo.LoadBlob(ctx, restic.DataBlob, id, buf) + if err != nil { + return err + } + buf = buf[:n] + + _, err = output.Write(buf) + if err != nil { + return errors.Wrap(err, "Write") + } + + } + return nil +} + +func tarTree(ctx context.Context, repo restic.Repository, rootNode *restic.Node, rootPath string) error { + + if stdoutIsTerminal() { + return fmt.Errorf("stdout is the terminal, please redirect output") + } + + tw := tar.NewWriter(os.Stdout) + defer tw.Close() + + // If we want to dump "/" we'll need to add the name of the first node, too + // as it would get lost otherwise. + if rootNode.Path == "/" { + rootNode.Path = path.Join(rootNode.Path, rootNode.Name) + rootPath = rootNode.Path + } + + // we know that rootNode is a folder and walker.Walk will already process + // the next node, so we have to tar this one first, too + if err := tarNode(ctx, tw, rootNode, repo); err != nil { + return err + } + + err := walker.Walk(ctx, repo, *rootNode.Subtree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { + if err != nil { + return false, err + } + if node == nil { + return false, nil + } + + node.Path = path.Join(rootPath, nodepath) + + if node.Type == "file" || node.Type == "symlink" || node.Type == "dir" { + err := tarNode(ctx, tw, node, repo) + if err != err { + return false, err + } + } + + return false, nil + }) + + return err +} + +func tarNode(ctx context.Context, tw *tar.Writer, node *restic.Node, repo restic.Repository) error { + + header := &tar.Header{ + Name: node.Path, + Size: int64(node.Size), + Mode: int64(node.Mode), + Uid: int(node.UID), + Gid: int(node.GID), + ModTime: node.ModTime, + AccessTime: node.AccessTime, + ChangeTime: node.ChangeTime, + PAXRecords: parseXattrs(node.ExtendedAttributes), + } + + if node.Type == "symlink" { + header.Typeflag = tar.TypeSymlink + header.Linkname = node.LinkTarget + } + + if node.Type == "dir" { + header.Typeflag = tar.TypeDir + } + + err := tw.WriteHeader(header) + + if err != nil { + return errors.Wrap(err, "TarHeader ") + } + + return getNodeData(ctx, tw, repo, node) + +} + +func parseXattrs(xattrs []restic.ExtendedAttribute) map[string]string { + tmpMap := make(map[string]string) + + for _, attr := range xattrs { + attrString := string(attr.Value) + + if strings.HasPrefix(attr.Name, "system.posix_acl_") { + na := acl{} + na.decode(attr.Value) + + if na.String() != "" { + if strings.Contains(attr.Name, "system.posix_acl_access") { + tmpMap["SCHILY.acl.access"] = na.String() + } else if strings.Contains(attr.Name, "system.posix_acl_default") { + tmpMap["SCHILY.acl.default"] = na.String() + } + } + + } else { + tmpMap["SCHILY.xattr."+attr.Name] = attrString + } + } + + return tmpMap +} diff --git a/doc/050_restore.rst b/doc/050_restore.rst index c6f5db855..4b781b9d9 100644 --- a/doc/050_restore.rst +++ b/doc/050_restore.rst @@ -124,3 +124,13 @@ e.g.: .. code-block:: console $ restic -r /srv/restic-repo dump --path /production.sql latest production.sql | mysql + +It is also possible to ``dump`` the contents of a whole folder structure to +stdout. To retain the information about the files and folders Restic will +output the contents in the tar format: + +.. code-block:: console + + $ restic -r /srv/restic-repo dump /home/other/work latest > restore.tar + +