mirror of https://github.com/restic/restic.git
Restore whole folder to sdtout as tar
With this change it is possible to dump a folder to stdout as a tar. The It can be used just like the normal dump command: `./restic dump fa97e6e1 "/data/test/" > test.tar` Where `/data/test/` is a a folder instead of a file.
This commit is contained in:
Simon Beck
Alexander Neumann
parent
870e7583a1
commit
cc8b690b52
12
build.go
12
build.go
|
|
@ -60,12 +60,12 @@ import (
|
|||
|
||||
// config contains the configuration for the program to build.
|
||||
var config = Config{
|
||||
Name: "restic", // name of the program executable and directory
|
||||
Namespace: "github.com/restic/restic", // subdir of GOPATH, e.g. "github.com/foo/bar"
|
||||
Main: "./cmd/restic", // package name for the main package
|
||||
DefaultBuildTags: []string{"selfupdate"}, // specify build tags which are always used
|
||||
Tests: []string{"./..."}, // tests to run
|
||||
MinVersion: GoVersion{Major: 1, Minor: 9, Patch: 0}, // minimum Go version supported
|
||||
Name: "restic", // name of the program executable and directory
|
||||
Namespace: "github.com/restic/restic", // subdir of GOPATH, e.g. "github.com/foo/bar"
|
||||
Main: "./cmd/restic", // package name for the main package
|
||||
DefaultBuildTags: []string{"selfupdate"}, // specify build tags which are always used
|
||||
Tests: []string{"./..."}, // tests to run
|
||||
MinVersion: GoVersion{Major: 1, Minor: 10, Patch: 0}, // minimum Go version supported
|
||||
}
|
||||
|
||||
// Config configures the build.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
Enhancement: Ability to dump folders to tar via stdout
|
||||
|
||||
We've added the ability to dump whole folders to stdout via the `dump` command.
|
||||
Restic now requires at least Go 1.10 due to a limitation of the standard
|
||||
library for Go <= 1.9.
|
||||
|
||||
https://github.com/restic/restic/pull/2124
|
||||
https://github.com/restic/restic/issues/2123
|
||||
|
|
@ -0,0 +1,131 @@
|
|||
package main
|
||||
|
||||
// Adapted from https://github.com/maxymania/go-system/blob/master/posix_acl/posix_acl.go
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
const (
|
||||
aclUserOwner = 0x0001
|
||||
aclUser = 0x0002
|
||||
aclGroupOwner = 0x0004
|
||||
aclGroup = 0x0008
|
||||
aclMask = 0x0010
|
||||
aclOthers = 0x0020
|
||||
)
|
||||
|
||||
type aclSID uint64
|
||||
|
||||
type aclElem struct {
|
||||
Tag uint16
|
||||
Perm uint16
|
||||
ID uint32
|
||||
}
|
||||
|
||||
type acl struct {
|
||||
Version uint32
|
||||
List []aclElement
|
||||
}
|
||||
|
||||
type aclElement struct {
|
||||
aclSID
|
||||
Perm uint16
|
||||
}
|
||||
|
||||
func (a *aclSID) setUID(uid uint32) {
|
||||
*a = aclSID(uid) | (aclUser << 32)
|
||||
}
|
||||
func (a *aclSID) setGID(gid uint32) {
|
||||
*a = aclSID(gid) | (aclGroup << 32)
|
||||
}
|
||||
|
||||
func (a *aclSID) setType(tp int) {
|
||||
*a = aclSID(tp) << 32
|
||||
}
|
||||
|
||||
func (a aclSID) getType() int {
|
||||
return int(a >> 32)
|
||||
}
|
||||
func (a aclSID) getID() uint32 {
|
||||
return uint32(a & 0xffffffff)
|
||||
}
|
||||
func (a aclSID) String() string {
|
||||
switch a >> 32 {
|
||||
case aclUserOwner:
|
||||
return "user::"
|
||||
case aclUser:
|
||||
return fmt.Sprintf("user:%v:", a.getID())
|
||||
case aclGroupOwner:
|
||||
return "group::"
|
||||
case aclGroup:
|
||||
return fmt.Sprintf("group:%v:", a.getID())
|
||||
case aclMask:
|
||||
return "mask::"
|
||||
case aclOthers:
|
||||
return "other::"
|
||||
}
|
||||
return "?:"
|
||||
}
|
||||
|
||||
func (a aclElement) String() string {
|
||||
str := ""
|
||||
if (a.Perm & 4) != 0 {
|
||||
str += "r"
|
||||
} else {
|
||||
str += "-"
|
||||
}
|
||||
if (a.Perm & 2) != 0 {
|
||||
str += "w"
|
||||
} else {
|
||||
str += "-"
|
||||
}
|
||||
if (a.Perm & 1) != 0 {
|
||||
str += "x"
|
||||
} else {
|
||||
str += "-"
|
||||
}
|
||||
return fmt.Sprintf("%v%v", a.aclSID, str)
|
||||
}
|
||||
|
||||
func (a *acl) decode(xattr []byte) {
|
||||
var elem aclElement
|
||||
ae := new(aclElem)
|
||||
nr := bytes.NewReader(xattr)
|
||||
e := binary.Read(nr, binary.LittleEndian, &a.Version)
|
||||
if e != nil {
|
||||
a.Version = 0
|
||||
return
|
||||
}
|
||||
if len(a.List) > 0 {
|
||||
a.List = a.List[:0]
|
||||
}
|
||||
for binary.Read(nr, binary.LittleEndian, ae) == nil {
|
||||
elem.aclSID = (aclSID(ae.Tag) << 32) | aclSID(ae.ID)
|
||||
elem.Perm = ae.Perm
|
||||
a.List = append(a.List, elem)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *acl) encode() []byte {
|
||||
buf := new(bytes.Buffer)
|
||||
ae := new(aclElem)
|
||||
binary.Write(buf, binary.LittleEndian, &a.Version)
|
||||
for _, elem := range a.List {
|
||||
ae.Tag = uint16(elem.getType())
|
||||
ae.Perm = elem.Perm
|
||||
ae.ID = elem.getID()
|
||||
binary.Write(buf, binary.LittleEndian, ae)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func (a *acl) String() string {
|
||||
var finalacl string
|
||||
for _, acl := range a.List {
|
||||
finalacl += acl.String() + "\n"
|
||||
}
|
||||
return finalacl
|
||||
}
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func Test_acl_decode(t *testing.T) {
|
||||
type args struct {
|
||||
xattr []byte
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "decode string",
|
||||
args: args{
|
||||
xattr: []byte{2, 0, 0, 0, 1, 0, 6, 0, 255, 255, 255, 255, 2, 0, 7, 0, 0, 0, 0, 0, 2, 0, 7, 0, 254, 255, 0, 0, 4, 0, 7, 0, 255, 255, 255, 255, 16, 0, 7, 0, 255, 255, 255, 255, 32, 0, 4, 0, 255, 255, 255, 255},
|
||||
},
|
||||
want: "user::rw-\nuser:0:rwx\nuser:65534:rwx\ngroup::rwx\nmask::rwx\nother::r--\n",
|
||||
},
|
||||
{
|
||||
name: "decode fail",
|
||||
args: args{
|
||||
xattr: []byte("abctest"),
|
||||
},
|
||||
want: "",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
a := &acl{}
|
||||
a.decode(tt.args.xattr)
|
||||
if tt.want != a.String() {
|
||||
t.Errorf("acl.decode() = %v, want: %v", a.String(), tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_acl_encode(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
want []byte
|
||||
args []aclElement
|
||||
}{
|
||||
{
|
||||
name: "encode values",
|
||||
want: []byte{2, 0, 0, 0, 1, 0, 6, 0, 255, 255, 255, 255, 2, 0, 7, 0, 0, 0, 0, 0, 2, 0, 7, 0, 254, 255, 0, 0, 4, 0, 7, 0, 255, 255, 255, 255, 16, 0, 7, 0, 255, 255, 255, 255, 32, 0, 4, 0, 255, 255, 255, 255},
|
||||
args: []aclElement{
|
||||
{
|
||||
aclSID: 8589934591,
|
||||
Perm: 6,
|
||||
},
|
||||
{
|
||||
aclSID: 8589934592,
|
||||
Perm: 7,
|
||||
},
|
||||
{
|
||||
aclSID: 8590000126,
|
||||
Perm: 7,
|
||||
},
|
||||
{
|
||||
aclSID: 21474836479,
|
||||
Perm: 7,
|
||||
},
|
||||
{
|
||||
aclSID: 73014444031,
|
||||
Perm: 7,
|
||||
},
|
||||
{
|
||||
aclSID: 141733920767,
|
||||
Perm: 4,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "encode fail",
|
||||
want: []byte{2, 0, 0, 0},
|
||||
args: []aclElement{},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
a := &acl{
|
||||
Version: 2,
|
||||
List: tt.args,
|
||||
}
|
||||
if got := a.encode(); !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("acl.encode() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,15 +1,19 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/walker"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
@ -50,41 +54,18 @@ func init() {
|
|||
|
||||
func splitPath(p string) []string {
|
||||
d, f := path.Split(p)
|
||||
if d == "" || d == "/" {
|
||||
if d == "" {
|
||||
return []string{f}
|
||||
}
|
||||
if d == "/" {
|
||||
return []string{d}
|
||||
}
|
||||
s := splitPath(path.Clean(d))
|
||||
return append(s, f)
|
||||
}
|
||||
|
||||
func dumpNode(ctx context.Context, repo restic.Repository, node *restic.Node) error {
|
||||
var buf []byte
|
||||
for _, id := range node.Content {
|
||||
size, found := repo.LookupBlobSize(id, restic.DataBlob)
|
||||
if !found {
|
||||
return errors.Errorf("id %v not found in repository", id)
|
||||
}
|
||||
func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, pathToPrint string) error {
|
||||
|
||||
buf = buf[:cap(buf)]
|
||||
if len(buf) < restic.CiphertextLength(int(size)) {
|
||||
buf = restic.NewBlobBuffer(int(size))
|
||||
}
|
||||
|
||||
n, err := repo.LoadBlob(ctx, restic.DataBlob, id, buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
buf = buf[:n]
|
||||
|
||||
_, err = os.Stdout.Write(buf)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Write")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string) error {
|
||||
if tree == nil {
|
||||
return fmt.Errorf("called with a nil tree")
|
||||
}
|
||||
|
|
@ -97,16 +78,19 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
|
|||
}
|
||||
item := filepath.Join(prefix, pathComponents[0])
|
||||
for _, node := range tree.Nodes {
|
||||
if node.Name == pathComponents[0] {
|
||||
if node.Name == pathComponents[0] || pathComponents[0] == "/" {
|
||||
switch {
|
||||
case l == 1 && node.Type == "file":
|
||||
return dumpNode(ctx, repo, node)
|
||||
return getNodeData(ctx, os.Stdout, repo, node)
|
||||
case l > 1 && node.Type == "dir":
|
||||
subtree, err := repo.LoadTree(ctx, *node.Subtree)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "cannot load subtree for %q", item)
|
||||
}
|
||||
return printFromTree(ctx, subtree, repo, item, pathComponents[1:])
|
||||
return printFromTree(ctx, subtree, repo, item, pathComponents[1:], pathToPrint)
|
||||
case node.Type == "dir":
|
||||
node.Path = pathToPrint
|
||||
return tarTree(ctx, repo, node, pathToPrint)
|
||||
case l > 1:
|
||||
return fmt.Errorf("%q should be a dir, but s a %q", item, node.Type)
|
||||
case node.Type != "file":
|
||||
|
|
@ -129,7 +113,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
|
|||
|
||||
debug.Log("dump file %q from %q", pathToPrint, snapshotIDString)
|
||||
|
||||
splittedPath := splitPath(pathToPrint)
|
||||
splittedPath := splitPath(path.Clean(pathToPrint))
|
||||
|
||||
repo, err := OpenRepository(gopts)
|
||||
if err != nil {
|
||||
|
|
@ -173,10 +157,143 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
|
|||
Exitf(2, "loading tree for snapshot %q failed: %v", snapshotIDString, err)
|
||||
}
|
||||
|
||||
err = printFromTree(ctx, tree, repo, "", splittedPath)
|
||||
err = printFromTree(ctx, tree, repo, "", splittedPath, pathToPrint)
|
||||
if err != nil {
|
||||
Exitf(2, "cannot dump file: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getNodeData(ctx context.Context, output io.Writer, repo restic.Repository, node *restic.Node) error {
|
||||
var buf []byte
|
||||
for _, id := range node.Content {
|
||||
|
||||
size, found := repo.LookupBlobSize(id, restic.DataBlob)
|
||||
if !found {
|
||||
return errors.Errorf("id %v not found in repository", id)
|
||||
}
|
||||
|
||||
buf = buf[:cap(buf)]
|
||||
if len(buf) < restic.CiphertextLength(int(size)) {
|
||||
buf = restic.NewBlobBuffer(int(size))
|
||||
}
|
||||
|
||||
n, err := repo.LoadBlob(ctx, restic.DataBlob, id, buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
buf = buf[:n]
|
||||
|
||||
_, err = output.Write(buf)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Write")
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func tarTree(ctx context.Context, repo restic.Repository, rootNode *restic.Node, rootPath string) error {
|
||||
|
||||
if stdoutIsTerminal() {
|
||||
return fmt.Errorf("stdout is the terminal, please redirect output")
|
||||
}
|
||||
|
||||
tw := tar.NewWriter(os.Stdout)
|
||||
defer tw.Close()
|
||||
|
||||
// If we want to dump "/" we'll need to add the name of the first node, too
|
||||
// as it would get lost otherwise.
|
||||
if rootNode.Path == "/" {
|
||||
rootNode.Path = path.Join(rootNode.Path, rootNode.Name)
|
||||
rootPath = rootNode.Path
|
||||
}
|
||||
|
||||
// we know that rootNode is a folder and walker.Walk will already process
|
||||
// the next node, so we have to tar this one first, too
|
||||
if err := tarNode(ctx, tw, rootNode, repo); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err := walker.Walk(ctx, repo, *rootNode.Subtree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if node == nil {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
node.Path = path.Join(rootPath, nodepath)
|
||||
|
||||
if node.Type == "file" || node.Type == "symlink" || node.Type == "dir" {
|
||||
err := tarNode(ctx, tw, node, repo)
|
||||
if err != err {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
return false, nil
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func tarNode(ctx context.Context, tw *tar.Writer, node *restic.Node, repo restic.Repository) error {
|
||||
|
||||
header := &tar.Header{
|
||||
Name: node.Path,
|
||||
Size: int64(node.Size),
|
||||
Mode: int64(node.Mode),
|
||||
Uid: int(node.UID),
|
||||
Gid: int(node.GID),
|
||||
ModTime: node.ModTime,
|
||||
AccessTime: node.AccessTime,
|
||||
ChangeTime: node.ChangeTime,
|
||||
PAXRecords: parseXattrs(node.ExtendedAttributes),
|
||||
}
|
||||
|
||||
if node.Type == "symlink" {
|
||||
header.Typeflag = tar.TypeSymlink
|
||||
header.Linkname = node.LinkTarget
|
||||
}
|
||||
|
||||
if node.Type == "dir" {
|
||||
header.Typeflag = tar.TypeDir
|
||||
}
|
||||
|
||||
err := tw.WriteHeader(header)
|
||||
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "TarHeader ")
|
||||
}
|
||||
|
||||
return getNodeData(ctx, tw, repo, node)
|
||||
|
||||
}
|
||||
|
||||
func parseXattrs(xattrs []restic.ExtendedAttribute) map[string]string {
|
||||
tmpMap := make(map[string]string)
|
||||
|
||||
for _, attr := range xattrs {
|
||||
attrString := string(attr.Value)
|
||||
|
||||
if strings.HasPrefix(attr.Name, "system.posix_acl_") {
|
||||
na := acl{}
|
||||
na.decode(attr.Value)
|
||||
|
||||
if na.String() != "" {
|
||||
if strings.Contains(attr.Name, "system.posix_acl_access") {
|
||||
tmpMap["SCHILY.acl.access"] = na.String()
|
||||
} else if strings.Contains(attr.Name, "system.posix_acl_default") {
|
||||
tmpMap["SCHILY.acl.default"] = na.String()
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
tmpMap["SCHILY.xattr."+attr.Name] = attrString
|
||||
}
|
||||
}
|
||||
|
||||
return tmpMap
|
||||
}
|
||||
|
|
|
|||
|
|
@ -124,3 +124,13 @@ e.g.:
|
|||
.. code-block:: console
|
||||
|
||||
$ restic -r /srv/restic-repo dump --path /production.sql latest production.sql | mysql
|
||||
|
||||
It is also possible to ``dump`` the contents of a whole folder structure to
|
||||
stdout. To retain the information about the files and folders Restic will
|
||||
output the contents in the tar format:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ restic -r /srv/restic-repo dump /home/other/work latest > restore.tar
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue