diff --git a/doc/Manual.md b/doc/Manual.md index 7d431937e..6ee5db947 100644 --- a/doc/Manual.md +++ b/doc/Manual.md @@ -182,6 +182,22 @@ see [`filepath.Match`](https://golang.org/pkg/path/filepath/#Match) for syntax. Additionally `**` exludes arbitrary subdirectories. Environment-variables in exclude-files are expanded with [`os.ExpandEnv`](https://golang.org/pkg/os/#ExpandEnv). +## Reading data from stdin + +Sometimes it can be nice to directly save the output of a program, e.g. +`mysqldump` so that the SQL can later be restored. Restic supports this mode of +operation, just supply the option `--stdin` to the `backup` command like this: + + $ mysqldump [...] | restic -r /tmp/backup backup --stdin + +This creates a new snapshot of the output of `mqsqldump`. You can then use e.g. +the fuse mounting option (see below) to mount the repository and read the file. + +By default, the file name `stdin` is used, a different name can be specified +with `--stdin-filename`, e.g. like this: + + $ mysqldump [...] | restic -r /tmp/backup backup --stdin --stdin-filenam production.sql + # List all snapshots Now, you can list all the snapshots stored in the repository: diff --git a/src/cmds/restic/cmd_backup.go b/src/cmds/restic/cmd_backup.go index 48e8f2d31..462f4698a 100644 --- a/src/cmds/restic/cmd_backup.go +++ b/src/cmds/restic/cmd_backup.go @@ -18,10 +18,12 @@ import ( ) type CmdBackup struct { - Parent string `short:"p" long:"parent" description:"use this parent snapshot (default: last snapshot in repo that has the same target)"` - Force bool `short:"f" long:"force" description:"Force re-reading the target. Overrides the \"parent\" flag"` - Excludes []string `short:"e" long:"exclude" description:"Exclude a pattern (can be specified multiple times)"` - ExcludeFile string `long:"exclude-file" description:"Read exclude-patterns from file"` + Parent string `short:"p" long:"parent" description:"use this parent snapshot (default: last snapshot in repo that has the same target)"` + Force bool `short:"f" long:"force" description:"Force re-reading the target. Overrides the \"parent\" flag"` + Excludes []string `short:"e" long:"exclude" description:"Exclude a pattern (can be specified multiple times)"` + ExcludeFile string `long:"exclude-file" description:"Read exclude-patterns from file"` + Stdin bool `long:"stdin" description:"read backup data from stdin"` + StdinFilename string `long:"stdin-filename" default:"stdin" description:"file name to use when reading from stdin"` global *GlobalOptions } @@ -175,6 +177,47 @@ func (cmd CmdBackup) newArchiveProgress(todo restic.Stat) *restic.Progress { return archiveProgress } +func (cmd CmdBackup) newArchiveStdinProgress() *restic.Progress { + if !cmd.global.ShowProgress() { + return nil + } + + archiveProgress := restic.NewProgress(time.Second) + + var bps uint64 + + archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) { + sec := uint64(d / time.Second) + if s.Bytes > 0 && sec > 0 && ticker { + bps = s.Bytes / sec + } + + status1 := fmt.Sprintf("[%s] %s %s/s", formatDuration(d), + formatBytes(s.Bytes), + formatBytes(bps)) + + w, _, err := terminal.GetSize(int(os.Stdout.Fd())) + if err == nil { + maxlen := w - len(status1) + + if maxlen < 4 { + status1 = "" + } else if len(status1) > maxlen { + status1 = status1[:maxlen-4] + status1 += "... " + } + } + + fmt.Printf("\x1b[2K%s\r", status1) + } + + archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) { + fmt.Printf("\nduration: %s, %s\n", formatDuration(d), formatRate(s.Bytes, d)) + } + + return archiveProgress +} + func samePaths(expected, actual []string) bool { if expected == nil || actual == nil { return true @@ -243,7 +286,41 @@ func filterExisting(items []string) (result []string, err error) { return } +func (cmd CmdBackup) readFromStdin(args []string) error { + if len(args) != 0 { + return fmt.Errorf("when reading from stdin, no additional files can be specified") + } + + repo, err := cmd.global.OpenRepository() + if err != nil { + return err + } + + lock, err := lockRepo(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + + err = repo.LoadIndex() + if err != nil { + return err + } + + _, id, err := restic.ArchiveReader(repo, cmd.newArchiveStdinProgress(), os.Stdin, cmd.StdinFilename) + if err != nil { + return err + } + + fmt.Printf("archived as %v\n", id.Str()) + return nil +} + func (cmd CmdBackup) Execute(args []string) error { + if cmd.Stdin { + return cmd.readFromStdin(args) + } + if len(args) == 0 { return fmt.Errorf("wrong number of parameters, Usage: %s", cmd.Usage()) } diff --git a/src/restic/archive_reader.go b/src/restic/archive_reader.go new file mode 100644 index 000000000..5a485e0fd --- /dev/null +++ b/src/restic/archive_reader.go @@ -0,0 +1,122 @@ +package restic + +import ( + "encoding/json" + "io" + "restic/backend" + "restic/debug" + "restic/pack" + "restic/repository" + "time" + + "github.com/restic/chunker" +) + +// saveTreeJSON stores a tree in the repository. +func saveTreeJSON(repo *repository.Repository, item interface{}) (backend.ID, error) { + data, err := json.Marshal(item) + if err != nil { + return backend.ID{}, err + } + data = append(data, '\n') + + // check if tree has been saved before + id := backend.Hash(data) + if repo.Index().Has(id) { + return id, nil + } + + return repo.SaveJSON(pack.Tree, item) +} + +// ArchiveReader reads from the reader and archives the data. Returned is the +// resulting snapshot and its ID. +func ArchiveReader(repo *repository.Repository, p *Progress, rd io.Reader, name string) (*Snapshot, backend.ID, error) { + debug.Log("ArchiveReader", "start archiving %s", name) + sn, err := NewSnapshot([]string{name}) + if err != nil { + return nil, backend.ID{}, err + } + + p.Start() + defer p.Done() + + chnker := chunker.New(rd, repo.Config.ChunkerPolynomial) + + var ids backend.IDs + var fileSize uint64 + + for { + chunk, err := chnker.Next(getBuf()) + if err == io.EOF { + break + } + + if err != nil { + return nil, backend.ID{}, err + } + + id := backend.Hash(chunk.Data) + + if !repo.Index().Has(id) { + _, err := repo.SaveAndEncrypt(pack.Data, chunk.Data, nil) + if err != nil { + return nil, backend.ID{}, err + } + debug.Log("ArchiveReader", "saved blob %v (%d bytes)\n", id.Str(), chunk.Length) + } else { + debug.Log("ArchiveReader", "blob %v already saved in the repo\n", id.Str()) + } + + freeBuf(chunk.Data) + + ids = append(ids, id) + + p.Report(Stat{Bytes: uint64(chunk.Length)}) + fileSize += uint64(chunk.Length) + } + + tree := &Tree{ + Nodes: []*Node{ + &Node{ + Name: name, + AccessTime: time.Now(), + ModTime: time.Now(), + Type: "file", + Mode: 0644, + Size: fileSize, + UID: sn.UID, + GID: sn.GID, + User: sn.Username, + Content: ids, + }, + }, + } + + treeID, err := saveTreeJSON(repo, tree) + if err != nil { + return nil, backend.ID{}, err + } + sn.Tree = &treeID + debug.Log("ArchiveReader", "tree saved as %v", treeID.Str()) + + id, err := repo.SaveJSONUnpacked(backend.Snapshot, sn) + if err != nil { + return nil, backend.ID{}, err + } + + sn.id = &id + debug.Log("ArchiveReader", "snapshot saved as %v", id.Str()) + + err = repo.Flush() + if err != nil { + return nil, backend.ID{}, err + } + + err = repo.SaveIndex() + if err != nil { + return nil, backend.ID{}, err + } + + return sn, id, nil +} diff --git a/src/restic/archive_reader_test.go b/src/restic/archive_reader_test.go new file mode 100644 index 000000000..4397bb90e --- /dev/null +++ b/src/restic/archive_reader_test.go @@ -0,0 +1,103 @@ +package restic + +import ( + "bytes" + "io" + "math/rand" + "restic/backend" + "restic/pack" + "restic/repository" + "testing" + + "github.com/restic/chunker" +) + +func loadBlob(t *testing.T, repo *repository.Repository, id backend.ID, buf []byte) []byte { + buf, err := repo.LoadBlob(pack.Data, id, buf) + if err != nil { + t.Fatalf("LoadBlob(%v) returned error %v", id, err) + } + + return buf +} + +func checkSavedFile(t *testing.T, repo *repository.Repository, treeID backend.ID, name string, rd io.Reader) { + tree, err := LoadTree(repo, treeID) + if err != nil { + t.Fatalf("LoadTree() returned error %v", err) + } + + if len(tree.Nodes) != 1 { + t.Fatalf("wrong number of nodes for tree, want %v, got %v", 1, len(tree.Nodes)) + } + + node := tree.Nodes[0] + if node.Name != "fakefile" { + t.Fatalf("wrong filename, want %v, got %v", "fakefile", node.Name) + } + + if len(node.Content) == 0 { + t.Fatalf("node.Content has length 0") + } + + // check blobs + buf := make([]byte, chunker.MaxSize) + buf2 := make([]byte, chunker.MaxSize) + for i, id := range node.Content { + buf = loadBlob(t, repo, id, buf) + + buf2 = buf2[:len(buf)] + _, err = io.ReadFull(rd, buf2) + + if !bytes.Equal(buf, buf2) { + t.Fatalf("blob %d (%v) is wrong", i, id.Str()) + } + } +} + +func TestArchiveReader(t *testing.T) { + repo, cleanup := repository.TestRepository(t) + defer cleanup() + + seed := rand.Int63() + size := int64(rand.Intn(50*1024*1024) + 50*1024*1024) + t.Logf("seed is 0x%016x, size is %v", seed, size) + + f := fakeFile(t, seed, size) + + sn, id, err := ArchiveReader(repo, nil, f, "fakefile") + if err != nil { + t.Fatalf("ArchiveReader() returned error %v", err) + } + + if id.IsNull() { + t.Fatalf("ArchiveReader() returned null ID") + } + + t.Logf("snapshot saved as %v, tree is %v", id.Str(), sn.Tree.Str()) + + checkSavedFile(t, repo, *sn.Tree, "fakefile", fakeFile(t, seed, size)) +} + +func BenchmarkArchiveReader(t *testing.B) { + repo, cleanup := repository.TestRepository(t) + defer cleanup() + + const size = 50 * 1024 * 1024 + + buf := make([]byte, size) + _, err := io.ReadFull(fakeFile(t, 23, size), buf) + if err != nil { + t.Fatal(err) + } + + t.SetBytes(size) + t.ResetTimer() + + for i := 0; i < t.N; i++ { + _, _, err := ArchiveReader(repo, nil, bytes.NewReader(buf), "fakefile") + if err != nil { + t.Fatal(err) + } + } +} diff --git a/src/restic/fuse/dir.go b/src/restic/fuse/dir.go index 0289a982f..a89617e5f 100644 --- a/src/restic/fuse/dir.go +++ b/src/restic/fuse/dir.go @@ -11,6 +11,7 @@ import ( "golang.org/x/net/context" "restic" + "restic/debug" "restic/repository" ) @@ -27,8 +28,10 @@ type dir struct { } func newDir(repo *repository.Repository, node *restic.Node, ownerIsRoot bool) (*dir, error) { + debug.Log("newDir", "new dir for %v (%v)", node.Name, node.Subtree.Str()) tree, err := restic.LoadTree(repo, *node.Subtree) if err != nil { + debug.Log("newDir", " error loading tree %v: %v", node.Subtree.Str(), err) return nil, err } items := make(map[string]*restic.Node) @@ -65,14 +68,17 @@ func replaceSpecialNodes(repo *repository.Repository, node *restic.Node) ([]*res } func newDirFromSnapshot(repo *repository.Repository, snapshot SnapshotWithId, ownerIsRoot bool) (*dir, error) { + debug.Log("newDirFromSnapshot", "new dir for snapshot %v (%v)", snapshot.ID.Str(), snapshot.Tree.Str()) tree, err := restic.LoadTree(repo, *snapshot.Tree) if err != nil { + debug.Log("newDirFromSnapshot", " loadTree(%v) failed: %v", snapshot.ID.Str(), err) return nil, err } items := make(map[string]*restic.Node) for _, n := range tree.Nodes { nodes, err := replaceSpecialNodes(repo, n) if err != nil { + debug.Log("newDirFromSnapshot", " replaceSpecialNodes(%v) failed: %v", n, err) return nil, err } @@ -98,6 +104,7 @@ func newDirFromSnapshot(repo *repository.Repository, snapshot SnapshotWithId, ow } func (d *dir) Attr(ctx context.Context, a *fuse.Attr) error { + debug.Log("dir.Attr", "called") a.Inode = d.inode a.Mode = os.ModeDir | d.node.Mode @@ -112,6 +119,7 @@ func (d *dir) Attr(ctx context.Context, a *fuse.Attr) error { } func (d *dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { + debug.Log("dir.ReadDirAll", "called") ret := make([]fuse.Dirent, 0, len(d.items)) for _, node := range d.items { @@ -136,8 +144,10 @@ func (d *dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { } func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) { + debug.Log("dir.Lookup", "Lookup(%v)", name) node, ok := d.items[name] if !ok { + debug.Log("dir.Lookup", " Lookup(%v) -> not found", name) return nil, fuse.ENOENT } switch node.Type { @@ -148,6 +158,7 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) { case "symlink": return newLink(d.repo, node, d.ownerIsRoot) default: + debug.Log("dir.Lookup", " node %v has unknown type %v", name, node.Type) return nil, fuse.ENOENT } } diff --git a/src/restic/fuse/file.go b/src/restic/fuse/file.go index ce7204249..2f0db5738 100644 --- a/src/restic/fuse/file.go +++ b/src/restic/fuse/file.go @@ -8,6 +8,7 @@ import ( "restic" "restic/backend" + "restic/debug" "restic/pack" "bazil.org/fuse" @@ -47,6 +48,8 @@ var blobPool = sync.Pool{ } func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error) { + debug.Log("newFile", "create new file for %v with %d blobs", node.Name, len(node.Content)) + var bytes uint64 sizes := make([]uint, len(node.Content)) for i, id := range node.Content { size, err := repo.LookupBlobSize(id) @@ -55,6 +58,12 @@ func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error } sizes[i] = size + bytes += uint64(size) + } + + if bytes != node.Size { + debug.Log("newFile", "sizes do not match: node.Size %v != size %v, using real size", node.Size, bytes) + node.Size = bytes } return &file{ @@ -67,6 +76,7 @@ func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error } func (f *file) Attr(ctx context.Context, a *fuse.Attr) error { + debug.Log("file.Attr", "Attr(%v)", f.node.Name) a.Inode = f.node.Inode a.Mode = f.node.Mode a.Size = f.node.Size @@ -84,6 +94,7 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error { } func (f *file) getBlobAt(i int) (blob []byte, err error) { + debug.Log("file.getBlobAt", "getBlobAt(%v, %v)", f.node.Name, i) if f.blobs[i] != nil { return f.blobs[i], nil } @@ -100,6 +111,7 @@ func (f *file) getBlobAt(i int) (blob []byte, err error) { blob, err = f.repo.LoadBlob(pack.Data, f.node.Content[i], buf) if err != nil { + debug.Log("file.getBlobAt", "LoadBlob(%v, %v) failed: %v", f.node.Name, f.node.Content[i], err) return nil, err } f.blobs[i] = blob @@ -108,6 +120,7 @@ func (f *file) getBlobAt(i int) (blob []byte, err error) { } func (f *file) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { + debug.Log("file.Read", "Read(%v), file size %v", req.Size, f.node.Size) offset := req.Offset // Skip blobs before the offset diff --git a/src/restic/fuse/snapshot.go b/src/restic/fuse/snapshot.go index 162873d84..a384e3fb5 100644 --- a/src/restic/fuse/snapshot.go +++ b/src/restic/fuse/snapshot.go @@ -13,6 +13,7 @@ import ( "restic" "restic/backend" + "restic/debug" "restic/repository" "golang.org/x/net/context" @@ -39,6 +40,7 @@ type SnapshotsDir struct { } func NewSnapshotsDir(repo *repository.Repository, ownerIsRoot bool) *SnapshotsDir { + debug.Log("NewSnapshotsDir", "fuse mount initiated") return &SnapshotsDir{ repo: repo, knownSnapshots: make(map[string]SnapshotWithId), @@ -54,10 +56,12 @@ func (sn *SnapshotsDir) Attr(ctx context.Context, attr *fuse.Attr) error { attr.Uid = uint32(os.Getuid()) attr.Gid = uint32(os.Getgid()) } + debug.Log("SnapshotsDir.Attr", "attr is %v", attr) return nil } func (sn *SnapshotsDir) updateCache(ctx context.Context) error { + debug.Log("SnapshotsDir.updateCache", "called") sn.Lock() defer sn.Unlock() @@ -75,10 +79,12 @@ func (sn *SnapshotsDir) get(name string) (snapshot SnapshotWithId, ok bool) { sn.RLock() snapshot, ok = sn.knownSnapshots[name] sn.RUnlock() + debug.Log("SnapshotsDir.get", "get(%s) -> %v %v", name, snapshot, ok) return snapshot, ok } func (sn *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { + debug.Log("SnapshotsDir.ReadDirAll", "called") err := sn.updateCache(ctx) if err != nil { return nil, err @@ -96,21 +102,25 @@ func (sn *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { }) } + debug.Log("SnapshotsDir.ReadDirAll", " -> %d entries", len(ret)) return ret, nil } func (sn *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) { + debug.Log("SnapshotsDir.updateCache", "Lookup(%s)", name) snapshot, ok := sn.get(name) if !ok { // We don't know about it, update the cache err := sn.updateCache(ctx) if err != nil { + debug.Log("SnapshotsDir.updateCache", " Lookup(%s) -> err %v", name, err) return nil, err } snapshot, ok = sn.get(name) if !ok { // We still don't know about it, this time it really doesn't exist + debug.Log("SnapshotsDir.updateCache", " Lookup(%s) -> not found", name) return nil, fuse.ENOENT } }