clitools
/
restic
mirror of https://github.com/restic/restic.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Use LRU cache in restic dump

This commit is contained in:
greatroar 2021-09-24 15:38:23 +02:00
parent 718966a81a
commit fe04d024c7
6 changed files with 55 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/restic/cmd_dump.go
View File

@ -95,7 +95,8 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
if node.Name == pathComponents[0] {
switch {
case l == 1 && dump.IsFile(node):
return dump.GetNodeData(ctx, os.Stdout, repo, node)
cache := dump.NewCache()
return dump.WriteNodeData(ctx, os.Stdout, repo, node, cache)
case l > 1 && dump.IsDir(node):
subtree, err := repo.LoadTree(ctx, *node.Subtree)
if err != nil {

13
internal/bloblru/cache.go
View File

@ -42,7 +42,9 @@ func New(size int) *Cache {
return c
}
func (c *Cache) Add(id restic.ID, blob []byte) {
// Add adds key id with value blob to c.
// It may return an evicted buffer for reuse.
func (c *Cache) Add(id restic.ID, blob []byte) (old []byte) {
debug.Log("bloblru.Cache: add %v", id)
size := len(blob) + overhead
@ -62,11 +64,18 @@ func (c *Cache) Add(id restic.ID, blob []byte) {
// This loop takes at most min(maxEntries, maxchunksize/overhead)
// iterations.
for size > c.free {
c.c.RemoveOldest()
_, val, _ := c.c.RemoveOldest()
b := val.([]byte)
if len(b) > len(old) {
// We can only return one buffer, so pick the largest.
old = b
}
}
c.c.Add(key, blob)
c.free -= size
return old
}
func (c *Cache) Get(id restic.ID) ([]byte, bool) {

27
internal/dump/common.go
View File

@ -5,6 +5,7 @@ import (
"io"
"path"
"github.com/restic/restic/internal/bloblru"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/walker"
@ -16,6 +17,14 @@ type dumper interface {
dumpNode(ctx context.Context, node *restic.Node, repo restic.Repository) error
}
// WriteDump will write the contents of the given tree to the given destination.
// It will loop over all nodes in the tree and dump them recursively.
type WriteDump func(ctx context.Context, repo restic.Repository, tree *restic.Tree, rootPath string, dst io.Writer) error
func NewCache() *bloblru.Cache {
return bloblru.New(64 << 20)
}
func writeDump(ctx context.Context, repo restic.Repository, tree *restic.Tree, rootPath string, dmp dumper) error {
for _, rootNode := range tree.Nodes {
rootNode.Path = rootPath
@ -67,20 +76,24 @@ func dumpTree(ctx context.Context, repo restic.Repository, rootNode *restic.Node
return err
}
// GetNodeData will write the contents of the node to the given output.
func GetNodeData(ctx context.Context, output io.Writer, repo restic.Repository, node *restic.Node) error {
// WriteNodeData writes the contents of the node to the given Writer.
func WriteNodeData(ctx context.Context, w io.Writer, repo restic.Repository, node *restic.Node, cache *bloblru.Cache) error {
var (
buf []byte
err error
)
for _, id := range node.Content {
buf, err = repo.LoadBlob(ctx, restic.DataBlob, id, buf)
if err != nil {
return err
blob, ok := cache.Get(id)
if !ok {
blob, err = repo.LoadBlob(ctx, restic.DataBlob, id, buf)
if err != nil {
return err
}
buf = cache.Add(id, blob) // Reuse evicted buffer.
}
_, err = output.Write(buf)
if err != nil {
if _, err := w.Write(blob); err != nil {
return errors.Wrap(err, "Write")
}
}

2
internal/dump/common_test.go
View File

@ -3,7 +3,6 @@ package dump
import (
"bytes"
"context"
"io"
"testing"
"github.com/restic/restic/internal/archiver"
@ -28,7 +27,6 @@ func prepareTempdirRepoSrc(t testing.TB, src archiver.TestDir) (tempdir string,
}
type CheckDump func(t *testing.T, testDir string, testDump *bytes.Buffer) error
type WriteDump func(ctx context.Context, repo restic.Repository, tree *restic.Tree, rootPath string, dst io.Writer) error
func WriteTest(t *testing.T, wd WriteDump, cd CheckDump) {
tests := []struct {

18
internal/dump/tar.go
View File

@ -8,25 +8,29 @@ import (
"path/filepath"
"strings"
"github.com/restic/restic/internal/bloblru"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
type tarDumper struct {
w *tar.Writer
cache *bloblru.Cache
w *tar.Writer
}
// Statically ensure that tarDumper implements dumper.
var _ dumper = tarDumper{}
var _ dumper = &tarDumper{}
// WriteTar will write the contents of the given tree, encoded as a tar to the given destination.
func WriteTar(ctx context.Context, repo restic.Repository, tree *restic.Tree, rootPath string, dst io.Writer) error {
dmp := tarDumper{w: tar.NewWriter(dst)}
dmp := &tarDumper{
cache: NewCache(),
w: tar.NewWriter(dst),
}
return writeDump(ctx, repo, tree, rootPath, dmp)
}
func (dmp tarDumper) Close() error {
func (dmp *tarDumper) Close() error {
return dmp.w.Close()
}
@ -39,7 +43,7 @@ const (
cISVTX = 0o1000 // Save text (sticky bit)
)
func (dmp tarDumper) dumpNode(ctx context.Context, node *restic.Node, repo restic.Repository) error {
func (dmp *tarDumper) dumpNode(ctx context.Context, node *restic.Node, repo restic.Repository) error {
relPath, err := filepath.Rel("/", node.Path)
if err != nil {
return err
@ -90,7 +94,7 @@ func (dmp tarDumper) dumpNode(ctx context.Context, node *restic.Node, repo resti
return errors.Wrap(err, "TarHeader")
}
return GetNodeData(ctx, dmp.w, repo, node)
return WriteNodeData(ctx, dmp.w, repo, node, dmp.cache)
}
func parseXattrs(xattrs []restic.ExtendedAttribute) map[string]string {

18
internal/dump/zip.go
View File

@ -6,29 +6,33 @@ import (
"io"
"path/filepath"
"github.com/restic/restic/internal/bloblru"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
type zipDumper struct {
w *zip.Writer
cache *bloblru.Cache
w *zip.Writer
}
// Statically ensure that zipDumper implements dumper.
var _ dumper = zipDumper{}
var _ dumper = &zipDumper{}
// WriteZip will write the contents of the given tree, encoded as a zip to the given destination.
func WriteZip(ctx context.Context, repo restic.Repository, tree *restic.Tree, rootPath string, dst io.Writer) error {
dmp := zipDumper{w: zip.NewWriter(dst)}
dmp := &zipDumper{
cache: NewCache(),
w: zip.NewWriter(dst),
}
return writeDump(ctx, repo, tree, rootPath, dmp)
}
func (dmp zipDumper) Close() error {
func (dmp *zipDumper) Close() error {
return dmp.w.Close()
}
func (dmp zipDumper) dumpNode(ctx context.Context, node *restic.Node, repo restic.Repository) error {
func (dmp *zipDumper) dumpNode(ctx context.Context, node *restic.Node, repo restic.Repository) error {
relPath, err := filepath.Rel("/", node.Path)
if err != nil {
return err
@ -58,5 +62,5 @@ func (dmp zipDumper) dumpNode(ctx context.Context, node *restic.Node, repo resti
return nil
}
return GetNodeData(ctx, w, repo, node)
return WriteNodeData(ctx, w, repo, node, dmp.cache)
}