diff --git a/CHANGELOG.md b/CHANGELOG.md index 92274a1be..a748694ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ Important Changes in 0.X.Y https://github.com/restic/restic/issues/989 https://github.com/restic/restic/pull/993 + * Improved performance for the fuse mount: Listing directories which contain + large files now is significantly faster. + https://github.com/restic/restic/pull/998 + Important Changes in 0.6.1 ========================== diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index 767c21dc0..dde45b789 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -8,7 +8,6 @@ import ( "path" "restic" "strings" - "sync" "time" "restic/backend" @@ -23,12 +22,10 @@ const connLimit = 10 // s3 is a backend which stores the data on an S3 endpoint. type s3 struct { - client *minio.Client - sem *backend.Semaphore - bucketname string - prefix string - cacheMutex sync.RWMutex - cacheObjSize map[string]int64 + client *minio.Client + sem *backend.Semaphore + bucketname string + prefix string backend.Layout } @@ -53,11 +50,10 @@ func Open(cfg Config) (restic.Backend, error) { } be := &s3{ - client: client, - sem: sem, - bucketname: cfg.Bucket, - prefix: cfg.Prefix, - cacheObjSize: make(map[string]int64), + client: client, + sem: sem, + bucketname: cfg.Bucket, + prefix: cfg.Prefix, } client.SetCustomTransport(backend.Transport()) diff --git a/src/restic/fuse/dir.go b/src/restic/fuse/dir.go index 49210bb90..f7920f6ca 100644 --- a/src/restic/fuse/dir.go +++ b/src/restic/fuse/dir.go @@ -24,9 +24,11 @@ type dir struct { inode uint64 node *restic.Node ownerIsRoot bool + + blobsize *BlobSizeCache } -func newDir(ctx context.Context, repo restic.Repository, node *restic.Node, ownerIsRoot bool) (*dir, error) { +func newDir(ctx context.Context, repo restic.Repository, node *restic.Node, ownerIsRoot bool, blobsize *BlobSizeCache) (*dir, error) { debug.Log("new dir for %v (%v)", node.Name, node.Subtree.Str()) tree, err := repo.LoadTree(ctx, *node.Subtree) if err != nil { @@ -44,6 +46,7 @@ func newDir(ctx context.Context, repo restic.Repository, node *restic.Node, owne items: items, inode: node.Inode, ownerIsRoot: ownerIsRoot, + blobsize: blobsize, }, nil } @@ -66,7 +69,7 @@ func replaceSpecialNodes(ctx context.Context, repo restic.Repository, node *rest return tree.Nodes, nil } -func newDirFromSnapshot(ctx context.Context, repo restic.Repository, snapshot SnapshotWithId, ownerIsRoot bool) (*dir, error) { +func newDirFromSnapshot(ctx context.Context, repo restic.Repository, snapshot SnapshotWithId, ownerIsRoot bool, blobsize *BlobSizeCache) (*dir, error) { debug.Log("new dir for snapshot %v (%v)", snapshot.ID.Str(), snapshot.Tree.Str()) tree, err := repo.LoadTree(ctx, *snapshot.Tree) if err != nil { @@ -99,6 +102,7 @@ func newDirFromSnapshot(ctx context.Context, repo restic.Repository, snapshot Sn items: items, inode: inodeFromBackendID(snapshot.ID), ownerIsRoot: ownerIsRoot, + blobsize: blobsize, }, nil } @@ -167,9 +171,9 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) { } switch node.Type { case "dir": - return newDir(ctx, d.repo, node, d.ownerIsRoot) + return newDir(ctx, d.repo, node, d.ownerIsRoot, d.blobsize) case "file": - return newFile(d.repo, node, d.ownerIsRoot) + return newFile(d.repo, node, d.ownerIsRoot, d.blobsize) case "symlink": return newLink(d.repo, node, d.ownerIsRoot) default: diff --git a/src/restic/fuse/file.go b/src/restic/fuse/file.go index ae3ba5a7d..cfb799a82 100644 --- a/src/restic/fuse/file.go +++ b/src/restic/fuse/file.go @@ -41,14 +41,17 @@ type file struct { const defaultBlobSize = 128 * 1024 -func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error) { +func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool, blobsize *BlobSizeCache) (fusefile *file, err error) { debug.Log("create new file for %v with %d blobs", node.Name, len(node.Content)) var bytes uint64 sizes := make([]int, len(node.Content)) for i, id := range node.Content { - size, err := repo.LookupBlobSize(id, restic.DataBlob) - if err != nil { - return nil, err + size, ok := blobsize.Lookup(id) + if !ok { + size, err = repo.LookupBlobSize(id, restic.DataBlob) + if err != nil { + return nil, err + } } sizes[i] = int(size) diff --git a/src/restic/fuse/file_test.go b/src/restic/fuse/file_test.go index dcb959fec..4c4550282 100644 --- a/src/restic/fuse/file_test.go +++ b/src/restic/fuse/file_test.go @@ -108,7 +108,7 @@ func TestFuseFile(t *testing.T) { Size: filesize, Content: content, } - f, err := newFile(repo, node, false) + f, err := newFile(repo, node, false, nil) OK(t, err) attr := fuse.Attr{} diff --git a/src/restic/fuse/snapshot.go b/src/restic/fuse/snapshot.go index 4057301f8..1b22ecd50 100644 --- a/src/restic/fuse/snapshot.go +++ b/src/restic/fuse/snapshot.go @@ -14,10 +14,39 @@ import ( "restic" "restic/debug" + "restic/repository" "golang.org/x/net/context" ) +// BlobSizeCache caches the size of blobs in the repo. +type BlobSizeCache struct { + m map[restic.ID]uint +} + +// NewBlobSizeCache returns a new blob size cache containing all entries from midx. +func NewBlobSizeCache(midx *repository.MasterIndex) *BlobSizeCache { + m := make(map[restic.ID]uint, 1000) + for _, idx := range midx.All() { + for pb := range idx.Each(nil) { + m[pb.ID] = pb.Length + } + } + return &BlobSizeCache{ + m: m, + } +} + +// Lookup returns the size of the blob id. +func (c *BlobSizeCache) Lookup(id restic.ID) (size uint, found bool) { + if c == nil { + return 0, false + } + + size, found = c.m[id] + return size, found +} + type SnapshotWithId struct { *restic.Snapshot restic.ID @@ -36,6 +65,8 @@ type SnapshotsDir struct { tags []string host string + blobsize *BlobSizeCache + // knownSnapshots maps snapshot timestamp to the snapshot sync.RWMutex knownSnapshots map[string]SnapshotWithId @@ -53,6 +84,7 @@ func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool, paths []string, t host: host, knownSnapshots: make(map[string]SnapshotWithId), processed: restic.NewIDSet(), + blobsize: NewBlobSizeCache(repo.Index().(*repository.MasterIndex)), } } @@ -158,5 +190,5 @@ func (sn *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error } } - return newDirFromSnapshot(ctx, sn.repo, snapshot, sn.ownerIsRoot) + return newDirFromSnapshot(ctx, sn.repo, snapshot, sn.ownerIsRoot, sn.blobsize) }