From 4df887406ff42320ca5d157e66847c3fba36ae5b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 19 May 2024 12:41:56 +0200 Subject: [PATCH] repository: inline MasterIndex interface into Repository interface --- cmd/restic/cmd_cat.go | 2 +- cmd/restic/cmd_copy.go | 6 +-- cmd/restic/cmd_debug.go | 2 +- cmd/restic/cmd_find.go | 6 +-- cmd/restic/cmd_recover.go | 2 +- cmd/restic/cmd_stats.go | 4 +- cmd/restic/integration_helpers_test.go | 4 +- internal/archiver/archiver.go | 4 +- internal/checker/checker.go | 24 +++++------ internal/index/master_index_test.go | 2 +- internal/pack/pack.go | 4 +- internal/repository/check.go | 3 +- internal/repository/prune.go | 19 ++++----- internal/repository/repack.go | 2 +- internal/repository/repack_test.go | 13 ++---- internal/repository/repair_index.go | 4 +- internal/repository/repair_pack.go | 2 +- internal/repository/repair_pack_test.go | 6 +-- internal/repository/repository.go | 33 +++++++++++---- internal/repository/repository_test.go | 4 +- internal/restic/repository.go | 55 ++++++++++++++----------- internal/restorer/restorer.go | 2 +- 22 files changed, 109 insertions(+), 94 deletions(-) diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index 8d11a9dc4..e776b67a1 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -168,7 +168,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { for _, t := range []restic.BlobType{restic.DataBlob, restic.TreeBlob} { bh := restic.BlobHandle{ID: id, Type: t} - if !repo.Index().Has(bh) { + if !repo.HasBlob(bh) { continue } diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index ad6c58a25..26b16a374 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -187,7 +187,7 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep packList := restic.NewIDSet() enqueue := func(h restic.BlobHandle) { - pb := srcRepo.Index().Lookup(h) + pb := srcRepo.LookupBlob(h) copyBlobs.Insert(h) for _, p := range pb { packList.Insert(p.PackID) @@ -202,7 +202,7 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep // Do we already have this tree blob? treeHandle := restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob} - if !dstRepo.Index().Has(treeHandle) { + if !dstRepo.HasBlob(treeHandle) { // copy raw tree bytes to avoid problems if the serialization changes enqueue(treeHandle) } @@ -212,7 +212,7 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep // Copy the blobs for this file. for _, blobID := range entry.Content { h := restic.BlobHandle{Type: restic.DataBlob, ID: blobID} - if !dstRepo.Index().Has(h) { + if !dstRepo.HasBlob(h) { enqueue(h) } } diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 9fb6969d0..7b0cdb53e 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -492,7 +492,7 @@ func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repo blobsLoaded := false // examine all data the indexes have for the pack file - for b := range repo.Index().ListPacks(ctx, restic.NewIDSet(id)) { + for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) { blobs := b.Blobs if len(blobs) == 0 { continue diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 81df0ab98..7ad8886c8 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -465,7 +465,7 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc // remember which packs were found in the index indexPackIDs := make(map[string]struct{}) - err := f.repo.Index().Each(wctx, func(pb restic.PackedBlob) { + err := f.repo.ListBlobs(wctx, func(pb restic.PackedBlob) { idStr := pb.PackID.String() // keep entry in packIDs as Each() returns individual index entries matchingID := false @@ -503,15 +503,13 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc } func (f *Finder) findObjectPack(id string, t restic.BlobType) { - idx := f.repo.Index() - rid, err := restic.ParseID(id) if err != nil { Printf("Note: cannot find pack for object '%s', unable to parse ID: %v\n", id, err) return } - blobs := idx.Lookup(restic.BlobHandle{ID: rid, Type: t}) + blobs := f.repo.LookupBlob(restic.BlobHandle{ID: rid, Type: t}) if len(blobs) == 0 { Printf("Object %s not found in the index\n", rid.Str()) return diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index debaa4e5b..726f1bf65 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -61,7 +61,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { // tree. If it is not referenced, we have a root tree. trees := make(map[restic.ID]bool) - err = repo.Index().Each(ctx, func(blob restic.PackedBlob) { + err = repo.ListBlobs(ctx, func(blob restic.PackedBlob) { if blob.Type == restic.TreeBlob { trees[blob.Blob.ID] = false } diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index a7891e5b0..3bec18f4c 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -124,7 +124,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args if opts.countMode == countModeRawData { // the blob handles have been collected, but not yet counted for blobHandle := range stats.blobs { - pbs := repo.Index().Lookup(blobHandle) + pbs := repo.LookupBlob(blobHandle) if len(pbs) == 0 { return fmt.Errorf("blob %v not found", blobHandle) } @@ -378,7 +378,7 @@ func statsDebugBlobs(ctx context.Context, repo restic.Repository) ([restic.NumBl hist[i] = newSizeHistogram(2 * chunker.MaxSize) } - err := repo.Index().Each(ctx, func(pb restic.PackedBlob) { + err := repo.ListBlobs(ctx, func(pb restic.PackedBlob) { hist[pb.Type].Add(uint64(pb.Length)) }) diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index 2812eda6d..978deab3d 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -252,7 +252,7 @@ func listTreePacks(gopts GlobalOptions, t *testing.T) restic.IDSet { rtest.OK(t, r.LoadIndex(ctx, nil)) treePacks := restic.NewIDSet() - rtest.OK(t, r.Index().Each(ctx, func(pb restic.PackedBlob) { + rtest.OK(t, r.ListBlobs(ctx, func(pb restic.PackedBlob) { if pb.Type == restic.TreeBlob { treePacks.Insert(pb.PackID) } @@ -280,7 +280,7 @@ func removePacksExcept(gopts GlobalOptions, t testing.TB, keep restic.IDSet, rem rtest.OK(t, r.LoadIndex(ctx, nil)) treePacks := restic.NewIDSet() - rtest.OK(t, r.Index().Each(ctx, func(pb restic.PackedBlob) { + rtest.OK(t, r.ListBlobs(ctx, func(pb restic.PackedBlob) { if pb.Type == restic.TreeBlob { treePacks.Insert(pb.PackID) } diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index c1f73eea6..10034afa1 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -276,7 +276,7 @@ func (arch *Archiver) loadSubtree(ctx context.Context, node *restic.Node) (*rest } func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error { - if arch.Repo.Index().Has(restic.BlobHandle{ID: id, Type: restic.TreeBlob}) { + if arch.Repo.HasBlob(restic.BlobHandle{ID: id, Type: restic.TreeBlob}) { err = errors.Errorf("tree %v could not be loaded; the repository could be damaged: %v", id, err) } else { err = errors.Errorf("tree %v is not known; the repository could be damaged, run `repair index` to try to repair it", id) @@ -390,7 +390,7 @@ func (fn *FutureNode) take(ctx context.Context) futureNodeResult { func (arch *Archiver) allBlobsPresent(previous *restic.Node) bool { // check if all blobs are contained in index for _, id := range previous.Content { - if !arch.Repo.Index().Has(restic.BlobHandle{ID: id, Type: restic.DataBlob}) { + if !arch.Repo.HasBlob(restic.BlobHandle{ID: id, Type: restic.DataBlob}) { return false } } diff --git a/internal/checker/checker.go b/internal/checker/checker.go index dc83aef5b..09b1dd7eb 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -91,9 +91,9 @@ func (c *Checker) LoadSnapshots(ctx context.Context) error { return err } -func computePackTypes(ctx context.Context, idx restic.MasterIndex) (map[restic.ID]restic.BlobType, error) { +func computePackTypes(ctx context.Context, idx restic.ListBlobser) (map[restic.ID]restic.BlobType, error) { packs := make(map[restic.ID]restic.BlobType) - err := idx.Each(ctx, func(pb restic.PackedBlob) { + err := idx.ListBlobs(ctx, func(pb restic.PackedBlob) { tpe, exists := packs[pb.PackID] if exists { if pb.Type != tpe { @@ -177,12 +177,18 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e return hints, append(errs, err) } + err = c.repo.SetIndex(c.masterIndex) + if err != nil { + debug.Log("SetIndex returned error: %v", err) + errs = append(errs, err) + } + // compute pack size using index entries - c.packs, err = pack.Size(ctx, c.masterIndex, false) + c.packs, err = pack.Size(ctx, c.repo, false) if err != nil { return hints, append(errs, err) } - packTypes, err := computePackTypes(ctx, c.masterIndex) + packTypes, err := computePackTypes(ctx, c.repo) if err != nil { return hints, append(errs, err) } @@ -203,12 +209,6 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e } } - err = c.repo.SetIndex(c.masterIndex) - if err != nil { - debug.Log("SetIndex returned error: %v", err) - errs = append(errs, err) - } - return hints, errs } @@ -488,7 +488,7 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, er ctx, cancel := context.WithCancel(ctx) defer cancel() - err = c.repo.Index().Each(ctx, func(blob restic.PackedBlob) { + err = c.repo.ListBlobs(ctx, func(blob restic.PackedBlob) { h := restic.BlobHandle{ID: blob.ID, Type: blob.Type} if !c.blobRefs.M.Has(h) { debug.Log("blob %v not referenced", h) @@ -573,7 +573,7 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p } // push packs to ch - for pbs := range c.repo.Index().ListPacks(ctx, packSet) { + for pbs := range c.repo.ListPacksFromIndex(ctx, packSet) { size := packs[pbs.PackID] debug.Log("listed %v", pbs.PackID) select { diff --git a/internal/index/master_index_test.go b/internal/index/master_index_test.go index c3560a7fb..36a028768 100644 --- a/internal/index/master_index_test.go +++ b/internal/index/master_index_test.go @@ -362,7 +362,7 @@ func testIndexSave(t *testing.T, version uint) { t.Fatal(err) } - err = repo.Index().Save(context.TODO(), repo, nil, nil, restic.MasterIndexSaveOpts{}) + err = repo.SaveIndex(context.TODO(), nil, nil, restic.MasterIndexSaveOpts{}) if err != nil { t.Fatalf("unable to save new index: %v", err) } diff --git a/internal/pack/pack.go b/internal/pack/pack.go index 7d8d87e71..57957ce91 100644 --- a/internal/pack/pack.go +++ b/internal/pack/pack.go @@ -389,10 +389,10 @@ func CalculateHeaderSize(blobs []restic.Blob) int { // If onlyHdr is set to true, only the size of the header is returned // Note that this function only gives correct sizes, if there are no // duplicates in the index. -func Size(ctx context.Context, mi restic.MasterIndex, onlyHdr bool) (map[restic.ID]int64, error) { +func Size(ctx context.Context, mi restic.ListBlobser, onlyHdr bool) (map[restic.ID]int64, error) { packSize := make(map[restic.ID]int64) - err := mi.Each(ctx, func(blob restic.PackedBlob) { + err := mi.ListBlobs(ctx, func(blob restic.PackedBlob) { size, ok := packSize[blob.PackID] if !ok { size = headerSize diff --git a/internal/repository/check.go b/internal/repository/check.go index 8018f4902..05605db86 100644 --- a/internal/repository/check.go +++ b/internal/repository/check.go @@ -158,11 +158,10 @@ func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs []re errs = append(errs, errors.Errorf("pack header size does not match, want %v, got %v", idxHdrSize, hdrSize)) } - idx := r.Index() for _, blob := range blobs { // Check if blob is contained in index and position is correct idxHas := false - for _, pb := range idx.Lookup(blob.BlobHandle) { + for _, pb := range r.LookupBlob(blob.BlobHandle) { if pb.PackID == id && pb.Blob == blob { idxHas = true break diff --git a/internal/repository/prune.go b/internal/repository/prune.go index 479439e6a..66eab28b2 100644 --- a/internal/repository/prune.go +++ b/internal/repository/prune.go @@ -7,7 +7,6 @@ import ( "sort" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" @@ -67,7 +66,7 @@ type PrunePlan struct { removePacks restic.IDSet // packs to remove ignorePacks restic.IDSet // packs to ignore when rebuilding the index - repo restic.Repository + repo *Repository stats PruneStats opts PruneOptions } @@ -89,7 +88,7 @@ type packInfoWithID struct { // PlanPrune selects which files to rewrite and which to delete and which blobs to keep. // Also some summary statistics are returned. -func PlanPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, getUsedBlobs func(ctx context.Context, repo restic.Repository) (usedBlobs restic.CountedBlobSet, err error), printer progress.Printer) (*PrunePlan, error) { +func PlanPrune(ctx context.Context, opts PruneOptions, repo *Repository, getUsedBlobs func(ctx context.Context, repo restic.Repository) (usedBlobs restic.CountedBlobSet, err error), printer progress.Printer) (*PrunePlan, error) { var stats PruneStats if opts.UnsafeRecovery { @@ -109,7 +108,7 @@ func PlanPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, g } printer.P("searching used packs...\n") - keepBlobs, indexPack, err := packInfoFromIndex(ctx, repo.Index(), usedBlobs, &stats, printer) + keepBlobs, indexPack, err := packInfoFromIndex(ctx, repo, usedBlobs, &stats, printer) if err != nil { return nil, err } @@ -124,7 +123,7 @@ func PlanPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, g blobCount := keepBlobs.Len() // when repacking, we do not want to keep blobs which are // already contained in kept packs, so delete them from keepBlobs - err := repo.Index().Each(ctx, func(blob restic.PackedBlob) { + err := repo.ListBlobs(ctx, func(blob restic.PackedBlob) { if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) { return } @@ -151,11 +150,11 @@ func PlanPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, g return &plan, nil } -func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs restic.CountedBlobSet, stats *PruneStats, printer progress.Printer) (restic.CountedBlobSet, map[restic.ID]packInfo, error) { +func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs restic.CountedBlobSet, stats *PruneStats, printer progress.Printer) (restic.CountedBlobSet, map[restic.ID]packInfo, error) { // iterate over all blobs in index to find out which blobs are duplicates // The counter in usedBlobs describes how many instances of the blob exist in the repository index // Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist - err := idx.Each(ctx, func(blob restic.PackedBlob) { + err := idx.ListBlobs(ctx, func(blob restic.PackedBlob) { bh := blob.BlobHandle count, ok := usedBlobs[bh] if ok { @@ -205,7 +204,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re hasDuplicates := false // iterate over all blobs in index to generate packInfo - err = idx.Each(ctx, func(blob restic.PackedBlob) { + err = idx.ListBlobs(ctx, func(blob restic.PackedBlob) { ip := indexPack[blob.PackID] // Set blob type if not yet set @@ -260,7 +259,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re // - if there are no used blobs in a pack, possibly mark duplicates as "unused" if hasDuplicates { // iterate again over all blobs in index (this is pretty cheap, all in-mem) - err = idx.Each(ctx, func(blob restic.PackedBlob) { + err = idx.ListBlobs(ctx, func(blob restic.PackedBlob) { bh := blob.BlobHandle count, ok := usedBlobs[bh] // skip non-duplicate, aka. normal blobs @@ -581,7 +580,7 @@ func (plan *PrunePlan) Execute(ctx context.Context, printer progress.Printer) (e if plan.opts.UnsafeRecovery { printer.P("deleting index files\n") - indexFiles := repo.Index().(*index.MasterIndex).IDs() + indexFiles := repo.idx.IDs() err = deleteFiles(ctx, false, repo, indexFiles, restic.IndexFile, printer) if err != nil { return errors.Fatalf("%s", err) diff --git a/internal/repository/repack.go b/internal/repository/repack.go index 5bedcfa56..8c9ca28bb 100644 --- a/internal/repository/repack.go +++ b/internal/repository/repack.go @@ -54,7 +54,7 @@ func repack(ctx context.Context, repo restic.Repository, dstRepo restic.Reposito downloadQueue := make(chan restic.PackBlobs) wg.Go(func() error { defer close(downloadQueue) - for pbs := range repo.Index().ListPacks(wgCtx, packs) { + for pbs := range repo.ListPacksFromIndex(wgCtx, packs) { var packBlobs []restic.Blob keepMutex.Lock() // filter out unnecessary blobs diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index 3fd56ccb1..e0f1b4254 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -145,9 +145,8 @@ func listFiles(t *testing.T, repo restic.Lister, tpe backend.FileType) restic.ID func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSet) restic.IDSet { packs := restic.NewIDSet() - idx := repo.Index() for h := range blobs { - list := idx.Lookup(h) + list := repo.LookupBlob(h) if len(list) == 0 { t.Fatal("Failed to find blob", h.ID.Str(), "with type", h.Type) } @@ -195,7 +194,7 @@ func rebuildIndex(t *testing.T, repo restic.Repository) { }) rtest.OK(t, err) - err = repo.Index().Save(context.TODO(), repo, restic.NewIDSet(), obsoleteIndexes, restic.MasterIndexSaveOpts{}) + err = repo.SaveIndex(context.TODO(), restic.NewIDSet(), obsoleteIndexes, restic.MasterIndexSaveOpts{}) rtest.OK(t, err) } @@ -252,10 +251,8 @@ func testRepack(t *testing.T, version uint) { } } - idx := repo.Index() - for h := range keepBlobs { - list := idx.Lookup(h) + list := repo.LookupBlob(h) if len(list) == 0 { t.Errorf("unable to find blob %v in repo", h.ID.Str()) continue @@ -318,10 +315,8 @@ func testRepackCopy(t *testing.T, version uint) { rebuildIndex(t, dstRepo) reloadIndex(t, dstRepo) - idx := dstRepo.Index() - for h := range keepBlobs { - list := idx.Lookup(h) + list := dstRepo.LookupBlob(h) if len(list) == 0 { t.Errorf("unable to find blob %v in repo", h.ID.Str()) continue diff --git a/internal/repository/repair_index.go b/internal/repository/repair_index.go index a6e732b44..a7d94fcf8 100644 --- a/internal/repository/repair_index.go +++ b/internal/repository/repair_index.go @@ -54,7 +54,7 @@ func RepairIndex(ctx context.Context, repo *Repository, opts RepairIndexOptions, if err != nil { return err } - packSizeFromIndex, err = pack.Size(ctx, repo.Index(), false) + packSizeFromIndex, err = pack.Size(ctx, repo, false) if err != nil { return err } @@ -115,7 +115,7 @@ func rebuildIndexFiles(ctx context.Context, repo restic.Repository, removePacks printer.P("rebuilding index\n") bar := printer.NewCounter("packs processed") - return repo.Index().Save(ctx, repo, removePacks, extraObsolete, restic.MasterIndexSaveOpts{ + return repo.SaveIndex(ctx, removePacks, extraObsolete, restic.MasterIndexSaveOpts{ SaveProgress: bar, DeleteProgress: func() *progress.Counter { return printer.NewCounter("old indexes deleted") diff --git a/internal/repository/repair_pack.go b/internal/repository/repair_pack.go index a0bd56012..cac7aac10 100644 --- a/internal/repository/repair_pack.go +++ b/internal/repository/repair_pack.go @@ -21,7 +21,7 @@ func RepairPacks(ctx context.Context, repo restic.Repository, ids restic.IDSet, wg.Go(func() error { // examine all data the indexes have for the pack file - for b := range repo.Index().ListPacks(wgCtx, ids) { + for b := range repo.ListPacksFromIndex(wgCtx, ids) { blobs := b.Blobs if len(blobs) == 0 { printer.E("no blobs found for pack %v", b.PackID) diff --git a/internal/repository/repair_pack_test.go b/internal/repository/repair_pack_test.go index 7acdc646e..28a5525a2 100644 --- a/internal/repository/repair_pack_test.go +++ b/internal/repository/repair_pack_test.go @@ -18,7 +18,7 @@ import ( func listBlobs(repo restic.Repository) restic.BlobSet { blobs := restic.NewBlobSet() - _ = repo.Index().Each(context.TODO(), func(pb restic.PackedBlob) { + _ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) { blobs.Insert(pb.BlobHandle) }) return blobs @@ -68,7 +68,7 @@ func testRepairBrokenPack(t *testing.T, version uint) { // find blob that starts at offset 0 var damagedBlob restic.BlobHandle - for blobs := range repo.Index().ListPacks(context.TODO(), restic.NewIDSet(damagedID)) { + for blobs := range repo.ListPacksFromIndex(context.TODO(), restic.NewIDSet(damagedID)) { for _, blob := range blobs.Blobs { if blob.Offset == 0 { damagedBlob = blob.BlobHandle @@ -91,7 +91,7 @@ func testRepairBrokenPack(t *testing.T, version uint) { // all blobs in the file are broken damagedBlobs := restic.NewBlobSet() - for blobs := range repo.Index().ListPacks(context.TODO(), restic.NewIDSet(damagedID)) { + for blobs := range repo.ListPacksFromIndex(context.TODO(), restic.NewIDSet(damagedID)) { for _, blob := range blobs.Blobs { damagedBlobs.Insert(blob.BlobHandle) } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 534edc9fd..4aa8106ab 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -300,11 +300,6 @@ func (r *Repository) loadBlob(ctx context.Context, blobs []restic.PackedBlob, bu return nil, errors.Errorf("loading %v from %v packs failed", blobs[0].BlobHandle, len(blobs)) } -// LookupBlobSize returns the size of blob id. -func (r *Repository) LookupBlobSize(id restic.ID, tpe restic.BlobType) (uint, bool) { - return r.idx.LookupSize(restic.BlobHandle{ID: id, Type: tpe}) -} - func (r *Repository) getZstdEncoder() *zstd.Encoder { r.allocEnc.Do(func() { level := zstd.SpeedDefault @@ -583,9 +578,31 @@ func (r *Repository) Connections() uint { return r.be.Connections() } -// Index returns the currently used MasterIndex. -func (r *Repository) Index() restic.MasterIndex { - return r.idx +func (r *Repository) HasBlob(bh restic.BlobHandle) bool { + return r.idx.Has(bh) +} + +func (r *Repository) LookupBlob(bh restic.BlobHandle) []restic.PackedBlob { + return r.idx.Lookup(bh) +} + +// LookupBlobSize returns the size of blob id. +func (r *Repository) LookupBlobSize(id restic.ID, tpe restic.BlobType) (uint, bool) { + return r.idx.LookupSize(restic.BlobHandle{ID: id, Type: tpe}) +} + +func (r *Repository) SaveIndex(ctx context.Context, excludePacks restic.IDSet, extraObsolete restic.IDs, opts restic.MasterIndexSaveOpts) error { + return r.idx.Save(ctx, r, excludePacks, extraObsolete, opts) +} + +// ListBlobs runs fn on all blobs known to the index. When the context is cancelled, +// the index iteration returns immediately with ctx.Err(). This blocks any modification of the index. +func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackedBlob)) error { + return r.idx.Each(ctx, fn) +} + +func (r *Repository) ListPacksFromIndex(ctx context.Context, packs restic.IDSet) <-chan restic.PackBlobs { + return r.idx.ListPacks(ctx, packs) } // SetIndex instructs the repository to use the given index. diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index f0d3ae486..31a588f62 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -161,7 +161,7 @@ func TestLoadBlobBroken(t *testing.T) { data, err := repo.LoadBlob(context.TODO(), restic.TreeBlob, id, nil) rtest.OK(t, err) rtest.Assert(t, bytes.Equal(buf, data), "data mismatch") - pack := repo.Index().Lookup(restic.BlobHandle{Type: restic.TreeBlob, ID: id})[0].PackID + pack := repo.LookupBlob(restic.BlobHandle{Type: restic.TreeBlob, ID: id})[0].PackID rtest.Assert(t, c.Has(backend.Handle{Type: restic.PackFile, Name: pack.String()}), "expected tree pack to be cached") } @@ -439,7 +439,7 @@ func TestListPack(t *testing.T) { repo.UseCache(c) // Forcibly cache pack file - packID := repo.Index().Lookup(restic.BlobHandle{Type: restic.TreeBlob, ID: id})[0].PackID + packID := repo.LookupBlob(restic.BlobHandle{Type: restic.TreeBlob, ID: id})[0].PackID rtest.OK(t, be.Load(context.TODO(), backend.Handle{Type: restic.PackFile, IsMetadata: true, Name: packID.String()}, 0, 0, func(rd io.Reader) error { return nil })) // Get size to list pack diff --git a/internal/restic/repository.go b/internal/restic/repository.go index bc0ec2d43..ee32beb7e 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -18,17 +18,36 @@ var ErrInvalidData = errors.New("invalid data returned") type Repository interface { // Connections returns the maximum number of concurrent backend operations Connections() uint - + Config() Config + PackSize() uint Key() *crypto.Key - Index() MasterIndex LoadIndex(context.Context, *progress.Counter) error ClearIndex() SetIndex(MasterIndex) error + SaveIndex(ctx context.Context, excludePacks IDSet, extraObsolete IDs, opts MasterIndexSaveOpts) error + + HasBlob(BlobHandle) bool + LookupBlob(BlobHandle) []PackedBlob LookupBlobSize(ID, BlobType) (uint, bool) - Config() Config - PackSize() uint + // ListBlobs runs fn on all blobs known to the index. When the context is cancelled, + // the index iteration returns immediately with ctx.Err(). This blocks any modification of the index. + ListBlobs(ctx context.Context, fn func(PackedBlob)) error + ListPacksFromIndex(ctx context.Context, packs IDSet) <-chan PackBlobs + // ListPack returns the list of blobs saved in the pack id and the length of + // the pack header. + ListPack(context.Context, ID, int64) ([]Blob, uint32, error) + + LoadBlob(context.Context, BlobType, ID, []byte) ([]byte, error) + LoadBlobsFromPack(ctx context.Context, packID ID, blobs []Blob, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error + + // StartPackUploader start goroutines to upload new pack files. The errgroup + // is used to immediately notify about an upload error. Flush() will also return + // that error. + StartPackUploader(ctx context.Context, wg *errgroup.Group) + SaveBlob(context.Context, BlobType, []byte, ID, bool) (ID, bool, int, error) + Flush(context.Context) error // List calls the function fn for each file of type t in the repository. // When an error is returned by fn, processing stops and List() returns the @@ -36,31 +55,15 @@ type Repository interface { // // The function fn is called in the same Goroutine List() was called from. List(ctx context.Context, t FileType, fn func(ID, int64) error) error - - // ListPack returns the list of blobs saved in the pack id and the length of - // the pack header. - ListPack(context.Context, ID, int64) ([]Blob, uint32, error) - - LoadBlob(context.Context, BlobType, ID, []byte) ([]byte, error) - LoadBlobsFromPack(ctx context.Context, packID ID, blobs []Blob, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error - SaveBlob(context.Context, BlobType, []byte, ID, bool) (ID, bool, int, error) - - // StartPackUploader start goroutines to upload new pack files. The errgroup - // is used to immediately notify about an upload error. Flush() will also return - // that error. - StartPackUploader(ctx context.Context, wg *errgroup.Group) - Flush(context.Context) error - + // LoadRaw reads all data stored in the backend for the file with id and filetype t. + // If the backend returns data that does not match the id, then the buffer is returned + // along with an error that is a restic.ErrInvalidData error. + LoadRaw(ctx context.Context, t FileType, id ID) (data []byte, err error) // LoadUnpacked loads and decrypts the file with the given type and ID. LoadUnpacked(ctx context.Context, t FileType, id ID) (data []byte, err error) SaveUnpacked(context.Context, FileType, []byte) (ID, error) // RemoveUnpacked removes a file from the repository. This will eventually be restricted to deleting only snapshots. RemoveUnpacked(ctx context.Context, t FileType, id ID) error - - // LoadRaw reads all data stored in the backend for the file with id and filetype t. - // If the backend returns data that does not match the id, then the buffer is returned - // along with an error that is a restic.ErrInvalidData error. - LoadRaw(ctx context.Context, t FileType, id ID) (data []byte, err error) } type FileType = backend.FileType @@ -141,3 +144,7 @@ type Unpacked interface { SaverUnpacked RemoverUnpacked } + +type ListBlobser interface { + ListBlobs(ctx context.Context, fn func(PackedBlob)) error +} diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 9f41f5cf2..721330a8c 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -240,7 +240,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { } idx := NewHardlinkIndex[string]() - filerestorer := newFileRestorer(dst, res.repo.LoadBlobsFromPack, res.repo.Index().Lookup, + filerestorer := newFileRestorer(dst, res.repo.LoadBlobsFromPack, res.repo.LookupBlob, res.repo.Connections(), res.sparse, res.progress) filerestorer.Error = res.Error