From 2e3f1c08c5ade338346c749a832bb6dd98795218 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 12 Jun 2022 14:43:43 +0200 Subject: [PATCH] repository: split index into a separate package --- cmd/restic/cmd_debug.go | 3 +- cmd/restic/cmd_list.go | 4 +-- cmd/restic/cmd_prune.go | 3 +- cmd/restic/cmd_rebuild_index.go | 5 +-- cmd/restic/integration_test.go | 7 ++-- internal/archiver/blob_saver_test.go | 6 ++-- internal/checker/checker.go | 7 ++-- internal/{repository => index}/index.go | 2 +- .../{repository => index}/index_parallel.go | 2 +- .../index_parallel_test.go | 10 ++++-- internal/{repository => index}/index_test.go | 34 +++++++++---------- internal/{repository => index}/indexmap.go | 2 +- .../{repository => index}/indexmap_test.go | 2 +- .../{repository => index}/master_index.go | 6 ++-- .../master_index_test.go | 31 +++++++++-------- internal/index/testing.go | 18 ++++++++++ internal/repository/repack_test.go | 5 +-- internal/repository/repository.go | 13 +++---- internal/repository/repository_test.go | 11 +++--- internal/repository/testing.go | 10 ------ 20 files changed, 101 insertions(+), 80 deletions(-) rename internal/{repository => index}/index.go (99%) rename internal/{repository => index}/index_parallel.go (99%) rename internal/{repository => index}/index_parallel_test.go (69%) rename internal/{repository => index}/index_test.go (94%) rename internal/{repository => index}/indexmap.go (99%) rename internal/{repository => index}/indexmap_test.go (99%) rename internal/{repository => index}/master_index.go (98%) rename internal/{repository => index}/master_index_test.go (93%) create mode 100644 internal/index/testing.go diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 66ce86522..9fcf1b6c7 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -22,6 +22,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -129,7 +130,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) } func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error { - return repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error { + return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { Printf("index_id: %v\n", id) if err != nil { return err diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index 81390ecf8..4809092c0 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -4,7 +4,7 @@ import ( "context" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/restic" "github.com/spf13/cobra" @@ -63,7 +63,7 @@ func runList(ctx context.Context, cmd *cobra.Command, opts GlobalOptions, args [ case "locks": t = restic.LockFile case "blobs": - return repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error { + return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { if err != nil { return err } diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index 74b0db026..63e46e31a 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -9,6 +9,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -699,7 +700,7 @@ func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo r if opts.unsafeRecovery { Verbosef("deleting index files\n") - indexFiles := repo.Index().(*repository.MasterIndex).IDs() + indexFiles := repo.Index().(*index.MasterIndex).IDs() err = DeleteFilesChecked(ctx, gopts, repo, indexFiles, restic.IndexFile) if err != nil { return errors.Fatalf("%s", err) diff --git a/cmd/restic/cmd_rebuild_index.go b/cmd/restic/cmd_rebuild_index.go index 15a21df65..6d49cb917 100644 --- a/cmd/restic/cmd_rebuild_index.go +++ b/cmd/restic/cmd_rebuild_index.go @@ -3,6 +3,7 @@ package main import ( "context" + "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -74,8 +75,8 @@ func rebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOpt } } else { Verbosef("loading indexes...\n") - mi := repository.NewMasterIndex() - err := repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error { + mi := index.NewMasterIndex() + err := index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { if err != nil { Warnf("removing invalid index %v: %v\n", id, err) obsoleteIndexes = append(obsoleteIndexes, id) diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 70d554632..7342475d2 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -23,6 +23,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -1477,11 +1478,11 @@ func TestRebuildIndex(t *testing.T) { } func TestRebuildIndexAlwaysFull(t *testing.T) { - indexFull := repository.IndexFull + indexFull := index.IndexFull defer func() { - repository.IndexFull = indexFull + index.IndexFull = indexFull }() - repository.IndexFull = func(*repository.Index, bool) bool { return true } + index.IndexFull = func(*index.Index, bool) bool { return true } testRebuildIndex(t, nil) } diff --git a/internal/archiver/blob_saver_test.go b/internal/archiver/blob_saver_test.go index 481139a3f..b25478b26 100644 --- a/internal/archiver/blob_saver_test.go +++ b/internal/archiver/blob_saver_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/restic" "golang.org/x/sync/errgroup" ) @@ -40,7 +40,7 @@ func TestBlobSaver(t *testing.T) { wg, ctx := errgroup.WithContext(ctx) saver := &saveFail{ - idx: repository.NewMasterIndex(), + idx: index.NewMasterIndex(), } b := NewBlobSaver(ctx, wg, saver, uint(runtime.NumCPU())) @@ -86,7 +86,7 @@ func TestBlobSaverError(t *testing.T) { wg, ctx := errgroup.WithContext(ctx) saver := &saveFail{ - idx: repository.NewMasterIndex(), + idx: index.NewMasterIndex(), failAt: int32(test.failAt), } diff --git a/internal/checker/checker.go b/internal/checker/checker.go index dba401390..ba04e5e0b 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -18,6 +18,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/hashing" + "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -38,7 +39,7 @@ type Checker struct { } trackUnused bool - masterIndex *repository.MasterIndex + masterIndex *index.MasterIndex snapshots restic.Lister repo restic.Repository @@ -48,7 +49,7 @@ type Checker struct { func New(repo restic.Repository, trackUnused bool) *Checker { c := &Checker{ packs: make(map[restic.ID]int64), - masterIndex: repository.NewMasterIndex(), + masterIndex: index.NewMasterIndex(), repo: repo, trackUnused: trackUnused, } @@ -121,7 +122,7 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) { debug.Log("Start") packToIndex := make(map[restic.ID]restic.IDSet) - err := repository.ForAllIndexes(ctx, c.repo, func(id restic.ID, index *repository.Index, oldFormat bool, err error) error { + err := index.ForAllIndexes(ctx, c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { debug.Log("process index %v, err %v", id, err) if oldFormat { diff --git a/internal/repository/index.go b/internal/index/index.go similarity index 99% rename from internal/repository/index.go rename to internal/index/index.go index 70a1de4fe..507f1a564 100644 --- a/internal/repository/index.go +++ b/internal/index/index.go @@ -1,4 +1,4 @@ -package repository +package index import ( "context" diff --git a/internal/repository/index_parallel.go b/internal/index/index_parallel.go similarity index 99% rename from internal/repository/index_parallel.go rename to internal/index/index_parallel.go index 4fe0797af..83f9122c6 100644 --- a/internal/repository/index_parallel.go +++ b/internal/index/index_parallel.go @@ -1,4 +1,4 @@ -package repository +package index import ( "context" diff --git a/internal/repository/index_parallel_test.go b/internal/index/index_parallel_test.go similarity index 69% rename from internal/repository/index_parallel_test.go rename to internal/index/index_parallel_test.go index 0202be05c..760374510 100644 --- a/internal/repository/index_parallel_test.go +++ b/internal/index/index_parallel_test.go @@ -1,15 +1,19 @@ -package repository_test +package index_test import ( "context" + "path/filepath" "testing" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) +var repoFixture = filepath.Join("..", "repository", "testdata", "test-repo.tar.gz") + func TestRepositoryForAllIndexes(t *testing.T) { repodir, cleanup := rtest.Env(t, repoFixture) defer cleanup() @@ -25,7 +29,7 @@ func TestRepositoryForAllIndexes(t *testing.T) { // check that all expected indexes are loaded without errors indexIDs := restic.NewIDSet() var indexErr error - rtest.OK(t, repository.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *repository.Index, oldFormat bool, err error) error { + rtest.OK(t, index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { if err != nil { indexErr = err } @@ -38,7 +42,7 @@ func TestRepositoryForAllIndexes(t *testing.T) { // must failed with the returned error iterErr := errors.New("error to pass upwards") - err := repository.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *repository.Index, oldFormat bool, err error) error { + err := index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { return iterErr }) diff --git a/internal/repository/index_test.go b/internal/index/index_test.go similarity index 94% rename from internal/repository/index_test.go rename to internal/index/index_test.go index aeaa91a60..efe4e6a5a 100644 --- a/internal/repository/index_test.go +++ b/internal/index/index_test.go @@ -1,4 +1,4 @@ -package repository_test +package index_test import ( "bytes" @@ -7,7 +7,7 @@ import ( "sync" "testing" - "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -15,7 +15,7 @@ import ( func TestIndexSerialize(t *testing.T) { tests := []restic.PackedBlob{} - idx := repository.NewIndex() + idx := index.NewIndex() // create 50 packs with 20 blobs each for i := 0; i < 50; i++ { @@ -51,7 +51,7 @@ func TestIndexSerialize(t *testing.T) { rtest.OK(t, err) idx2ID := restic.NewRandomID() - idx2, oldFormat, err := repository.DecodeIndex(wr.Bytes(), idx2ID) + idx2, oldFormat, err := index.DecodeIndex(wr.Bytes(), idx2ID) rtest.OK(t, err) rtest.Assert(t, idx2 != nil, "nil returned for decoded index") @@ -121,7 +121,7 @@ func TestIndexSerialize(t *testing.T) { rtest.OK(t, err) rtest.Equals(t, restic.IDs{id}, ids) - idx3, oldFormat, err := repository.DecodeIndex(wr3.Bytes(), id) + idx3, oldFormat, err := index.DecodeIndex(wr3.Bytes(), id) rtest.OK(t, err) rtest.Assert(t, idx3 != nil, "nil returned for decoded index") @@ -143,7 +143,7 @@ func TestIndexSerialize(t *testing.T) { } func TestIndexSize(t *testing.T) { - idx := repository.NewIndex() + idx := index.NewIndex() packs := 200 blobCount := 100 @@ -309,7 +309,7 @@ func TestIndexUnserialize(t *testing.T) { } { oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")} - idx, oldFormat, err := repository.DecodeIndex(task.idxBytes, restic.NewRandomID()) + idx, oldFormat, err := index.DecodeIndex(task.idxBytes, restic.NewRandomID()) rtest.OK(t, err) rtest.Assert(t, !oldFormat, "new index format recognized as old format") @@ -354,7 +354,7 @@ func TestIndexUnserialize(t *testing.T) { } } -func listPack(idx *repository.Index, id restic.ID) (pbs []restic.PackedBlob) { +func listPack(idx *index.Index, id restic.ID) (pbs []restic.PackedBlob) { idx.Each(context.TODO(), func(pb restic.PackedBlob) { if pb.PackID.Equal(id) { pbs = append(pbs, pb) @@ -386,7 +386,7 @@ func BenchmarkDecodeIndex(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - _, _, err := repository.DecodeIndex(benchmarkIndexJSON, id) + _, _, err := index.DecodeIndex(benchmarkIndexJSON, id) rtest.OK(b, err) } } @@ -399,14 +399,14 @@ func BenchmarkDecodeIndexParallel(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - _, _, err := repository.DecodeIndex(benchmarkIndexJSON, id) + _, _, err := index.DecodeIndex(benchmarkIndexJSON, id) rtest.OK(b, err) } }) } func TestIndexUnserializeOld(t *testing.T) { - idx, oldFormat, err := repository.DecodeIndex(docOldExample, restic.NewRandomID()) + idx, oldFormat, err := index.DecodeIndex(docOldExample, restic.NewRandomID()) rtest.OK(t, err) rtest.Assert(t, oldFormat, "old index format recognized as new format") @@ -427,7 +427,7 @@ func TestIndexUnserializeOld(t *testing.T) { } func TestIndexPacks(t *testing.T) { - idx := repository.NewIndex() + idx := index.NewIndex() packs := restic.NewIDSet() for i := 0; i < 20; i++ { @@ -456,8 +456,8 @@ func NewRandomTestID(rng *rand.Rand) restic.ID { return id } -func createRandomIndex(rng *rand.Rand, packfiles int) (idx *repository.Index, lookupBh restic.BlobHandle) { - idx = repository.NewIndex() +func createRandomIndex(rng *rand.Rand, packfiles int) (idx *index.Index, lookupBh restic.BlobHandle) { + idx = index.NewIndex() // create index with given number of pack files for i := 0; i < packfiles; i++ { @@ -536,7 +536,7 @@ func BenchmarkIndexAllocParallel(b *testing.B) { func TestIndexHas(t *testing.T) { tests := []restic.PackedBlob{} - idx := repository.NewIndex() + idx := index.NewIndex() // create 50 packs with 20 blobs each for i := 0; i < 50; i++ { @@ -576,7 +576,7 @@ func TestIndexHas(t *testing.T) { } func TestMixedEachByPack(t *testing.T) { - idx := repository.NewIndex() + idx := index.NewIndex() expected := make(map[restic.ID]int) // create 50 packs with 2 blobs each @@ -615,7 +615,7 @@ func TestMixedEachByPack(t *testing.T) { } func TestEachByPackIgnoes(t *testing.T) { - idx := repository.NewIndex() + idx := index.NewIndex() ignores := restic.NewIDSet() expected := make(map[restic.ID]int) diff --git a/internal/repository/indexmap.go b/internal/index/indexmap.go similarity index 99% rename from internal/repository/indexmap.go rename to internal/index/indexmap.go index 99c3fd331..ef3539d48 100644 --- a/internal/repository/indexmap.go +++ b/internal/index/indexmap.go @@ -1,4 +1,4 @@ -package repository +package index import ( "hash/maphash" diff --git a/internal/repository/indexmap_test.go b/internal/index/indexmap_test.go similarity index 99% rename from internal/repository/indexmap_test.go rename to internal/index/indexmap_test.go index 6699b3601..391131ca0 100644 --- a/internal/repository/indexmap_test.go +++ b/internal/index/indexmap_test.go @@ -1,4 +1,4 @@ -package repository +package index import ( "math/rand" diff --git a/internal/repository/master_index.go b/internal/index/master_index.go similarity index 98% rename from internal/repository/master_index.go rename to internal/index/master_index.go index 93e83dca0..ca7c16135 100644 --- a/internal/repository/master_index.go +++ b/internal/index/master_index.go @@ -1,4 +1,4 @@ -package repository +package index import ( "bytes" @@ -31,7 +31,7 @@ func NewMasterIndex() *MasterIndex { return &MasterIndex{idx: idx, pendingBlobs: restic.NewBlobSet()} } -func (mi *MasterIndex) markCompressed() { +func (mi *MasterIndex) MarkCompressed() { mi.compress = true } @@ -65,7 +65,7 @@ func (mi *MasterIndex) LookupSize(bh restic.BlobHandle) (uint, bool) { // Before doing so it checks if this blob is already known. // Returns true if adding was successful and false if the blob // was already known -func (mi *MasterIndex) addPending(bh restic.BlobHandle) bool { +func (mi *MasterIndex) AddPending(bh restic.BlobHandle) bool { mi.idxMutex.Lock() defer mi.idxMutex.Unlock() diff --git a/internal/repository/master_index_test.go b/internal/index/master_index_test.go similarity index 93% rename from internal/repository/master_index_test.go rename to internal/index/master_index_test.go index 77cbc6178..f0c7805de 100644 --- a/internal/repository/master_index_test.go +++ b/internal/index/master_index_test.go @@ -1,4 +1,4 @@ -package repository_test +package index_test import ( "context" @@ -9,6 +9,7 @@ import ( "github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -57,15 +58,15 @@ func TestMasterIndex(t *testing.T) { }, } - idx1 := repository.NewIndex() + idx1 := index.NewIndex() idx1.StorePack(blob1.PackID, []restic.Blob{blob1.Blob}) idx1.StorePack(blob12a.PackID, []restic.Blob{blob12a.Blob}) - idx2 := repository.NewIndex() + idx2 := index.NewIndex() idx2.StorePack(blob2.PackID, []restic.Blob{blob2.Blob}) idx2.StorePack(blob12b.PackID, []restic.Blob{blob12b.Blob}) - mIdx := repository.NewMasterIndex() + mIdx := index.NewMasterIndex() mIdx.Insert(idx1) mIdx.Insert(idx2) @@ -148,18 +149,18 @@ func TestMasterMergeFinalIndexes(t *testing.T) { }, } - idx1 := repository.NewIndex() + idx1 := index.NewIndex() idx1.StorePack(blob1.PackID, []restic.Blob{blob1.Blob}) - idx2 := repository.NewIndex() + idx2 := index.NewIndex() idx2.StorePack(blob2.PackID, []restic.Blob{blob2.Blob}) - mIdx := repository.NewMasterIndex() + mIdx := index.NewMasterIndex() mIdx.Insert(idx1) mIdx.Insert(idx2) - finalIndexes, idxCount := repository.TestMergeIndex(t, mIdx) - rtest.Equals(t, []*repository.Index{idx1, idx2}, finalIndexes) + finalIndexes, idxCount := index.TestMergeIndex(t, mIdx) + rtest.Equals(t, []*index.Index{idx1, idx2}, finalIndexes) rtest.Equals(t, 1, idxCount) blobCount := 0 @@ -178,13 +179,13 @@ func TestMasterMergeFinalIndexes(t *testing.T) { rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id") // merge another index containing identical blobs - idx3 := repository.NewIndex() + idx3 := index.NewIndex() idx3.StorePack(blob1.PackID, []restic.Blob{blob1.Blob}) idx3.StorePack(blob2.PackID, []restic.Blob{blob2.Blob}) mIdx.Insert(idx3) - finalIndexes, idxCount = repository.TestMergeIndex(t, mIdx) - rtest.Equals(t, []*repository.Index{idx3}, finalIndexes) + finalIndexes, idxCount = index.TestMergeIndex(t, mIdx) + rtest.Equals(t, []*index.Index{idx3}, finalIndexes) rtest.Equals(t, 1, idxCount) // Index should have same entries as before! @@ -201,8 +202,8 @@ func TestMasterMergeFinalIndexes(t *testing.T) { rtest.Equals(t, 2, blobCount) } -func createRandomMasterIndex(t testing.TB, rng *rand.Rand, num, size int) (*repository.MasterIndex, restic.BlobHandle) { - mIdx := repository.NewMasterIndex() +func createRandomMasterIndex(t testing.TB, rng *rand.Rand, num, size int) (*index.MasterIndex, restic.BlobHandle) { + mIdx := index.NewMasterIndex() for i := 0; i < num-1; i++ { idx, _ := createRandomIndex(rng, size) mIdx.Insert(idx) @@ -210,7 +211,7 @@ func createRandomMasterIndex(t testing.TB, rng *rand.Rand, num, size int) (*repo idx1, lookupBh := createRandomIndex(rng, size) mIdx.Insert(idx1) - repository.TestMergeIndex(t, mIdx) + index.TestMergeIndex(t, mIdx) return mIdx, lookupBh } diff --git a/internal/index/testing.go b/internal/index/testing.go new file mode 100644 index 000000000..7c05ac651 --- /dev/null +++ b/internal/index/testing.go @@ -0,0 +1,18 @@ +package index + +import ( + "testing" + + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" +) + +func TestMergeIndex(t testing.TB, mi *MasterIndex) ([]*Index, int) { + finalIndexes := mi.finalizeNotFinalIndexes() + for _, idx := range finalIndexes { + test.OK(t, idx.SetID(restic.NewRandomID())) + } + + test.OK(t, mi.MergeFinalIndexes()) + return finalIndexes, len(mi.idx) +} diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index f21dbddf7..5c4218ea5 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -170,7 +171,7 @@ func flush(t *testing.T, repo restic.Repository) { } func rebuildIndex(t *testing.T, repo restic.Repository) { - err := repo.SetIndex(repository.NewMasterIndex()) + err := repo.SetIndex(index.NewMasterIndex()) if err != nil { t.Fatal(err) } @@ -207,7 +208,7 @@ func rebuildIndex(t *testing.T, repo restic.Repository) { } func reloadIndex(t *testing.T, repo restic.Repository) { - err := repo.SetIndex(repository.NewMasterIndex()) + err := repo.SetIndex(index.NewMasterIndex()) if err != nil { t.Fatal(err) } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 996f2a42b..cee0d762b 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -19,6 +19,7 @@ import ( "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" @@ -38,7 +39,7 @@ type Repository struct { cfg restic.Config key *crypto.Key keyName string - idx *MasterIndex + idx *index.MasterIndex Cache *cache.Cache opts Options @@ -118,7 +119,7 @@ func New(be restic.Backend, opts Options) (*Repository, error) { repo := &Repository{ be: be, opts: opts, - idx: NewMasterIndex(), + idx: index.NewMasterIndex(), } return repo, nil @@ -134,7 +135,7 @@ func (r *Repository) DisableAutoIndexUpdate() { func (r *Repository) setConfig(cfg restic.Config) { r.cfg = cfg if r.cfg.Version >= 2 { - r.idx.markCompressed() + r.idx.MarkCompressed() } } @@ -563,7 +564,7 @@ func (r *Repository) Index() restic.MasterIndex { // SetIndex instructs the repository to use the given index. func (r *Repository) SetIndex(i restic.MasterIndex) error { - r.idx = i.(*MasterIndex) + r.idx = i.(*index.MasterIndex) return r.prepareCache() } @@ -572,7 +573,7 @@ func (r *Repository) SetIndex(i restic.MasterIndex) error { func (r *Repository) LoadIndex(ctx context.Context) error { debug.Log("Loading index") - err := ForAllIndexes(ctx, r, func(id restic.ID, idx *Index, oldFormat bool, err error) error { + err := index.ForAllIndexes(ctx, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { if err != nil { return err } @@ -829,7 +830,7 @@ func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte } // first try to add to pending blobs; if not successful, this blob is already known - known = !r.idx.addPending(restic.BlobHandle{ID: newID, Type: t}) + known = !r.idx.AddPending(restic.BlobHandle{ID: newID, Type: t}) // only save when needed or explicitly told if !known || storeDuplicate { diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 109a00cd6..7b5789605 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -18,6 +18,7 @@ import ( "github.com/klauspost/compress/zstd" "github.com/restic/restic/internal/backend/local" "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" @@ -267,13 +268,13 @@ func TestRepositoryLoadIndex(t *testing.T) { } // loadIndex loads the index id from backend and returns it. -func loadIndex(ctx context.Context, repo restic.Repository, id restic.ID) (*repository.Index, error) { +func loadIndex(ctx context.Context, repo restic.Repository, id restic.ID) (*index.Index, error) { buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil) if err != nil { return nil, err } - idx, oldFormat, err := repository.DecodeIndex(buf, id) + idx, oldFormat, err := index.DecodeIndex(buf, id) if oldFormat { fmt.Fprintf(os.Stderr, "index %v has old format\n", id.Str()) } @@ -345,7 +346,7 @@ func benchmarkLoadIndex(b *testing.B, version uint) { repo, cleanup := repository.TestRepositoryWithVersion(b, version) defer cleanup() - idx := repository.NewIndex() + idx := index.NewIndex() for i := 0; i < 5000; i++ { idx.StorePack(restic.NewRandomID(), []restic.Blob{ @@ -357,7 +358,7 @@ func benchmarkLoadIndex(b *testing.B, version uint) { }) } - id, err := repository.SaveIndex(context.TODO(), repo, idx) + id, err := index.SaveIndex(context.TODO(), repo, idx) rtest.OK(b, err) b.Logf("index saved as %v", id.Str()) @@ -400,7 +401,7 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) { repo := r.(*repository.Repository) - repository.IndexFull = func(*repository.Index, bool) bool { return true } + index.IndexFull = func(*index.Index, bool) bool { return true } // add a few rounds of packs for j := 0; j < 5; j++ { diff --git a/internal/repository/testing.go b/internal/repository/testing.go index 380a47d04..6d3f7d07b 100644 --- a/internal/repository/testing.go +++ b/internal/repository/testing.go @@ -138,13 +138,3 @@ func BenchmarkAllVersions(b *testing.B, bench VersionedBenchmark) { }) } } - -func TestMergeIndex(t testing.TB, mi *MasterIndex) ([]*Index, int) { - finalIndexes := mi.finalizeNotFinalIndexes() - for _, idx := range finalIndexes { - test.OK(t, idx.SetID(restic.NewRandomID())) - } - - test.OK(t, mi.MergeFinalIndexes()) - return finalIndexes, len(mi.idx) -}