From 231da4ff80871a10661be78f4a67620d301eefa7 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 3 Aug 2016 22:06:55 +0200 Subject: [PATCH] Remove old repacking code --- src/restic/checker/repacker.go | 163 ---------------------------- src/restic/checker/repacker_test.go | 127 ---------------------- 2 files changed, 290 deletions(-) delete mode 100644 src/restic/checker/repacker.go delete mode 100644 src/restic/checker/repacker_test.go diff --git a/src/restic/checker/repacker.go b/src/restic/checker/repacker.go deleted file mode 100644 index 8aedfc089..000000000 --- a/src/restic/checker/repacker.go +++ /dev/null @@ -1,163 +0,0 @@ -package checker - -import ( - "errors" - - "restic/backend" - "restic/debug" - "restic/repository" -) - -// Repacker extracts still used blobs from packs with unused blobs and creates -// new packs. -type Repacker struct { - unusedBlobs backend.IDSet - repo *repository.Repository -} - -// NewRepacker returns a new repacker that (when Repack() in run) cleans up the -// repository and creates new packs and indexs so that all blobs in unusedBlobs -// aren't used any more. -func NewRepacker(repo *repository.Repository, unusedBlobs backend.IDSet) *Repacker { - return &Repacker{ - repo: repo, - unusedBlobs: unusedBlobs, - } -} - -// Repack runs the process of finding still used blobs in packs with unused -// blobs, extracts them and creates new packs with just the still-in-use blobs. -func (r *Repacker) Repack() error { - debug.Log("Repacker.Repack", "searching packs for %v", r.unusedBlobs) - - unneededPacks, err := FindPacksForBlobs(r.repo, r.unusedBlobs) - if err != nil { - return err - } - - debug.Log("Repacker.Repack", "found packs: %v", unneededPacks) - - blobs, err := FindBlobsForPacks(r.repo, unneededPacks) - if err != nil { - return err - } - - debug.Log("Repacker.Repack", "found blobs: %v", blobs) - - for id := range r.unusedBlobs { - debug.Log("Repacker.Repack", "remove unused blob %v", id.Str()) - blobs.Delete(id) - } - - debug.Log("Repacker.Repack", "need to repack blobs: %v", blobs) - - err = RepackBlobs(r.repo, r.repo, blobs) - if err != nil { - return err - } - - debug.Log("Repacker.Repack", "remove unneeded packs: %v", unneededPacks) - for packID := range unneededPacks { - err = r.repo.Backend().Remove(backend.Data, packID.String()) - if err != nil { - return err - } - } - - debug.Log("Repacker.Repack", "rebuild index, unneeded packs: %v", unneededPacks) - idx, err := r.repo.Index().RebuildIndex(unneededPacks) - - newIndexID, err := repository.SaveIndex(r.repo, idx) - debug.Log("Repacker.Repack", "saved new index at %v, err %v", newIndexID.Str(), err) - if err != nil { - return err - } - - debug.Log("Repacker.Repack", "remove old indexes: %v", idx.Supersedes()) - for _, id := range idx.Supersedes() { - err = r.repo.Backend().Remove(backend.Index, id.String()) - if err != nil { - debug.Log("Repacker.Repack", "error removing index %v: %v", id.Str(), err) - return err - } - - debug.Log("Repacker.Repack", "removed index %v", id.Str()) - } - - return nil -} - -// FindPacksForBlobs returns the set of packs that contain the blobs. -func FindPacksForBlobs(repo *repository.Repository, blobs backend.IDSet) (backend.IDSet, error) { - packs := backend.NewIDSet() - idx := repo.Index() - for id := range blobs { - blob, err := idx.Lookup(id) - if err != nil { - return nil, err - } - - packs.Insert(blob.PackID) - } - - return packs, nil -} - -// FindBlobsForPacks returns the set of blobs contained in a pack of packs. -func FindBlobsForPacks(repo *repository.Repository, packs backend.IDSet) (backend.IDSet, error) { - blobs := backend.NewIDSet() - - for packID := range packs { - for _, packedBlob := range repo.Index().ListPack(packID) { - blobs.Insert(packedBlob.ID) - } - } - - return blobs, nil -} - -// repackBlob loads a single blob from src and saves it in dst. -func repackBlob(src, dst *repository.Repository, id backend.ID) error { - blob, err := src.Index().Lookup(id) - if err != nil { - return err - } - - debug.Log("RepackBlobs", "repacking blob %v, len %v", id.Str(), blob.PlaintextLength()) - - buf := make([]byte, 0, blob.PlaintextLength()) - buf, err = src.LoadBlob(blob.Type, id, buf) - if err != nil { - return err - } - - if uint(len(buf)) != blob.PlaintextLength() { - debug.Log("RepackBlobs", "repack blob %v: len(buf) isn't equal to length: %v = %v", id.Str(), len(buf), blob.PlaintextLength()) - return errors.New("LoadBlob returned wrong data, len() doesn't match") - } - - _, err = dst.SaveAndEncrypt(blob.Type, buf, &id) - if err != nil { - return err - } - - return nil -} - -// RepackBlobs reads all blobs in blobIDs from src and saves them into new pack -// files in dst. Source and destination repo may be the same. -func RepackBlobs(src, dst *repository.Repository, blobIDs backend.IDSet) (err error) { - for id := range blobIDs { - err = repackBlob(src, dst, id) - if err != nil { - return err - } - } - - err = dst.Flush() - if err != nil { - return err - } - - return nil -} diff --git a/src/restic/checker/repacker_test.go b/src/restic/checker/repacker_test.go deleted file mode 100644 index 821828c8b..000000000 --- a/src/restic/checker/repacker_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package checker_test - -import ( - "testing" - - "restic/backend" - "restic/checker" - - . "restic/test" -) - -var findPackTests = []struct { - blobIDs backend.IDSet - packIDs backend.IDSet -}{ - { - backend.IDSet{ - ParseID("534f211b4fc2cf5b362a24e8eba22db5372a75b7e974603ff9263f5a471760f4"): struct{}{}, - ParseID("51aa04744b518c6a85b4e7643cfa99d58789c2a6ca2a3fda831fa3032f28535c"): struct{}{}, - ParseID("454515bca5f4f60349a527bd814cc2681bc3625716460cc6310771c966d8a3bf"): struct{}{}, - ParseID("c01952de4d91da1b1b80bc6e06eaa4ec21523f4853b69dc8231708b9b7ec62d8"): struct{}{}, - }, - backend.IDSet{ - ParseID("19a731a515618ec8b75fc0ff3b887d8feb83aef1001c9899f6702761142ed068"): struct{}{}, - ParseID("657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6"): struct{}{}, - }, - }, -} - -var findBlobTests = []struct { - packIDs backend.IDSet - blobIDs backend.IDSet -}{ - { - backend.IDSet{ - ParseID("60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"): struct{}{}, - }, - backend.IDSet{ - ParseID("356493f0b00a614d36c698591bbb2b1d801932d85328c1f508019550034549fc"): struct{}{}, - ParseID("b8a6bcdddef5c0f542b4648b2ef79bc0ed4377d4109755d2fb78aff11e042663"): struct{}{}, - ParseID("5714f7274a8aa69b1692916739dc3835d09aac5395946b8ec4f58e563947199a"): struct{}{}, - ParseID("b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"): struct{}{}, - ParseID("08d0444e9987fa6e35ce4232b2b71473e1a8f66b2f9664cc44dc57aad3c5a63a"): struct{}{}, - }, - }, - { - backend.IDSet{ - ParseID("60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"): struct{}{}, - ParseID("ff7e12cd66d896b08490e787d1915c641e678d7e6b4a00e60db5d13054f4def4"): struct{}{}, - }, - backend.IDSet{ - ParseID("356493f0b00a614d36c698591bbb2b1d801932d85328c1f508019550034549fc"): struct{}{}, - ParseID("b8a6bcdddef5c0f542b4648b2ef79bc0ed4377d4109755d2fb78aff11e042663"): struct{}{}, - ParseID("5714f7274a8aa69b1692916739dc3835d09aac5395946b8ec4f58e563947199a"): struct{}{}, - ParseID("b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"): struct{}{}, - ParseID("08d0444e9987fa6e35ce4232b2b71473e1a8f66b2f9664cc44dc57aad3c5a63a"): struct{}{}, - ParseID("aa79d596dbd4c863e5400deaca869830888fe1ce9f51b4a983f532c77f16a596"): struct{}{}, - ParseID("b2396c92781307111accf2ebb1cd62b58134b744d90cb6f153ca456a98dc3e76"): struct{}{}, - ParseID("5249af22d3b2acd6da8048ac37b2a87fa346fabde55ed23bb866f7618843c9fe"): struct{}{}, - ParseID("f41c2089a9d58a4b0bf39369fa37588e6578c928aea8e90a4490a6315b9905c1"): struct{}{}, - }, - }, -} - -func TestRepackerFindPacks(t *testing.T) { - WithTestEnvironment(t, checkerTestData, func(repodir string) { - repo := OpenLocalRepo(t, repodir) - - OK(t, repo.LoadIndex()) - - for _, test := range findPackTests { - packIDs, err := checker.FindPacksForBlobs(repo, test.blobIDs) - OK(t, err) - Equals(t, test.packIDs, packIDs) - } - - for _, test := range findBlobTests { - blobs, err := checker.FindBlobsForPacks(repo, test.packIDs) - OK(t, err) - - Assert(t, test.blobIDs.Equals(blobs), - "list of blobs for packs %v does not match, expected:\n %v\ngot:\n %v", - test.packIDs, test.blobIDs, blobs) - } - }) -} - -func TestRepacker(t *testing.T) { - WithTestEnvironment(t, checkerTestData, func(repodir string) { - repo := OpenLocalRepo(t, repodir) - OK(t, repo.LoadIndex()) - - repo.Backend().Remove(backend.Snapshot, "c2b53c5e6a16db92fbb9aa08bd2794c58b379d8724d661ee30d20898bdfdff22") - - unusedBlobs := backend.IDSet{ - ParseID("5714f7274a8aa69b1692916739dc3835d09aac5395946b8ec4f58e563947199a"): struct{}{}, - ParseID("08d0444e9987fa6e35ce4232b2b71473e1a8f66b2f9664cc44dc57aad3c5a63a"): struct{}{}, - ParseID("356493f0b00a614d36c698591bbb2b1d801932d85328c1f508019550034549fc"): struct{}{}, - ParseID("b8a6bcdddef5c0f542b4648b2ef79bc0ed4377d4109755d2fb78aff11e042663"): struct{}{}, - } - - chkr := checker.New(repo) - _, errs := chkr.LoadIndex() - OKs(t, errs) - - errs = checkStruct(chkr) - OKs(t, errs) - - list := backend.NewIDSet(chkr.UnusedBlobs()...) - if !unusedBlobs.Equals(list) { - t.Fatalf("expected unused blobs:\n %v\ngot:\n %v", unusedBlobs, list) - } - - repacker := checker.NewRepacker(repo, unusedBlobs) - OK(t, repacker.Repack()) - - chkr = checker.New(repo) - _, errs = chkr.LoadIndex() - OKs(t, errs) - OKs(t, checkPacks(chkr)) - OKs(t, checkStruct(chkr)) - - blobs := chkr.UnusedBlobs() - Assert(t, len(blobs) == 0, - "expected zero unused blobs, got %v", blobs) - }) -}