From d6f78163d4e88cfa14ea1313d767c693aa5826b8 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Fri, 28 Sep 2018 14:30:43 +0200 Subject: [PATCH 01/15] Add 'debug examine' command to debug #1999 --- cmd/restic/cmd_debug.go | 177 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 3403b259a..80b94b980 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -7,6 +7,8 @@ import ( "encoding/json" "fmt" "io" + "os" + "sort" "github.com/spf13/cobra" @@ -42,6 +44,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er func init() { cmdRoot.AddCommand(cmdDebug) cmdDebug.AddCommand(cmdDebugDump) + cmdDebug.AddCommand(cmdDebugExamine) } func prettyPrintJSON(wr io.Writer, item interface{}) error { @@ -165,3 +168,177 @@ func runDebugDump(gopts GlobalOptions, args []string) error { return errors.Fatalf("no such type %q", tpe) } } + +var cmdDebugExamine = &cobra.Command{ + Use: "examine", + Short: "Examine a pack file", + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runDebugExamine(globalOptions, args) + }, +} + +func loadBlobs(ctx context.Context, repo restic.Repository, pack string, list []restic.PackedBlob) error { + be := repo.Backend() + for _, blob := range list { + fmt.Printf(" loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length) + buf := make([]byte, blob.Length) + h := restic.Handle{ + Name: pack, + Type: restic.PackFile, + } + err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error { + n, err := io.ReadFull(rd, buf) + if err != nil { + fmt.Fprintf(os.Stderr, "read error after %d bytes: %v\n", n, err) + return err + } + return nil + }) + if err != nil { + fmt.Fprintf(os.Stderr, "error read: %v\n", err) + continue + } + + key := repo.Key() + + nonce, buf := buf[:key.NonceSize()], buf[key.NonceSize():] + buf, err = key.Open(buf[:0], nonce, buf, nil) + if err != nil { + fmt.Fprintf(os.Stderr, "error decrypting blob: %v\n", err) + continue + } + + id := restic.Hash(buf) + fmt.Printf(" successfully decrypted blob (length %v), hash is %v\n", len(buf), id) + if !id.Equal(blob.ID) { + fmt.Printf(" IDs do not match, want %v, got %v\n", blob.ID, id) + } else { + fmt.Printf(" IDs match\n") + } + } + + return nil +} + +func runDebugExamine(gopts GlobalOptions, args []string) error { + repo, err := OpenRepository(gopts) + if err != nil { + return err + } + + if !gopts.NoLock { + lock, err := lockRepo(gopts.ctx, repo) + defer unlockRepo(lock) + if err != nil { + return err + } + } + + err = repo.LoadIndex(gopts.ctx) + if err != nil { + return err + } + + for _, name := range args { + fmt.Printf("examine %v\n", name) + id, err := restic.ParseID(name) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + } + + h := restic.Handle{ + Type: restic.PackFile, + Name: name, + } + fi, err := repo.Backend().Stat(gopts.ctx, h) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + } + + fmt.Printf(" file size is %v\n", fi.Size) + + // examine all data the indexes have for the pack file + for _, idx := range repo.Index().(*repository.MasterIndex).All() { + idxIDs, err := idx.IDs() + if err != nil { + idxIDs = restic.IDs{} + } + + blobs := idx.ListPack(id) + if len(blobs) == 0 { + fmt.Printf(" index %v does not contain the file\n", idxIDs) + continue + } + + fmt.Printf(" index %v:\n", idxIDs) + + // track current size and offset + var size, offset uint64 + + sort.Slice(blobs, func(i, j int) bool { + return blobs[i].Offset < blobs[j].Offset + }) + + for _, pb := range blobs { + fmt.Printf(" %v blob %v, offset %-6d, raw length %-6d\n", pb.Type, pb.ID, pb.Offset, pb.Length) + if offset != uint64(pb.Offset) { + fmt.Printf(" hole in file, want offset %v, got %v\n", offset, pb.Offset) + } + offset += uint64(pb.Length) + size += uint64(pb.Length) + } + + // compute header size, per blob: 1 byte type, 4 byte length, 32 byte id + size += uint64(restic.CiphertextLength(len(blobs) * (1 + 4 + 32))) + // length in uint32 little endian + size += 4 + + if uint64(fi.Size) != size { + fmt.Printf(" file sizes do not match: computed %v from index, file size is %v\n", size, fi.Size) + } else { + fmt.Printf(" file sizes match\n") + } + + err = loadBlobs(gopts.ctx, repo, name, blobs) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + } + } + + // inspect the pack file itself + blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(gopts.ctx, repo.Backend(), h), fi.Size) + if err != nil { + fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", id.Str(), err) + return nil + } + + // track current size and offset + var size, offset uint64 + + sort.Slice(blobs, func(i, j int) bool { + return blobs[i].Offset < blobs[j].Offset + }) + + for _, pb := range blobs { + fmt.Printf(" %v blob %v, offset %-6d, raw length %-6d\n", pb.Type, pb.ID, pb.Offset, pb.Length) + if offset != uint64(pb.Offset) { + fmt.Printf(" hole in file, want offset %v, got %v\n", offset, pb.Offset) + } + offset += uint64(pb.Length) + size += uint64(pb.Length) + } + + // compute header size, per blob: 1 byte type, 4 byte length, 32 byte id + size += uint64(restic.CiphertextLength(len(blobs) * (1 + 4 + 32))) + // length in uint32 little endian + size += 4 + + if uint64(fi.Size) != size { + fmt.Printf(" file sizes do not match: computed %v from index, file size is %v\n", size, fi.Size) + } else { + fmt.Printf(" file sizes match\n") + } + } + return nil +} From 086993bae11d4d639a1e1904e3b245f62aeb91ab Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 29 Sep 2018 14:16:01 +0200 Subject: [PATCH 02/15] debug: check packs not in index, implement repair --- cmd/restic/cmd_debug.go | 125 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 117 insertions(+), 8 deletions(-) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 80b94b980..934628771 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -8,10 +8,14 @@ import ( "fmt" "io" "os" + "runtime" "sort" + "time" "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" + "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" @@ -41,10 +45,13 @@ Exit status is 0 if the command was successful, and non-zero if there was any er }, } +var tryRepair bool + func init() { cmdRoot.AddCommand(cmdDebug) cmdDebug.AddCommand(cmdDebugDump) cmdDebug.AddCommand(cmdDebugExamine) + cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips") } func prettyPrintJSON(wr io.Writer, item interface{}) error { @@ -178,7 +185,87 @@ var cmdDebugExamine = &cobra.Command{ }, } -func loadBlobs(ctx context.Context, repo restic.Repository, pack string, list []restic.PackedBlob) error { +func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte) { + fmt.Printf(" trying to repair blob with single bit flip\n") + + ch := make(chan int) + var wg errgroup.Group + done := make(chan struct{}) + + fmt.Printf(" spinning up %d worker functions\n", runtime.NumCPU()) + for i := 0; i < runtime.NumCPU(); i++ { + wg.Go(func() error { + // make a local copy of the buffer + buf := make([]byte, len(input)) + copy(buf, input) + + for { + select { + case <-done: + return nil + case i := <-ch: + for j := 0; j < 7; j++ { + // flip bit + buf[i] ^= (1 << uint(j)) + + nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] + plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil) + if err == nil { + fmt.Printf("\n") + fmt.Printf(" blob could be repaired by flipping bit %v in byte %v\n", j, i) + fmt.Printf(" hash is %v\n", restic.Hash(plaintext)) + close(done) + return nil + } + + // flip bit back + buf[i] ^= (1 << uint(j)) + } + } + } + }) + } + + start := time.Now() + info := time.Now() +outer: + for i := range input { + select { + case ch <- i: + case <-done: + fmt.Printf(" done after %v\n", time.Since(start)) + break outer + } + + if time.Since(info) > time.Second { + secs := time.Since(start).Seconds() + gps := float64(i) / secs + remaining := len(input) - i + eta := time.Duration(float64(remaining)/gps) * time.Second + + fmt.Printf("\r%d byte of %d done (%.2f%%), %.2f guesses per second, ETA %v", + i, len(input), float32(i)/float32(len(input)), + gps, eta) + info = time.Now() + } + } + + var found bool + select { + case <-done: + found = true + default: + close(done) + } + + wg.Wait() + + if !found { + fmt.Printf("\n blob could not be repaired by single bit flip\n") + } +} + +func loadBlobs(ctx context.Context, repo restic.Repository, pack string, list []restic.Blob) error { be := repo.Backend() for _, blob := range list { fmt.Printf(" loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length) @@ -202,15 +289,18 @@ func loadBlobs(ctx context.Context, repo restic.Repository, pack string, list [] key := repo.Key() - nonce, buf := buf[:key.NonceSize()], buf[key.NonceSize():] - buf, err = key.Open(buf[:0], nonce, buf, nil) + nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] + plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil) if err != nil { fmt.Fprintf(os.Stderr, "error decrypting blob: %v\n", err) + if tryRepair { + tryRepairWithBitflip(ctx, key, buf) + } continue } - id := restic.Hash(buf) - fmt.Printf(" successfully decrypted blob (length %v), hash is %v\n", len(buf), id) + id := restic.Hash(plaintext) + fmt.Printf(" successfully decrypted blob (length %v), hash is %v\n", len(plaintext), id) if !id.Equal(blob.ID) { fmt.Printf(" IDs do not match, want %v, got %v\n", blob.ID, id) } else { @@ -240,6 +330,7 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { return err } + blobsLoaded := false for _, name := range args { fmt.Printf("examine %v\n", name) id, err := restic.ParseID(name) @@ -257,6 +348,8 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { } fmt.Printf(" file size is %v\n", fi.Size) + fmt.Printf(" ========================================\n") + fmt.Printf(" looking for info in the indexes\n") // examine all data the indexes have for the pack file for _, idx := range repo.Index().(*repository.MasterIndex).All() { @@ -267,7 +360,6 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { blobs := idx.ListPack(id) if len(blobs) == 0 { - fmt.Printf(" index %v does not contain the file\n", idxIDs) continue } @@ -300,13 +392,23 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { fmt.Printf(" file sizes match\n") } - err = loadBlobs(gopts.ctx, repo, name, blobs) + // convert list of blobs to []restic.Blob + var list []restic.Blob + for _, b := range blobs { + list = append(list, b.Blob) + } + + err = loadBlobs(gopts.ctx, repo, name, list) if err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) + } else { + blobsLoaded = true } } - // inspect the pack file itself + fmt.Printf(" ========================================\n") + fmt.Printf(" inspect the pack itself\n") + blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(gopts.ctx, repo.Backend(), h), fi.Size) if err != nil { fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", id.Str(), err) @@ -339,6 +441,13 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { } else { fmt.Printf(" file sizes match\n") } + + if !blobsLoaded { + err = loadBlobs(gopts.ctx, repo, name, blobs) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + } + } } return nil } From 90f975fa1cf78067d6a74c3e22e93026ccb59e45 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 29 Sep 2018 14:20:53 +0200 Subject: [PATCH 03/15] debug: make output less verbose --- cmd/restic/cmd_debug.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 934628771..1a5f971a7 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -300,11 +300,10 @@ func loadBlobs(ctx context.Context, repo restic.Repository, pack string, list [] } id := restic.Hash(plaintext) - fmt.Printf(" successfully decrypted blob (length %v), hash is %v\n", len(plaintext), id) if !id.Equal(blob.ID) { - fmt.Printf(" IDs do not match, want %v, got %v\n", blob.ID, id) + fmt.Printf(" successfully decrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plaintext), id, blob.ID) } else { - fmt.Printf(" IDs match\n") + fmt.Printf(" successfully decrypted blob (length %v), hash is %v, ID matches\n", len(plaintext), id) } } From 133ac42a0b3fa576f0283343eb6c1d9e63d39ca8 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 29 Sep 2018 14:40:48 +0200 Subject: [PATCH 04/15] debug: check file content hash --- cmd/restic/cmd_debug.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 1a5f971a7..2230c7b1c 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -15,6 +15,7 @@ import ( "github.com/spf13/cobra" "golang.org/x/sync/errgroup" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/pack" @@ -347,6 +348,19 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { } fmt.Printf(" file size is %v\n", fi.Size) + + buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + } + + gotID := restic.Hash(buf) + if !id.Equal(gotID) { + fmt.Printf(" wanted hash %v, got %v\n", id, gotID) + } else { + fmt.Printf(" hash for file content matches\n") + } + fmt.Printf(" ========================================\n") fmt.Printf(" looking for info in the indexes\n") From 52061e817c02a6fb9b1048e8f090d09e821538c3 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 29 Sep 2018 14:42:19 +0200 Subject: [PATCH 05/15] debug: fix percentage --- cmd/restic/cmd_debug.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 2230c7b1c..59a8aad2d 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -245,7 +245,7 @@ outer: eta := time.Duration(float64(remaining)/gps) * time.Second fmt.Printf("\r%d byte of %d done (%.2f%%), %.2f guesses per second, ETA %v", - i, len(input), float32(i)/float32(len(input)), + i, len(input), float32(i)/float32(len(input)*100), gps, eta) info = time.Now() } From ce4b6d087482b2a7b0de1a68d9bb1a676d82b5e2 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 29 Sep 2018 18:28:39 +0200 Subject: [PATCH 06/15] examine: add byte repair mode --- cmd/restic/cmd_debug.go | 62 +++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 59a8aad2d..a62b7ac37 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -47,12 +47,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er } var tryRepair bool +var repairByte bool func init() { cmdRoot.AddCommand(cmdDebug) cmdDebug.AddCommand(cmdDebugDump) cmdDebug.AddCommand(cmdDebugExamine) cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips") + cmdDebugExamine.Flags().BoolVar(&repairByte, "repair-byte", false, "try to repair broken blobs by trying bytes") } func prettyPrintJSON(wr io.Writer, item interface{}) error { @@ -186,8 +188,12 @@ var cmdDebugExamine = &cobra.Command{ }, } -func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte) { - fmt.Printf(" trying to repair blob with single bit flip\n") +func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, bytewise bool) { + if bytewise { + fmt.Printf(" trying to repair blob by finding a broken byte\n") + } else { + fmt.Printf(" trying to repair blob with single bit flip\n") + } ch := make(chan int) var wg errgroup.Group @@ -205,22 +211,42 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte) { case <-done: return nil case i := <-ch: - for j := 0; j < 7; j++ { - // flip bit - buf[i] ^= (1 << uint(j)) + if bytewise { + for j := 0; j < 255; j++ { + // flip bits + buf[i] ^= byte(j) - nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] - plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil) - if err == nil { - fmt.Printf("\n") - fmt.Printf(" blob could be repaired by flipping bit %v in byte %v\n", j, i) - fmt.Printf(" hash is %v\n", restic.Hash(plaintext)) - close(done) - return nil + nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] + plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil) + if err == nil { + fmt.Printf("\n") + fmt.Printf(" blob could be repaired by XORing byte %v with 0x%02x\n", i, j) + fmt.Printf(" hash is %v\n", restic.Hash(plaintext)) + close(done) + return nil + } + + // flip bits back + buf[i] ^= byte(j) } + } else { + for j := 0; j < 7; j++ { + // flip bit + buf[i] ^= (1 << uint(j)) - // flip bit back - buf[i] ^= (1 << uint(j)) + nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] + plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil) + if err == nil { + fmt.Printf("\n") + fmt.Printf(" blob could be repaired by flipping bit %v in byte %v\n", j, i) + fmt.Printf(" hash is %v\n", restic.Hash(plaintext)) + close(done) + return nil + } + + // flip bit back + buf[i] ^= (1 << uint(j)) + } } } } @@ -244,7 +270,7 @@ outer: remaining := len(input) - i eta := time.Duration(float64(remaining)/gps) * time.Second - fmt.Printf("\r%d byte of %d done (%.2f%%), %.2f guesses per second, ETA %v", + fmt.Printf("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v", i, len(input), float32(i)/float32(len(input)*100), gps, eta) info = time.Now() @@ -294,8 +320,8 @@ func loadBlobs(ctx context.Context, repo restic.Repository, pack string, list [] plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil) if err != nil { fmt.Fprintf(os.Stderr, "error decrypting blob: %v\n", err) - if tryRepair { - tryRepairWithBitflip(ctx, key, buf) + if tryRepair || repairByte { + tryRepairWithBitflip(ctx, key, buf, repairByte) } continue } From b3c312162274b6de6115074f72d1bdfab63db419 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 3 Oct 2018 11:46:58 +0200 Subject: [PATCH 07/15] debug: Save raw decrypt (disregarding signature) --- cmd/restic/cmd_debug.go | 50 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index a62b7ac37..f726a7557 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -4,6 +4,8 @@ package main import ( "context" + "crypto/aes" + "crypto/cipher" "encoding/json" "fmt" "io" @@ -292,6 +294,35 @@ outer: } } +func sliceForAppend(in []byte, n int) (head, tail []byte) { + if total := len(in) + n; cap(in) >= total { + head = in[:total] + } else { + head = make([]byte, total) + copy(head, in) + } + tail = head[len(in):] + return +} + +func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte { + // strip signature at the end + l := len(buf) + nonce, ct := buf[:16], buf[16:l-16] + dst := make([]byte, 0, len(ct)) + + ret, out := sliceForAppend(dst, len(ct)) + + c, err := aes.NewCipher(k.EncryptionKey[:]) + if err != nil { + panic(fmt.Sprintf("unable to create cipher: %v", err)) + } + e := cipher.NewCTR(c, nonce) + e.XORKeyStream(out, ct) + + return ret +} + func loadBlobs(ctx context.Context, repo restic.Repository, pack string, list []restic.Blob) error { be := repo.Backend() for _, blob := range list { @@ -323,6 +354,25 @@ func loadBlobs(ctx context.Context, repo restic.Repository, pack string, list [] if tryRepair || repairByte { tryRepairWithBitflip(ctx, key, buf, repairByte) } + plain := decryptUnsigned(ctx, key, buf) + filename := fmt.Sprintf("%s.bin", blob.ID.String()) + f, err := os.Create(filename) + if err != nil { + return err + } + + _, err = f.Write(plain) + if err != nil { + _ = f.Close() + return err + } + + err = f.Close() + if err != nil { + return err + } + + fmt.Printf("decrypt of blob %v stored at %v\n", blob.ID.Str(), filename) continue } From 84491ff40b2ca9611c10684f1d098d693489ac3b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 9 Oct 2020 14:33:07 +0200 Subject: [PATCH 08/15] debug: store repaired and correct blobs --- cmd/restic/cmd_debug.go | 114 +++++++++++++++++++++++----------------- 1 file changed, 67 insertions(+), 47 deletions(-) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index f726a7557..067aee09b 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -50,11 +50,13 @@ Exit status is 0 if the command was successful, and non-zero if there was any er var tryRepair bool var repairByte bool +var extractPack bool func init() { cmdRoot.AddCommand(cmdDebug) cmdDebug.AddCommand(cmdDebugDump) cmdDebug.AddCommand(cmdDebugExamine) + cmdDebugExamine.Flags().BoolVar(&extractPack, "extract-pack", false, "write blobs to the current directory") cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips") cmdDebugExamine.Flags().BoolVar(&repairByte, "repair-byte", false, "try to repair broken blobs by trying bytes") } @@ -190,7 +192,7 @@ var cmdDebugExamine = &cobra.Command{ }, } -func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, bytewise bool) { +func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, bytewise bool) []byte { if bytewise { fmt.Printf(" trying to repair blob by finding a broken byte\n") } else { @@ -200,9 +202,12 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by ch := make(chan int) var wg errgroup.Group done := make(chan struct{}) + var fixed []byte + var found bool - fmt.Printf(" spinning up %d worker functions\n", runtime.NumCPU()) - for i := 0; i < runtime.NumCPU(); i++ { + workers := runtime.GOMAXPROCS(0) + fmt.Printf(" spinning up %d worker functions\n", runtime.GOMAXPROCS(0)) + for i := 0; i < workers; i++ { wg.Go(func() error { // make a local copy of the buffer buf := make([]byte, len(input)) @@ -210,9 +215,10 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by for { select { - case <-done: - return nil - case i := <-ch: + case i, ok := <-ch: + if !ok { + return nil + } if bytewise { for j := 0; j < 255; j++ { // flip bits @@ -225,6 +231,8 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by fmt.Printf(" blob could be repaired by XORing byte %v with 0x%02x\n", i, j) fmt.Printf(" hash is %v\n", restic.Hash(plaintext)) close(done) + found = true + fixed = plaintext return nil } @@ -243,6 +251,8 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by fmt.Printf(" blob could be repaired by flipping bit %v in byte %v\n", j, i) fmt.Printf(" hash is %v\n", restic.Hash(plaintext)) close(done) + found = true + fixed = plaintext return nil } @@ -278,40 +288,20 @@ outer: info = time.Now() } } - - var found bool - select { - case <-done: - found = true - default: - close(done) - } - + close(ch) wg.Wait() if !found { fmt.Printf("\n blob could not be repaired by single bit flip\n") } -} - -func sliceForAppend(in []byte, n int) (head, tail []byte) { - if total := len(in) + n; cap(in) >= total { - head = in[:total] - } else { - head = make([]byte, total) - copy(head, in) - } - tail = head[len(in):] - return + return fixed } func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte { // strip signature at the end l := len(buf) nonce, ct := buf[:16], buf[16:l-16] - dst := make([]byte, 0, len(ct)) - - ret, out := sliceForAppend(dst, len(ct)) + out := make([]byte, len(ct)) c, err := aes.NewCipher(k.EncryptionKey[:]) if err != nil { @@ -320,7 +310,7 @@ func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte { e := cipher.NewCTR(c, nonce) e.XORKeyStream(out, ct) - return ret + return out } func loadBlobs(ctx context.Context, repo restic.Repository, pack string, list []restic.Blob) error { @@ -351,42 +341,72 @@ func loadBlobs(ctx context.Context, repo restic.Repository, pack string, list [] plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil) if err != nil { fmt.Fprintf(os.Stderr, "error decrypting blob: %v\n", err) + var plain []byte if tryRepair || repairByte { - tryRepairWithBitflip(ctx, key, buf, repairByte) + plain = tryRepairWithBitflip(ctx, key, buf, repairByte) } - plain := decryptUnsigned(ctx, key, buf) - filename := fmt.Sprintf("%s.bin", blob.ID.String()) - f, err := os.Create(filename) + var prefix string + if plain != nil { + id := restic.Hash(plain) + if !id.Equal(blob.ID) { + fmt.Printf(" successfully repaired blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plain), id, blob.ID) + prefix = "repaired-wrong-hash-" + } else { + prefix = "repaired-" + } + } else { + plain = decryptUnsigned(ctx, key, buf) + prefix = "damaged-" + } + err = storePlainBlob(blob.ID, prefix, plain) if err != nil { return err } - - _, err = f.Write(plain) - if err != nil { - _ = f.Close() - return err - } - - err = f.Close() - if err != nil { - return err - } - - fmt.Printf("decrypt of blob %v stored at %v\n", blob.ID.Str(), filename) continue } id := restic.Hash(plaintext) + var prefix string if !id.Equal(blob.ID) { fmt.Printf(" successfully decrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plaintext), id, blob.ID) + prefix = "wrong-hash-" } else { fmt.Printf(" successfully decrypted blob (length %v), hash is %v, ID matches\n", len(plaintext), id) + prefix = "correct-" + } + if extractPack { + err = storePlainBlob(id, prefix, plaintext) + if err != nil { + return err + } } } return nil } +func storePlainBlob(id restic.ID, prefix string, plain []byte) error { + filename := fmt.Sprintf("%s%s.bin", prefix, id) + f, err := os.Create(filename) + if err != nil { + return err + } + + _, err = f.Write(plain) + if err != nil { + _ = f.Close() + return err + } + + err = f.Close() + if err != nil { + return err + } + + fmt.Printf("decrypt of blob %v stored at %v\n", id, filename) + return nil +} + func runDebugExamine(gopts GlobalOptions, args []string) error { repo, err := OpenRepository(gopts) if err != nil { From 096f15db5c27a7638ecb757078d4e706732df137 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 10 Mar 2021 20:21:05 +0100 Subject: [PATCH 09/15] debug: extract examinePack function --- cmd/restic/cmd_debug.go | 198 ++++++++++++++++++++-------------------- 1 file changed, 101 insertions(+), 97 deletions(-) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 067aee09b..5134f5dca 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -426,103 +426,59 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { return err } - blobsLoaded := false for _, name := range args { - fmt.Printf("examine %v\n", name) - id, err := restic.ParseID(name) + examinePack(gopts.ctx, repo, name) + } + return nil +} + +func examinePack(ctx context.Context, repo restic.Repository, name string) { + blobsLoaded := false + fmt.Printf("examine %v\n", name) + id, err := restic.ParseID(name) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + } + + h := restic.Handle{ + Type: restic.PackFile, + Name: name, + } + fi, err := repo.Backend().Stat(ctx, h) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + } + + fmt.Printf(" file size is %v\n", fi.Size) + + buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + } + + gotID := restic.Hash(buf) + if !id.Equal(gotID) { + fmt.Printf(" wanted hash %v, got %v\n", id, gotID) + } else { + fmt.Printf(" hash for file content matches\n") + } + + fmt.Printf(" ========================================\n") + fmt.Printf(" looking for info in the indexes\n") + + // examine all data the indexes have for the pack file + for _, idx := range repo.Index().(*repository.MasterIndex).All() { + idxIDs, err := idx.IDs() if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) + idxIDs = restic.IDs{} } - h := restic.Handle{ - Type: restic.PackFile, - Name: name, - } - fi, err := repo.Backend().Stat(gopts.ctx, h) - if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) + blobs := idx.ListPack(id) + if len(blobs) == 0 { + continue } - fmt.Printf(" file size is %v\n", fi.Size) - - buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h) - if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - } - - gotID := restic.Hash(buf) - if !id.Equal(gotID) { - fmt.Printf(" wanted hash %v, got %v\n", id, gotID) - } else { - fmt.Printf(" hash for file content matches\n") - } - - fmt.Printf(" ========================================\n") - fmt.Printf(" looking for info in the indexes\n") - - // examine all data the indexes have for the pack file - for _, idx := range repo.Index().(*repository.MasterIndex).All() { - idxIDs, err := idx.IDs() - if err != nil { - idxIDs = restic.IDs{} - } - - blobs := idx.ListPack(id) - if len(blobs) == 0 { - continue - } - - fmt.Printf(" index %v:\n", idxIDs) - - // track current size and offset - var size, offset uint64 - - sort.Slice(blobs, func(i, j int) bool { - return blobs[i].Offset < blobs[j].Offset - }) - - for _, pb := range blobs { - fmt.Printf(" %v blob %v, offset %-6d, raw length %-6d\n", pb.Type, pb.ID, pb.Offset, pb.Length) - if offset != uint64(pb.Offset) { - fmt.Printf(" hole in file, want offset %v, got %v\n", offset, pb.Offset) - } - offset += uint64(pb.Length) - size += uint64(pb.Length) - } - - // compute header size, per blob: 1 byte type, 4 byte length, 32 byte id - size += uint64(restic.CiphertextLength(len(blobs) * (1 + 4 + 32))) - // length in uint32 little endian - size += 4 - - if uint64(fi.Size) != size { - fmt.Printf(" file sizes do not match: computed %v from index, file size is %v\n", size, fi.Size) - } else { - fmt.Printf(" file sizes match\n") - } - - // convert list of blobs to []restic.Blob - var list []restic.Blob - for _, b := range blobs { - list = append(list, b.Blob) - } - - err = loadBlobs(gopts.ctx, repo, name, list) - if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - } else { - blobsLoaded = true - } - } - - fmt.Printf(" ========================================\n") - fmt.Printf(" inspect the pack itself\n") - - blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(gopts.ctx, repo.Backend(), h), fi.Size) - if err != nil { - fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", id.Str(), err) - return nil - } + fmt.Printf(" index %v:\n", idxIDs) // track current size and offset var size, offset uint64 @@ -551,12 +507,60 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { fmt.Printf(" file sizes match\n") } - if !blobsLoaded { - err = loadBlobs(gopts.ctx, repo, name, blobs) - if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - } + // convert list of blobs to []restic.Blob + var list []restic.Blob + for _, b := range blobs { + list = append(list, b.Blob) + } + + err = loadBlobs(ctx, repo, name, list) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + } else { + blobsLoaded = true + } + } + + fmt.Printf(" ========================================\n") + fmt.Printf(" inspect the pack itself\n") + + blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), fi.Size) + if err != nil { + fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", id.Str(), err) + return + } + + // track current size and offset + var size, offset uint64 + + sort.Slice(blobs, func(i, j int) bool { + return blobs[i].Offset < blobs[j].Offset + }) + + for _, pb := range blobs { + fmt.Printf(" %v blob %v, offset %-6d, raw length %-6d\n", pb.Type, pb.ID, pb.Offset, pb.Length) + if offset != uint64(pb.Offset) { + fmt.Printf(" hole in file, want offset %v, got %v\n", offset, pb.Offset) + } + offset += uint64(pb.Length) + size += uint64(pb.Length) + } + + // compute header size, per blob: 1 byte type, 4 byte length, 32 byte id + size += uint64(restic.CiphertextLength(len(blobs) * (1 + 4 + 32))) + // length in uint32 little endian + size += 4 + + if uint64(fi.Size) != size { + fmt.Printf(" file sizes do not match: computed %v from index, file size is %v\n", size, fi.Size) + } else { + fmt.Printf(" file sizes match\n") + } + + if !blobsLoaded { + err = loadBlobs(ctx, repo, name, blobs) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) } } - return nil } From 6774fc64545d739518a1f82dc0f2966393067bcc Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 10 Mar 2021 20:41:11 +0100 Subject: [PATCH 10/15] debug: check arguments and cleanup error handling --- cmd/restic/cmd_debug.go | 56 +++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 5134f5dca..3f60ba8db 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -313,13 +313,13 @@ func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte { return out } -func loadBlobs(ctx context.Context, repo restic.Repository, pack string, list []restic.Blob) error { +func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list []restic.Blob) error { be := repo.Backend() for _, blob := range list { fmt.Printf(" loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length) buf := make([]byte, blob.Length) h := restic.Handle{ - Name: pack, + Name: pack.String(), Type: restic.PackFile, } err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error { @@ -408,6 +408,20 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error { } func runDebugExamine(gopts GlobalOptions, args []string) error { + ids := make([]restic.ID, 0) + for _, name := range args { + id, err := restic.ParseID(name) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + continue + } + ids = append(ids, id) + } + + if len(ids) == 0 { + return errors.Fatal("no pack files to examine") + } + repo, err := OpenRepository(gopts) if err != nil { return err @@ -426,34 +440,34 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { return err } - for _, name := range args { - examinePack(gopts.ctx, repo, name) + for _, id := range ids { + err := examinePack(gopts.ctx, repo, id) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + } + if err == context.Canceled { + break + } } return nil } -func examinePack(ctx context.Context, repo restic.Repository, name string) { - blobsLoaded := false - fmt.Printf("examine %v\n", name) - id, err := restic.ParseID(name) - if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - } +func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error { + fmt.Printf("examine %v\n", id) h := restic.Handle{ Type: restic.PackFile, - Name: name, + Name: id.String(), } fi, err := repo.Backend().Stat(ctx, h) if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) + return err } - fmt.Printf(" file size is %v\n", fi.Size) buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h) if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) + return err } gotID := restic.Hash(buf) @@ -466,6 +480,7 @@ func examinePack(ctx context.Context, repo restic.Repository, name string) { fmt.Printf(" ========================================\n") fmt.Printf(" looking for info in the indexes\n") + blobsLoaded := false // examine all data the indexes have for the pack file for _, idx := range repo.Index().(*repository.MasterIndex).All() { idxIDs, err := idx.IDs() @@ -513,7 +528,7 @@ func examinePack(ctx context.Context, repo restic.Repository, name string) { list = append(list, b.Blob) } - err = loadBlobs(ctx, repo, name, list) + err = loadBlobs(ctx, repo, id, list) if err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) } else { @@ -526,8 +541,7 @@ func examinePack(ctx context.Context, repo restic.Repository, name string) { blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), fi.Size) if err != nil { - fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", id.Str(), err) - return + return fmt.Errorf("pack %v: %v", id.Str(), err) } // track current size and offset @@ -558,9 +572,7 @@ func examinePack(ctx context.Context, repo restic.Repository, name string) { } if !blobsLoaded { - err = loadBlobs(ctx, repo, name, blobs) - if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - } + return loadBlobs(ctx, repo, id, blobs) } + return nil } From fa7b9d5dfe7bb714a88e97f66adc5e6b42aa4fc3 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 10 Mar 2021 20:57:14 +0100 Subject: [PATCH 11/15] debug: Cleanup pack size checks --- cmd/restic/cmd_debug.go | 46 ++++++++++------------------------------- 1 file changed, 11 insertions(+), 35 deletions(-) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 3f60ba8db..77ddaed9d 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -469,7 +469,6 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro if err != nil { return err } - gotID := restic.Hash(buf) if !id.Equal(gotID) { fmt.Printf(" wanted hash %v, got %v\n", id, gotID) @@ -495,38 +494,12 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro fmt.Printf(" index %v:\n", idxIDs) - // track current size and offset - var size, offset uint64 - - sort.Slice(blobs, func(i, j int) bool { - return blobs[i].Offset < blobs[j].Offset - }) - - for _, pb := range blobs { - fmt.Printf(" %v blob %v, offset %-6d, raw length %-6d\n", pb.Type, pb.ID, pb.Offset, pb.Length) - if offset != uint64(pb.Offset) { - fmt.Printf(" hole in file, want offset %v, got %v\n", offset, pb.Offset) - } - offset += uint64(pb.Length) - size += uint64(pb.Length) - } - - // compute header size, per blob: 1 byte type, 4 byte length, 32 byte id - size += uint64(restic.CiphertextLength(len(blobs) * (1 + 4 + 32))) - // length in uint32 little endian - size += 4 - - if uint64(fi.Size) != size { - fmt.Printf(" file sizes do not match: computed %v from index, file size is %v\n", size, fi.Size) - } else { - fmt.Printf(" file sizes match\n") - } - // convert list of blobs to []restic.Blob var list []restic.Blob for _, b := range blobs { list = append(list, b.Blob) } + checkPackSize(list, fi.Size) err = loadBlobs(ctx, repo, id, list) if err != nil { @@ -543,7 +516,15 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro if err != nil { return fmt.Errorf("pack %v: %v", id.Str(), err) } + checkPackSize(blobs, fi.Size) + if !blobsLoaded { + return loadBlobs(ctx, repo, id, blobs) + } + return nil +} + +func checkPackSize(blobs []restic.Blob, fileSize int64) { // track current size and offset var size, offset uint64 @@ -565,14 +546,9 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro // length in uint32 little endian size += 4 - if uint64(fi.Size) != size { - fmt.Printf(" file sizes do not match: computed %v from index, file size is %v\n", size, fi.Size) + if uint64(fileSize) != size { + fmt.Printf(" file sizes do not match: computed %v from index, file size is %v\n", size, fileSize) } else { fmt.Printf(" file sizes match\n") } - - if !blobsLoaded { - return loadBlobs(ctx, repo, id, blobs) - } - return nil } From 547d9b384d4da8bfad45ccf90ed7c0ae09bdc696 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 10 Mar 2021 21:07:52 +0100 Subject: [PATCH 12/15] debug: cleanup repair code --- cmd/restic/cmd_debug.go | 136 +++++++++++++++++++--------------------- 1 file changed, 65 insertions(+), 71 deletions(-) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 77ddaed9d..636fcc743 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -213,86 +213,80 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by buf := make([]byte, len(input)) copy(buf, input) - for { - select { - case i, ok := <-ch: - if !ok { - return nil - } - if bytewise { - for j := 0; j < 255; j++ { - // flip bits - buf[i] ^= byte(j) + testFlip := func(idx int, pattern byte) bool { + // flip bits + buf[idx] ^= pattern - nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] - plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil) - if err == nil { - fmt.Printf("\n") - fmt.Printf(" blob could be repaired by XORing byte %v with 0x%02x\n", i, j) - fmt.Printf(" hash is %v\n", restic.Hash(plaintext)) - close(done) - found = true - fixed = plaintext - return nil - } + nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] + plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil) + if err == nil { + fmt.Printf("\n") + fmt.Printf(" blob could be repaired by XORing byte %v with 0x%02x\n", idx, pattern) + fmt.Printf(" hash is %v\n", restic.Hash(plaintext)) + close(done) + found = true + fixed = plaintext + return true + } - // flip bits back - buf[i] ^= byte(j) + // flip bits back + buf[idx] ^= pattern + return false + } + + for i := range ch { + if bytewise { + for j := 0; j < 255; j++ { + if testFlip(i, byte(j)) { + return nil } - } else { - for j := 0; j < 7; j++ { - // flip bit - buf[i] ^= (1 << uint(j)) - - nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] - plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil) - if err == nil { - fmt.Printf("\n") - fmt.Printf(" blob could be repaired by flipping bit %v in byte %v\n", j, i) - fmt.Printf(" hash is %v\n", restic.Hash(plaintext)) - close(done) - found = true - fixed = plaintext - return nil - } - - // flip bit back - buf[i] ^= (1 << uint(j)) + } + } else { + for j := 0; j < 7; j++ { + // flip each bit once + if testFlip(i, (1 << uint(j))) { + return nil } } } } + return nil }) } - start := time.Now() - info := time.Now() -outer: - for i := range input { - select { - case ch <- i: - case <-done: - fmt.Printf(" done after %v\n", time.Since(start)) - break outer - } + wg.Go(func() error { + defer close(ch) - if time.Since(info) > time.Second { - secs := time.Since(start).Seconds() - gps := float64(i) / secs - remaining := len(input) - i - eta := time.Duration(float64(remaining)/gps) * time.Second + start := time.Now() + info := time.Now() + for i := range input { + select { + case ch <- i: + case <-done: + fmt.Printf(" done after %v\n", time.Since(start)) + return nil + } - fmt.Printf("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v", - i, len(input), float32(i)/float32(len(input)*100), - gps, eta) - info = time.Now() + if time.Since(info) > time.Second { + secs := time.Since(start).Seconds() + gps := float64(i) / secs + remaining := len(input) - i + eta := time.Duration(float64(remaining)/gps) * time.Second + + fmt.Printf("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v", + i, len(input), float32(i)/float32(len(input))*100, gps, eta) + info = time.Now() + } } + return nil + }) + err := wg.Wait() + if err != nil { + panic("all go rountines can only return nil") } - close(ch) - wg.Wait() if !found { - fmt.Printf("\n blob could not be repaired by single bit flip\n") + fmt.Printf("\n blob could not be repaired\n") } return fixed } @@ -315,18 +309,17 @@ func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte { func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list []restic.Blob) error { be := repo.Backend() + h := restic.Handle{ + Name: pack.String(), + Type: restic.PackFile, + } for _, blob := range list { fmt.Printf(" loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length) buf := make([]byte, blob.Length) - h := restic.Handle{ - Name: pack.String(), - Type: restic.PackFile, - } err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error { n, err := io.ReadFull(rd, buf) if err != nil { - fmt.Fprintf(os.Stderr, "read error after %d bytes: %v\n", n, err) - return err + return fmt.Errorf("read error after %d bytes: %v\n", n, err) } return nil }) @@ -349,9 +342,10 @@ func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list if plain != nil { id := restic.Hash(plain) if !id.Equal(blob.ID) { - fmt.Printf(" successfully repaired blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plain), id, blob.ID) + fmt.Printf(" repaired blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plain), id, blob.ID) prefix = "repaired-wrong-hash-" } else { + fmt.Printf(" successfully repaired blob (length %v), hash is %v, ID matches\n", len(plain), id) prefix = "repaired-" } } else { From dc62ec5933bbd8bd667351102d115dc7494e0250 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 10 Mar 2021 21:20:21 +0100 Subject: [PATCH 13/15] debug: use Printf/Warnf for output --- cmd/restic/cmd_debug.go | 66 ++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 636fcc743..2e5b62f47 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -194,9 +194,9 @@ var cmdDebugExamine = &cobra.Command{ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, bytewise bool) []byte { if bytewise { - fmt.Printf(" trying to repair blob by finding a broken byte\n") + Printf(" trying to repair blob by finding a broken byte\n") } else { - fmt.Printf(" trying to repair blob with single bit flip\n") + Printf(" trying to repair blob with single bit flip\n") } ch := make(chan int) @@ -206,7 +206,7 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by var found bool workers := runtime.GOMAXPROCS(0) - fmt.Printf(" spinning up %d worker functions\n", runtime.GOMAXPROCS(0)) + Printf(" spinning up %d worker functions\n", runtime.GOMAXPROCS(0)) for i := 0; i < workers; i++ { wg.Go(func() error { // make a local copy of the buffer @@ -220,9 +220,9 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil) if err == nil { - fmt.Printf("\n") - fmt.Printf(" blob could be repaired by XORing byte %v with 0x%02x\n", idx, pattern) - fmt.Printf(" hash is %v\n", restic.Hash(plaintext)) + Printf("\n") + Printf(" blob could be repaired by XORing byte %v with 0x%02x\n", idx, pattern) + Printf(" hash is %v\n", restic.Hash(plaintext)) close(done) found = true fixed = plaintext @@ -263,7 +263,7 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by select { case ch <- i: case <-done: - fmt.Printf(" done after %v\n", time.Since(start)) + Printf(" done after %v\n", time.Since(start)) return nil } @@ -273,7 +273,7 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by remaining := len(input) - i eta := time.Duration(float64(remaining)/gps) * time.Second - fmt.Printf("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v", + Printf("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v", i, len(input), float32(i)/float32(len(input))*100, gps, eta) info = time.Now() } @@ -286,7 +286,7 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by } if !found { - fmt.Printf("\n blob could not be repaired\n") + Printf("\n blob could not be repaired\n") } return fixed } @@ -314,7 +314,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list Type: restic.PackFile, } for _, blob := range list { - fmt.Printf(" loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length) + Printf(" loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length) buf := make([]byte, blob.Length) err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error { n, err := io.ReadFull(rd, buf) @@ -324,7 +324,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list return nil }) if err != nil { - fmt.Fprintf(os.Stderr, "error read: %v\n", err) + Warnf("error read: %v\n", err) continue } @@ -333,7 +333,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil) if err != nil { - fmt.Fprintf(os.Stderr, "error decrypting blob: %v\n", err) + Warnf("error decrypting blob: %v\n", err) var plain []byte if tryRepair || repairByte { plain = tryRepairWithBitflip(ctx, key, buf, repairByte) @@ -342,10 +342,10 @@ func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list if plain != nil { id := restic.Hash(plain) if !id.Equal(blob.ID) { - fmt.Printf(" repaired blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plain), id, blob.ID) + Printf(" repaired blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plain), id, blob.ID) prefix = "repaired-wrong-hash-" } else { - fmt.Printf(" successfully repaired blob (length %v), hash is %v, ID matches\n", len(plain), id) + Printf(" successfully repaired blob (length %v), hash is %v, ID matches\n", len(plain), id) prefix = "repaired-" } } else { @@ -362,10 +362,10 @@ func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list id := restic.Hash(plaintext) var prefix string if !id.Equal(blob.ID) { - fmt.Printf(" successfully decrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plaintext), id, blob.ID) + Printf(" successfully decrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plaintext), id, blob.ID) prefix = "wrong-hash-" } else { - fmt.Printf(" successfully decrypted blob (length %v), hash is %v, ID matches\n", len(plaintext), id) + Printf(" successfully decrypted blob (length %v), hash is %v, ID matches\n", len(plaintext), id) prefix = "correct-" } if extractPack { @@ -397,7 +397,7 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error { return err } - fmt.Printf("decrypt of blob %v stored at %v\n", id, filename) + Printf("decrypt of blob %v stored at %v\n", id, filename) return nil } @@ -406,7 +406,7 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { for _, name := range args { id, err := restic.ParseID(name) if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) + Warnf("error: %v\n", err) continue } ids = append(ids, id) @@ -437,7 +437,7 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { for _, id := range ids { err := examinePack(gopts.ctx, repo, id) if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) + Warnf("error: %v\n", err) } if err == context.Canceled { break @@ -447,7 +447,7 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { } func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error { - fmt.Printf("examine %v\n", id) + Printf("examine %v\n", id) h := restic.Handle{ Type: restic.PackFile, @@ -457,7 +457,7 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro if err != nil { return err } - fmt.Printf(" file size is %v\n", fi.Size) + Printf(" file size is %v\n", fi.Size) buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h) if err != nil { @@ -465,13 +465,13 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro } gotID := restic.Hash(buf) if !id.Equal(gotID) { - fmt.Printf(" wanted hash %v, got %v\n", id, gotID) + Printf(" wanted hash %v, got %v\n", id, gotID) } else { - fmt.Printf(" hash for file content matches\n") + Printf(" hash for file content matches\n") } - fmt.Printf(" ========================================\n") - fmt.Printf(" looking for info in the indexes\n") + Printf(" ========================================\n") + Printf(" looking for info in the indexes\n") blobsLoaded := false // examine all data the indexes have for the pack file @@ -486,7 +486,7 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro continue } - fmt.Printf(" index %v:\n", idxIDs) + Printf(" index %v:\n", idxIDs) // convert list of blobs to []restic.Blob var list []restic.Blob @@ -497,14 +497,14 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro err = loadBlobs(ctx, repo, id, list) if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) + Warnf("error: %v\n", err) } else { blobsLoaded = true } } - fmt.Printf(" ========================================\n") - fmt.Printf(" inspect the pack itself\n") + Printf(" ========================================\n") + Printf(" inspect the pack itself\n") blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), fi.Size) if err != nil { @@ -527,9 +527,9 @@ func checkPackSize(blobs []restic.Blob, fileSize int64) { }) for _, pb := range blobs { - fmt.Printf(" %v blob %v, offset %-6d, raw length %-6d\n", pb.Type, pb.ID, pb.Offset, pb.Length) + Printf(" %v blob %v, offset %-6d, raw length %-6d\n", pb.Type, pb.ID, pb.Offset, pb.Length) if offset != uint64(pb.Offset) { - fmt.Printf(" hole in file, want offset %v, got %v\n", offset, pb.Offset) + Printf(" hole in file, want offset %v, got %v\n", offset, pb.Offset) } offset += uint64(pb.Length) size += uint64(pb.Length) @@ -541,8 +541,8 @@ func checkPackSize(blobs []restic.Blob, fileSize int64) { size += 4 if uint64(fileSize) != size { - fmt.Printf(" file sizes do not match: computed %v from index, file size is %v\n", size, fileSize) + Printf(" file sizes do not match: computed %v from index, file size is %v\n", size, fileSize) } else { - fmt.Printf(" file sizes match\n") + Printf(" file sizes match\n") } } From 5975ed61f37e403c3b3ed7b630f329b82b25bfb6 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 10 Mar 2021 21:22:53 +0100 Subject: [PATCH 14/15] debug: fix linter warning --- cmd/restic/cmd_debug.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 2e5b62f47..653c4a99c 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -319,7 +319,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error { n, err := io.ReadFull(rd, buf) if err != nil { - return fmt.Errorf("read error after %d bytes: %v\n", n, err) + return fmt.Errorf("read error after %d bytes: %v", n, err) } return nil }) From 54d58edacc4af8b33883733c8b7fa86b099fc50c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 10 Mar 2021 22:22:33 +0100 Subject: [PATCH 15/15] debug: fix usage for examine command --- cmd/restic/cmd_debug.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 653c4a99c..42433d5b9 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -184,7 +184,7 @@ func runDebugDump(gopts GlobalOptions, args []string) error { } var cmdDebugExamine = &cobra.Command{ - Use: "examine", + Use: "examine pack-ID...", Short: "Examine a pack file", DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error {