2020-08-05 21:32:15 +02:00
package main
import (
2022-12-10 17:18:04 +01:00
"context"
2020-08-05 21:32:15 +02:00
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
2022-12-10 17:18:04 +01:00
"golang.org/x/sync/errgroup"
2020-08-05 21:32:15 +02:00
"github.com/spf13/cobra"
)
var cmdRepair = & cobra . Command {
2022-12-10 17:25:38 +01:00
Use : "repair" ,
Short : "Repair commands" ,
}
var cmdRepairSnapshots = & cobra . Command {
Use : "snapshots [flags] [snapshot ID] [...]" ,
2020-08-05 21:32:15 +02:00
Short : "Repair snapshots" ,
Long : `
2022-12-10 17:25:38 +01:00
The "repair snapshots" command allows to repair broken snapshots .
2020-08-05 21:32:15 +02:00
It scans the given snapshots and generates new ones where
damaged tress and file contents are removed .
If the broken snapshots are deleted , a prune run will
be able to refit the repository .
The command depends on a good state of the index , so if
there are inaccurancies in the index , make sure to run
"rebuild-index" before !
WARNING :
== == == ==
Repairing and deleting broken snapshots causes data loss !
It will remove broken dirs and modify broken files in
the modified snapshots .
If the contents of directories and files are still available ,
the better option is to redo a backup which in that case is
able to "heal" already present snapshots .
Only use this command if you need to recover an old and
broken snapshot !
EXIT STATUS
== == == == == =
Exit status is 0 if the command was successful , and non - zero if there was any error .
` ,
DisableAutoGenTag : true ,
RunE : func ( cmd * cobra . Command , args [ ] string ) error {
2022-12-10 17:25:38 +01:00
return runRepairSnapshots ( cmd . Context ( ) , globalOptions , repairSnapshotOptions , args )
2020-08-05 21:32:15 +02:00
} ,
}
2021-02-20 20:56:03 +01:00
// RepairOptions collects all options for the repair command.
2020-08-05 21:32:15 +02:00
type RepairOptions struct {
2022-12-10 17:18:04 +01:00
restic . SnapshotFilter
2020-08-05 21:32:15 +02:00
AddTag string
Append string
DryRun bool
DeleteSnapshots bool
}
2022-12-10 17:25:38 +01:00
var repairSnapshotOptions RepairOptions
2020-08-05 21:32:15 +02:00
func init ( ) {
cmdRoot . AddCommand ( cmdRepair )
2022-12-10 17:25:38 +01:00
cmdRepair . AddCommand ( cmdRepairSnapshots )
flags := cmdRepairSnapshots . Flags ( )
2022-12-10 17:18:04 +01:00
2022-12-10 17:25:38 +01:00
initMultiSnapshotFilter ( flags , & repairSnapshotOptions . SnapshotFilter , true )
2022-12-10 17:18:04 +01:00
2022-12-10 17:25:38 +01:00
flags . StringVar ( & repairSnapshotOptions . AddTag , "add-tag" , "repaired" , "tag to add to repaired snapshots" )
flags . StringVar ( & repairSnapshotOptions . Append , "append" , ".repaired" , "string to append to repaired dirs/files; remove files if empty or impossible to repair" )
flags . BoolVarP ( & repairSnapshotOptions . DryRun , "dry-run" , "n" , true , "don't do anything, only show what would be done" )
flags . BoolVar ( & repairSnapshotOptions . DeleteSnapshots , "delete-snapshots" , false , "delete original snapshots" )
2020-08-05 21:32:15 +02:00
}
2022-12-10 17:25:38 +01:00
func runRepairSnapshots ( ctx context . Context , gopts GlobalOptions , opts RepairOptions , args [ ] string ) error {
2020-08-05 21:32:15 +02:00
switch {
case opts . DryRun :
Printf ( "\n note: --dry-run is set\n-> repair will only show what it would do.\n\n" )
case opts . DeleteSnapshots :
2021-02-20 20:56:03 +01:00
Printf ( "\n note: --dry-run is not set and --delete-snapshots is set\n-> this may result in data loss!\n\n" )
2020-08-05 21:32:15 +02:00
}
2022-12-10 17:18:04 +01:00
repo , err := OpenRepository ( ctx , globalOptions )
2020-08-05 21:32:15 +02:00
if err != nil {
return err
}
2022-12-10 17:18:04 +01:00
lock , ctx , err := lockRepoExclusive ( ctx , repo , gopts . RetryLock , gopts . JSON )
2021-02-20 20:16:05 +01:00
defer unlockRepo ( lock )
2020-08-05 21:32:15 +02:00
if err != nil {
return err
}
2022-12-10 17:18:04 +01:00
if err := repo . LoadIndex ( ctx ) ; err != nil {
2020-08-05 21:32:15 +02:00
return err
}
// get snapshots to check & repair
var snapshots [ ] * restic . Snapshot
2022-12-10 17:18:04 +01:00
for sn := range FindFilteredSnapshots ( ctx , repo . Backend ( ) , repo , & opts . SnapshotFilter , args ) {
2020-08-05 21:32:15 +02:00
snapshots = append ( snapshots , sn )
}
2022-12-10 17:18:04 +01:00
return repairSnapshots ( ctx , opts , repo , snapshots )
2020-08-05 21:32:15 +02:00
}
2022-12-10 17:18:04 +01:00
func repairSnapshots ( ctx context . Context , opts RepairOptions , repo restic . Repository , snapshots [ ] * restic . Snapshot ) error {
2020-08-05 21:32:15 +02:00
replaces := make ( idMap )
seen := restic . NewIDSet ( )
deleteSn := restic . NewIDSet ( )
Verbosef ( "check and repair %d snapshots\n" , len ( snapshots ) )
bar := newProgressMax ( ! globalOptions . Quiet , uint64 ( len ( snapshots ) ) , "snapshots" )
2022-12-10 17:18:04 +01:00
wg , ctx := errgroup . WithContext ( ctx )
repo . StartPackUploader ( ctx , wg )
wg . Go ( func ( ) error {
for _ , sn := range snapshots {
debug . Log ( "process snapshot %v" , sn . ID ( ) )
Printf ( "%v:\n" , sn )
newID , changed , lErr , err := repairTree ( ctx , opts , repo , "/" , sn . Tree , replaces , seen )
switch {
case err != nil :
2020-08-05 21:32:15 +02:00
return err
2022-12-10 17:18:04 +01:00
case lErr :
Printf ( "the root tree is damaged -> delete snapshot.\n" )
deleteSn . Insert ( * sn . ID ( ) )
case changed :
err = changeSnapshot ( ctx , opts , repo , sn , newID )
if err != nil {
return err
}
deleteSn . Insert ( * sn . ID ( ) )
default :
Printf ( "is ok.\n" )
2020-08-05 21:32:15 +02:00
}
2022-12-10 17:18:04 +01:00
debug . Log ( "processed snapshot %v" , sn . ID ( ) )
bar . Add ( 1 )
2020-08-05 21:32:15 +02:00
}
2022-12-10 17:18:04 +01:00
bar . Done ( )
return repo . Flush ( ctx )
} )
2020-08-05 21:32:15 +02:00
2022-12-10 17:18:04 +01:00
err := wg . Wait ( )
2020-08-05 21:32:15 +02:00
if err != nil {
return err
}
if len ( deleteSn ) > 0 && opts . DeleteSnapshots {
Verbosef ( "delete %d snapshots...\n" , len ( deleteSn ) )
if ! opts . DryRun {
2022-12-10 17:18:04 +01:00
DeleteFiles ( ctx , globalOptions , repo , deleteSn , restic . SnapshotFile )
2020-08-05 21:32:15 +02:00
}
}
return nil
}
// changeSnapshot creates a modified snapshot:
// - set the tree to newID
// - add the rag opts.AddTag
// - preserve original ID
// if opts.DryRun is set, it doesn't change anything but only
2022-12-10 17:18:04 +01:00
func changeSnapshot ( ctx context . Context , opts RepairOptions , repo restic . Repository , sn * restic . Snapshot , newID * restic . ID ) error {
2020-08-05 21:32:15 +02:00
sn . AddTags ( [ ] string { opts . AddTag } )
// Retain the original snapshot id over all tag changes.
if sn . Original == nil {
sn . Original = sn . ID ( )
}
2021-02-20 20:16:05 +01:00
sn . Tree = newID
2020-08-05 21:32:15 +02:00
if ! opts . DryRun {
2022-12-10 17:18:04 +01:00
newID , err := restic . SaveSnapshot ( ctx , repo , sn )
2020-08-05 21:32:15 +02:00
if err != nil {
return err
}
Printf ( "snapshot repaired -> %v created.\n" , newID . Str ( ) )
} else {
2021-02-20 20:56:03 +01:00
Printf ( "would have repaired snapshot %v.\n" , sn . ID ( ) . Str ( ) )
2020-08-05 21:32:15 +02:00
}
return nil
}
type idMap map [ restic . ID ] restic . ID
// repairTree checks and repairs a tree and all its subtrees
2021-02-20 20:16:05 +01:00
// Three error cases are checked:
// - tree is a nil tree (-> will be replaced by an empty tree)
2020-08-05 21:32:15 +02:00
// - trees which cannot be loaded (-> the tree contents will be removed)
// - files whose contents are not fully available (-> file will be modified)
// In case of an error, the changes made depends on:
// - opts.Append: string to append to "repared" names; if empty files will not repaired but deleted
// - opts.DryRun: if set to true, only print out what to but don't change anything
2021-02-20 20:16:05 +01:00
// Returns:
// - the new ID
// - whether the ID changed
// - whether there was a load error when loading this tre
// - error for other errors (these are errors when saving a tree)
2022-12-10 17:18:04 +01:00
func repairTree ( ctx context . Context , opts RepairOptions , repo restic . Repository , path string , treeID * restic . ID , replaces idMap , seen restic . IDSet ) ( * restic . ID , bool , bool , error ) {
2021-02-20 20:16:05 +01:00
// handle and repair nil trees
if treeID == nil {
2022-12-10 17:18:04 +01:00
empty , err := emptyTree ( ctx , repo , opts . DryRun )
2021-02-20 20:16:05 +01:00
Printf ( "repaired nil tree '%v'\n" , path )
return & empty , true , false , err
}
2020-08-05 21:32:15 +02:00
// check if tree was already changed
2021-02-20 20:16:05 +01:00
newID , ok := replaces [ * treeID ]
2020-08-05 21:32:15 +02:00
if ok {
2021-02-20 20:16:05 +01:00
return & newID , true , false , nil
2020-08-05 21:32:15 +02:00
}
// check if tree was seen but not changed
2021-02-20 20:16:05 +01:00
if seen . Has ( * treeID ) {
return treeID , false , false , nil
2020-08-05 21:32:15 +02:00
}
2022-12-10 17:18:04 +01:00
tree , err := restic . LoadTree ( ctx , repo , * treeID )
2020-08-05 21:32:15 +02:00
if err != nil {
2021-02-20 20:16:05 +01:00
// mark as load error
return & newID , false , true , nil
2020-08-05 21:32:15 +02:00
}
var newNodes [ ] * restic . Node
changed := false
for _ , node := range tree . Nodes {
switch node . Type {
case "file" :
ok := true
var newContent restic . IDs
var newSize uint64
// check all contents and remove if not available
for _ , id := range node . Content {
if size , found := repo . LookupBlobSize ( id , restic . DataBlob ) ; ! found {
ok = false
} else {
newContent = append ( newContent , id )
newSize += uint64 ( size )
}
}
if ! ok {
changed = true
if opts . Append == "" || newSize == 0 {
2021-02-20 20:56:03 +01:00
Printf ( "removed defective file '%v'\n" , path + node . Name )
2020-08-05 21:32:15 +02:00
continue
}
2021-02-20 20:56:03 +01:00
Printf ( "repaired defective file '%v'" , path + node . Name )
2020-08-05 21:32:15 +02:00
node . Name = node . Name + opts . Append
Printf ( " to '%v'\n" , node . Name )
node . Content = newContent
node . Size = newSize
}
case "dir" :
// rewrite if necessary
2022-12-10 17:18:04 +01:00
newID , c , lErr , err := repairTree ( ctx , opts , repo , path + node . Name + "/" , node . Subtree , replaces , seen )
2020-08-05 21:32:15 +02:00
switch {
case err != nil :
2021-02-20 20:16:05 +01:00
return newID , true , false , err
case lErr :
2020-08-05 21:32:15 +02:00
// If we get an error, we remove this subtree
changed = true
2021-02-20 20:56:03 +01:00
Printf ( "removed defective dir '%v'" , path + node . Name )
2020-08-05 21:32:15 +02:00
node . Name = node . Name + opts . Append
2021-02-20 20:56:03 +01:00
Printf ( "(now empty '%v')\n" , node . Name )
2022-12-10 17:18:04 +01:00
empty , err := emptyTree ( ctx , repo , opts . DryRun )
2021-02-20 20:16:05 +01:00
if err != nil {
return newID , true , false , err
}
node . Subtree = & empty
2020-08-05 21:32:15 +02:00
case c :
2021-02-20 20:16:05 +01:00
node . Subtree = newID
2020-08-05 21:32:15 +02:00
changed = true
}
}
newNodes = append ( newNodes , node )
}
if ! changed {
2021-02-20 20:16:05 +01:00
seen . Insert ( * treeID )
return treeID , false , false , nil
2020-08-05 21:32:15 +02:00
}
tree . Nodes = newNodes
if ! opts . DryRun {
2022-12-10 17:18:04 +01:00
newID , err = restic . SaveTree ( ctx , repo , tree )
2020-08-05 21:32:15 +02:00
if err != nil {
2021-02-20 20:16:05 +01:00
return & newID , true , false , err
2020-08-05 21:32:15 +02:00
}
Printf ( "modified tree %v, new id: %v\n" , treeID . Str ( ) , newID . Str ( ) )
} else {
Printf ( "would have modified tree %v\n" , treeID . Str ( ) )
}
2021-02-20 20:16:05 +01:00
replaces [ * treeID ] = newID
return & newID , true , false , nil
}
2022-12-10 17:18:04 +01:00
func emptyTree ( ctx context . Context , repo restic . Repository , dryRun bool ) ( restic . ID , error ) {
2021-02-20 20:16:05 +01:00
if ! dryRun {
2022-12-10 17:18:04 +01:00
return restic . SaveTree ( ctx , repo , & restic . Tree { } )
2021-02-20 20:16:05 +01:00
}
return restic . ID { } , nil
2020-08-05 21:32:15 +02:00
}