diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 57f18d057..626b822b1 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -22,7 +22,6 @@ import ( "github.com/restic/restic/internal/archiver" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -79,30 +78,28 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea // BackupOptions bundles all options for the backup command. type BackupOptions struct { - Parent string - Force bool - Excludes []string - InsensitiveExcludes []string - ExcludeFiles []string - InsensitiveExcludeFiles []string - ExcludeOtherFS bool - ExcludeIfPresent []string - ExcludeCaches bool - ExcludeLargerThan string - Stdin bool - StdinFilename string - Tags restic.TagLists - Host string - FilesFrom []string - FilesFromVerbatim []string - FilesFromRaw []string - TimeStamp string - WithAtime bool - IgnoreInode bool - IgnoreCtime bool - UseFsSnapshot bool - DryRun bool - ReadConcurrency uint + excludePatternOptions + + Parent string + Force bool + ExcludeOtherFS bool + ExcludeIfPresent []string + ExcludeCaches bool + ExcludeLargerThan string + Stdin bool + StdinFilename string + Tags restic.TagLists + Host string + FilesFrom []string + FilesFromVerbatim []string + FilesFromRaw []string + TimeStamp string + WithAtime bool + IgnoreInode bool + IgnoreCtime bool + UseFsSnapshot bool + DryRun bool + ReadConcurrency uint } var backupOptions BackupOptions @@ -116,10 +113,9 @@ func init() { f := cmdBackup.Flags() f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repository that has the same target files/directories, and is not newer than the snapshot time)") f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`) - f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") - f.StringArrayVar(&backupOptions.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames") - f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)") - f.StringArrayVar(&backupOptions.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns") + + initExcludePatternOptions(f, &backupOptions.excludePatternOptions) + f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes") f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)") f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file. See https://bford.info/cachedir/ for the Cache Directory Tagging Standard`) @@ -306,48 +302,11 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, t fs = append(fs, f) } - // add patterns from file - if len(opts.ExcludeFiles) > 0 { - excludes, err := readExcludePatternsFromFiles(opts.ExcludeFiles) - if err != nil { - return nil, err - } - - if err := filter.ValidatePatterns(excludes); err != nil { - return nil, errors.Fatalf("--exclude-file: %s", err) - } - - opts.Excludes = append(opts.Excludes, excludes...) - } - - if len(opts.InsensitiveExcludeFiles) > 0 { - excludes, err := readExcludePatternsFromFiles(opts.InsensitiveExcludeFiles) - if err != nil { - return nil, err - } - - if err := filter.ValidatePatterns(excludes); err != nil { - return nil, errors.Fatalf("--iexclude-file: %s", err) - } - - opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...) - } - - if len(opts.InsensitiveExcludes) > 0 { - if err := filter.ValidatePatterns(opts.InsensitiveExcludes); err != nil { - return nil, errors.Fatalf("--iexclude: %s", err) - } - - fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes)) - } - - if len(opts.Excludes) > 0 { - if err := filter.ValidatePatterns(opts.Excludes); err != nil { - return nil, errors.Fatalf("--exclude: %s", err) - } - - fs = append(fs, rejectByPattern(opts.Excludes)) + fsPatterns, err := collectExcludePatterns(opts.excludePatternOptions) + if err != nil { + return nil, err } + fs = append(fs, fsPatterns...) if opts.ExcludeCaches { opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55") @@ -388,53 +347,6 @@ func collectRejectFuncs(opts BackupOptions, repo *repository.Repository, targets return fs, nil } -// readExcludePatternsFromFiles reads all exclude files and returns the list of -// exclude patterns. For each line, leading and trailing white space is removed -// and comment lines are ignored. For each remaining pattern, environment -// variables are resolved. For adding a literal dollar sign ($), write $$ to -// the file. -func readExcludePatternsFromFiles(excludeFiles []string) ([]string, error) { - getenvOrDollar := func(s string) string { - if s == "$" { - return "$" - } - return os.Getenv(s) - } - - var excludes []string - for _, filename := range excludeFiles { - err := func() (err error) { - data, err := textfile.Read(filename) - if err != nil { - return err - } - - scanner := bufio.NewScanner(bytes.NewReader(data)) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - - // ignore empty lines - if line == "" { - continue - } - - // strip comments - if strings.HasPrefix(line, "#") { - continue - } - - line = os.Expand(line, getenvOrDollar) - excludes = append(excludes, line) - } - return scanner.Err() - }() - if err != nil { - return nil, err - } - } - return excludes, nil -} - // collectTargets returns a list of target files/dirs from several sources. func collectTargets(opts BackupOptions, args []string) (targets []string, err error) { if opts.Stdin { diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go index 1ddd8932c..86f85f133 100644 --- a/cmd/restic/exclude.go +++ b/cmd/restic/exclude.go @@ -1,6 +1,7 @@ package main import ( + "bufio" "bytes" "fmt" "io" @@ -15,6 +16,8 @@ import ( "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/textfile" + "github.com/spf13/pflag" ) type rejectionCache struct { @@ -410,3 +413,111 @@ func parseSizeStr(sizeStr string) (int64, error) { } return value * unit, nil } + +// readExcludePatternsFromFiles reads all exclude files and returns the list of +// exclude patterns. For each line, leading and trailing white space is removed +// and comment lines are ignored. For each remaining pattern, environment +// variables are resolved. For adding a literal dollar sign ($), write $$ to +// the file. +func readExcludePatternsFromFiles(excludeFiles []string) ([]string, error) { + getenvOrDollar := func(s string) string { + if s == "$" { + return "$" + } + return os.Getenv(s) + } + + var excludes []string + for _, filename := range excludeFiles { + err := func() (err error) { + data, err := textfile.Read(filename) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + + // ignore empty lines + if line == "" { + continue + } + + // strip comments + if strings.HasPrefix(line, "#") { + continue + } + + line = os.Expand(line, getenvOrDollar) + excludes = append(excludes, line) + } + return scanner.Err() + }() + if err != nil { + return nil, err + } + } + return excludes, nil +} + +type excludePatternOptions struct { + Excludes []string + InsensitiveExcludes []string + ExcludeFiles []string + InsensitiveExcludeFiles []string +} + +func initExcludePatternOptions(f *pflag.FlagSet, opts *excludePatternOptions) { + f.StringArrayVarP(&opts.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") + f.StringArrayVar(&opts.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames") + f.StringArrayVar(&opts.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)") + f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns") +} + +func collectExcludePatterns(opts excludePatternOptions) ([]RejectByNameFunc, error) { + var fs []RejectByNameFunc + // add patterns from file + if len(opts.ExcludeFiles) > 0 { + excludePatterns, err := readExcludePatternsFromFiles(opts.ExcludeFiles) + if err != nil { + return nil, err + } + + if err := filter.ValidatePatterns(excludePatterns); err != nil { + return nil, errors.Fatalf("--exclude-file: %s", err) + } + + opts.Excludes = append(opts.Excludes, excludePatterns...) + } + + if len(opts.InsensitiveExcludeFiles) > 0 { + excludes, err := readExcludePatternsFromFiles(opts.InsensitiveExcludeFiles) + if err != nil { + return nil, err + } + + if err := filter.ValidatePatterns(excludes); err != nil { + return nil, errors.Fatalf("--iexclude-file: %s", err) + } + + opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...) + } + + if len(opts.InsensitiveExcludes) > 0 { + if err := filter.ValidatePatterns(opts.InsensitiveExcludes); err != nil { + return nil, errors.Fatalf("--iexclude: %s", err) + } + + fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes)) + } + + if len(opts.Excludes) > 0 { + if err := filter.ValidatePatterns(opts.Excludes); err != nil { + return nil, errors.Fatalf("--exclude: %s", err) + } + + fs = append(fs, rejectByPattern(opts.Excludes)) + } + return fs, nil +} diff --git a/cmd/restic/integration_filter_pattern_test.go b/cmd/restic/integration_filter_pattern_test.go index c0c1d932f..962158d94 100644 --- a/cmd/restic/integration_filter_pattern_test.go +++ b/cmd/restic/integration_filter_pattern_test.go @@ -24,14 +24,14 @@ func TestBackupFailsWhenUsingInvalidPatterns(t *testing.T) { var err error // Test --exclude - err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts) + err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) // Test --iexclude - err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts) + err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided: *[._]log[.-][0-9] @@ -54,14 +54,14 @@ func TestBackupFailsWhenUsingInvalidPatternsFromFile(t *testing.T) { var err error // Test --exclude-file: - err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludeFiles: []string{excludeFile}}, env.gopts) + err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{excludeFile}}}, env.gopts) rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) // Test --iexclude-file - err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{InsensitiveExcludeFiles: []string{excludeFile}}, env.gopts) + err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{excludeFile}}}, env.gopts) rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided: *[._]log[.-][0-9]