clitools
/
restic
mirror of https://github.com/restic/restic.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #2087 from ArcticXWolf/add_group_by_option_for_snapshots 

Add GroupBy option to snapshots command
This commit is contained in:
Alexander Neumann 2019-04-22 20:27:24 +02:00
parent ecc2458de8 9a26be4e5b
commit a164dc9391
5 changed files with 312 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
changelog/unreleased/pull-2087 Normal file
View File

@ -0,0 +1,10 @@
Enhancement: Add group-by option to snapshots command
We have added an option to group the output of the snapshots command, similar
to the output of the forget command. The option has been called "--group-by"
and accepts any combination of the values "host", "paths" and "tags", separated
by commas. Default behavior (not specifying --group-by) has not been changed.
We have added support of the grouping to the JSON output.
https://github.com/restic/restic/issues/2037
https://github.com/restic/restic/pull/2087

232
cmd/restic/cmd_forget.go
View File

@ -4,10 +4,7 @@ import (
"context"
"encoding/json"
"io"
"sort"
"strings"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/spf13/cobra"
)
@ -91,178 +88,129 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
return err
}
// group by hostname and dirs
type key struct {
Hostname string
Paths []string
Tags []string
}
snapshotGroups := make(map[string]restic.Snapshots)
var GroupByTag bool
var GroupByHost bool
var GroupByPath bool
var GroupOptionList []string
GroupOptionList = strings.Split(opts.GroupBy, ",")
for _, option := range GroupOptionList {
switch option {
case "host":
GroupByHost = true
case "paths":
GroupByPath = true
case "tags":
GroupByTag = true
case "":
default:
return errors.Fatal("unknown grouping option: '" + option + "'")
}
}
removeSnapshots := 0
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
var snapshots restic.Snapshots
for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
if len(args) > 0 {
// When explicit snapshots args are given, remove them immediately.
snapshots = append(snapshots, sn)
}
if len(args) > 0 {
// When explicit snapshots args are given, remove them immediately.
for _, sn := range snapshots {
if !opts.DryRun {
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
if err = repo.Backend().Remove(gopts.ctx, h); err != nil {
return err
}
Verbosef("removed snapshot %v\n", sn.ID().Str())
if !gopts.JSON {
Verbosef("removed snapshot %v\n", sn.ID().Str())
}
removeSnapshots++
} else {
Verbosef("would have removed snapshot %v\n", sn.ID().Str())
if !gopts.JSON {
Verbosef("would have removed snapshot %v\n", sn.ID().Str())
}
}
} else {
// Determining grouping-keys
var tags []string
var hostname string
var paths []string
if GroupByTag {
tags = sn.Tags
sort.StringSlice(tags).Sort()
}
if GroupByHost {
hostname = sn.Hostname
}
if GroupByPath {
paths = sn.Paths
}
sort.StringSlice(sn.Paths).Sort()
var k []byte
var err error
k, err = json.Marshal(key{Tags: tags, Hostname: hostname, Paths: paths})
if err != nil {
return err
}
snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
}
}
policy := restic.ExpirePolicy{
Last: opts.Last,
Hourly: opts.Hourly,
Daily: opts.Daily,
Weekly: opts.Weekly,
Monthly: opts.Monthly,
Yearly: opts.Yearly,
Within: opts.Within,
Tags: opts.KeepTags,
}
if policy.Empty() && len(args) == 0 {
Verbosef("no policy was specified, no snapshots will be removed\n")
}
if !policy.Empty() {
if !gopts.JSON {
Verbosef("Applying Policy: %v\n", policy)
} else {
snapshotGroups, _, err := restic.GroupSnapshots(snapshots, opts.GroupBy)
if err != nil {
return err
}
var jsonGroups []*ForgetGroup
policy := restic.ExpirePolicy{
Last: opts.Last,
Hourly: opts.Hourly,
Daily: opts.Daily,
Weekly: opts.Weekly,
Monthly: opts.Monthly,
Yearly: opts.Yearly,
Within: opts.Within,
Tags: opts.KeepTags,
}
for k, snapshotGroup := range snapshotGroups {
var key key
if json.Unmarshal([]byte(k), &key) != nil {
return err
}
var fg ForgetGroup
// Info
if policy.Empty() && len(args) == 0 {
if !gopts.JSON {
Verbosef("snapshots")
}
var infoStrings []string
if GroupByTag {
infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]")
fg.Tags = key.Tags
}
if GroupByHost {
infoStrings = append(infoStrings, "host ["+key.Hostname+"]")
fg.Host = key.Hostname
}
if GroupByPath {
infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]")
fg.Paths = key.Paths
}
if infoStrings != nil && !gopts.JSON {
Verbosef(" for (" + strings.Join(infoStrings, ", ") + ")")
Verbosef("no policy was specified, no snapshots will be removed\n")
}
}
if !policy.Empty() {
if !gopts.JSON {
Verbosef(":\n\n")
Verbosef("Applying Policy: %v\n", policy)
}
keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
var jsonGroups []*ForgetGroup
if len(keep) != 0 && !gopts.Quiet && !gopts.JSON {
Printf("keep %d snapshots:\n", len(keep))
PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact)
Printf("\n")
}
addJSONSnapshots(&fg.Keep, keep)
if len(remove) != 0 && !gopts.Quiet && !gopts.JSON {
Printf("remove %d snapshots:\n", len(remove))
PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact)
Printf("\n")
}
addJSONSnapshots(&fg.Remove, remove)
fg.Reasons = reasons
jsonGroups = append(jsonGroups, &fg)
removeSnapshots += len(remove)
if !opts.DryRun {
for _, sn := range remove {
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
err = repo.Backend().Remove(gopts.ctx, h)
for k, snapshotGroup := range snapshotGroups {
if gopts.Verbose >= 1 && !gopts.JSON {
err = PrintSnapshotGroupHeader(gopts.stdout, k)
if err != nil {
return err
}
}
}
}
if gopts.JSON {
err = printJSONForget(gopts.stdout, jsonGroups)
if err != nil {
return err
var key restic.SnapshotGroupKey
if json.Unmarshal([]byte(k), &key) != nil {
return err
}
var fg ForgetGroup
fg.Tags = key.Tags
fg.Host = key.Hostname
fg.Paths = key.Paths
keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
if len(keep) != 0 && !gopts.Quiet && !gopts.JSON {
Printf("keep %d snapshots:\n", len(keep))
PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact)
Printf("\n")
}
addJSONSnapshots(&fg.Keep, keep)
if len(remove) != 0 && !gopts.Quiet && !gopts.JSON {
Printf("remove %d snapshots:\n", len(remove))
PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact)
Printf("\n")
}
addJSONSnapshots(&fg.Remove, remove)
fg.Reasons = reasons
jsonGroups = append(jsonGroups, &fg)
removeSnapshots += len(remove)
if !opts.DryRun {
for _, sn := range remove {
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
err = repo.Backend().Remove(gopts.ctx, h)
if err != nil {
return err
}
}
}
}
if gopts.JSON {
err = printJSONForget(gopts.stdout, jsonGroups)
if err != nil {
return err
}
}
}
}
if removeSnapshots > 0 && opts.Prune {
Verbosef("%d snapshots have been removed, running prune\n", removeSnapshots)
if !gopts.JSON {
Verbosef("%d snapshots have been removed, running prune\n", removeSnapshots)
}
if !opts.DryRun {
return pruneRepository(gopts, repo)
}

129
cmd/restic/cmd_snapshots.go
View File

@ -32,6 +32,7 @@ type SnapshotOptions struct {
Paths []string
Compact bool
Last bool
GroupBy string
}
var snapshotOptions SnapshotOptions
@ -45,6 +46,7 @@ func init() {
f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact format")
f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path")
f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "string for grouping snapshots by host,paths,tags")
}
func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
@ -64,25 +66,41 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
var list restic.Snapshots
var snapshots restic.Snapshots
for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
list = append(list, sn)
snapshots = append(snapshots, sn)
}
snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy)
if err != nil {
return err
}
if opts.Last {
list = FilterLastSnapshots(list)
for k, list := range snapshotGroups {
if opts.Last {
list = FilterLastSnapshots(list)
}
sort.Sort(sort.Reverse(list))
snapshotGroups[k] = list
}
sort.Sort(sort.Reverse(list))
if gopts.JSON {
err := printSnapshotsJSON(gopts.stdout, list)
err := printSnapshotGroupJSON(gopts.stdout, snapshotGroups, grouped)
if err != nil {
Warnf("error printing snapshot: %v\n", err)
Warnf("error printing snapshots: %v\n", err)
}
return nil
}
PrintSnapshots(gopts.stdout, list, nil, opts.Compact)
for k, list := range snapshotGroups {
if grouped {
err := PrintSnapshotGroupHeader(gopts.stdout, k)
if err != nil {
Warnf("error printing snapshots: %v\n", err)
return nil
}
}
PrintSnapshots(gopts.stdout, list, nil, opts.Compact)
}
return nil
}
@ -223,6 +241,42 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
tab.Write(stdout)
}
// PrintSnapshotGroupHeader prints which group of the group-by option the
// following snapshots belong to.
// Prints nothing, if we did not group at all.
func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string) error {
var key restic.SnapshotGroupKey
var err error
err = json.Unmarshal([]byte(groupKeyJSON), &key)
if err != nil {
return err
}
if key.Hostname == "" && key.Tags == nil && key.Paths == nil {
return nil
}
// Info
fmt.Fprintf(stdout, "snapshots")
var infoStrings []string
if key.Hostname != "" {
infoStrings = append(infoStrings, "host ["+key.Hostname+"]")
}
if key.Tags != nil {
infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]")
}
if key.Paths != nil {
infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]")
}
if infoStrings != nil {
fmt.Fprintf(stdout, " for (%s)", strings.Join(infoStrings, ", "))
}
fmt.Fprintf(stdout, ":\n")
return nil
}
// Snapshot helps to print Snaphots as JSON with their ID included.
type Snapshot struct {
*restic.Snapshot
@ -231,19 +285,58 @@ type Snapshot struct {
ShortID string `json:"short_id"`
}
// printSnapshotsJSON writes the JSON representation of list to stdout.
func printSnapshotsJSON(stdout io.Writer, list restic.Snapshots) error {
// SnapshotGroup helps to print SnaphotGroups as JSON with their GroupReasons included.
type SnapshotGroup struct {
GroupKey restic.SnapshotGroupKey `json:"group_key"`
Snapshots []Snapshot `json:"snapshots"`
}
// printSnapshotsJSON writes the JSON representation of list to stdout.
func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]restic.Snapshots, grouped bool) error {
if grouped {
var snapshotGroups []SnapshotGroup
for k, list := range snGroups {
var key restic.SnapshotGroupKey
var err error
var snapshots []Snapshot
err = json.Unmarshal([]byte(k), &key)
if err != nil {
return err
}
for _, sn := range list {
k := Snapshot{
Snapshot: sn,
ID: sn.ID(),
ShortID: sn.ID().Str(),
}
snapshots = append(snapshots, k)
}
group := SnapshotGroup{
GroupKey: key,
Snapshots: snapshots,
}
snapshotGroups = append(snapshotGroups, group)
}
return json.NewEncoder(stdout).Encode(snapshotGroups)
}
// Old behavior
var snapshots []Snapshot
for _, sn := range list {
k := Snapshot{
Snapshot: sn,
ID: sn.ID(),
ShortID: sn.ID().Str(),
for _, list := range snGroups {
for _, sn := range list {
k := Snapshot{
Snapshot: sn,
ID: sn.ID(),
ShortID: sn.ID().Str(),
}
snapshots = append(snapshots, k)
}
snapshots = append(snapshots, k)
}
return json.NewEncoder(stdout).Encode(snapshots)

25
doc/045_working_with_repos.rst
View File

@ -56,6 +56,31 @@ Or filter by host:
Combining filters is also possible.
Furthermore you can group the output by the same filters (host, paths, tags):
.. code-block:: console
$ restic -r /srv/restic-repo snapshots --group-by host
enter password for repository:
snapshots for (host [kasimir])
ID Date Host Tags Directory
----------------------------------------------------------------------
40dc1520 2015-05-08 21:38:30 kasimir /home/user/work
79766175 2015-05-08 21:40:19 kasimir /home/user/work
2 snapshots
snapshots for (host [luigi])
ID Date Host Tags Directory
----------------------------------------------------------------------
bdbd3439 2015-05-08 21:45:17 luigi /home/art
9f0bc19e 2015-05-08 21:46:11 luigi /srv
2 snapshots
snapshots for (host [kazik])
ID Date Host Tags Directory
----------------------------------------------------------------------
590c8fc8 2015-05-08 21:47:38 kazik /srv
1 snapshots
Checking a repo's integrity and consistency
===========================================

76
internal/restic/snapshot_group.go Normal file
View File

@ -0,0 +1,76 @@
package restic
import (
"encoding/json"
"sort"
"strings"
"github.com/restic/restic/internal/errors"
)
// SnapshotGroupKey is the structure for identifying groups in a grouped
// snapshot list. This is used by GroupSnapshots()
type SnapshotGroupKey struct {
Hostname string `json:"hostname"`
Paths []string `json:"paths"`
Tags []string `json:"tags"`
}
// GroupSnapshots takes a list of snapshots and a grouping criteria and creates
// a group list of snapshots.
func GroupSnapshots(snapshots Snapshots, options string) (map[string]Snapshots, bool, error) {
// group by hostname and dirs
snapshotGroups := make(map[string]Snapshots)
var GroupByTag bool
var GroupByHost bool
var GroupByPath bool
var GroupOptionList []string
GroupOptionList = strings.Split(options, ",")
for _, option := range GroupOptionList {
switch option {
case "host":
GroupByHost = true
case "paths":
GroupByPath = true
case "tags":
GroupByTag = true
case "":
default:
return nil, false, errors.Fatal("unknown grouping option: '" + option + "'")
}
}
for _, sn := range snapshots {
// Determining grouping-keys
var tags []string
var hostname string
var paths []string
if GroupByTag {
tags = sn.Tags
sort.StringSlice(tags).Sort()
}
if GroupByHost {
hostname = sn.Hostname
}
if GroupByPath {
paths = sn.Paths
}
sort.StringSlice(sn.Paths).Sort()
var k []byte
var err error
k, err = json.Marshal(SnapshotGroupKey{Tags: tags, Hostname: hostname, Paths: paths})
if err != nil {
return nil, false, err
}
snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
}
return snapshotGroups, GroupByTag || GroupByHost || GroupByPath, nil
}