From 3541a3a1bf94b8bb2ec315b8c3a16752f6410b8e Mon Sep 17 00:00:00 2001 From: Alex Duchesne Date: Tue, 2 Apr 2024 17:55:28 -0400 Subject: [PATCH 1/7] Implemented a web server to browse a repository (snapshots, files) The code is pretty bad and unorganized, I am very new at Go. But it works! --- changelog/unreleased/pull-4276 | 13 ++ cmd/restic/cmd_serve.go | 268 +++++++++++++++++++++++++++++++++ 2 files changed, 281 insertions(+) create mode 100644 changelog/unreleased/pull-4276 create mode 100644 cmd/restic/cmd_serve.go diff --git a/changelog/unreleased/pull-4276 b/changelog/unreleased/pull-4276 new file mode 100644 index 000000000..71073052f --- /dev/null +++ b/changelog/unreleased/pull-4276 @@ -0,0 +1,13 @@ +Enhancement: Implement web server to browse snapshots + +Currently the canonical way of browsing a repository's snapshots to view +or restore files is `mount`. Unfortunately `mount` depends on fuse which +is not available on all operating systems. + +The new `restic serve` command presents a web interface to browse a +repository's snapshots. It allows to view and download files individually +or as a group (as a tar archive) from snapshots. + +https://github.com/restic/restic/pull/4276 +https://github.com/restic/restic/issues/60 + \ No newline at end of file diff --git a/cmd/restic/cmd_serve.go b/cmd/restic/cmd_serve.go new file mode 100644 index 000000000..80309b539 --- /dev/null +++ b/cmd/restic/cmd_serve.go @@ -0,0 +1,268 @@ +package main + +import ( + "context" + "html/template" + "net/http" + "sort" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/restic/restic/internal/dump" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/walker" +) + +var cmdServe = &cobra.Command{ + Use: "serve", + Short: "runs a web server to browse a repository", + Long: ` +The serve command runs a web server to browse a repository. +`, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runWebServer(cmd.Context(), serveOptions, globalOptions, args) + }, +} + +type ServeOptions struct { + Listen string +} + +var serveOptions ServeOptions + +func init() { + cmdRoot.AddCommand(cmdServe) + cmdFlags := cmdServe.Flags() + cmdFlags.StringVarP(&serveOptions.Listen, "listen", "l", "localhost:3080", "set the listen host name and `address`") +} + +type fileNode struct { + Path string + Node *restic.Node +} + +func listNodes(ctx context.Context, repo restic.Repository, tree restic.ID, path string) ([]fileNode, error) { + var files []fileNode + err := walker.Walk(ctx, repo, tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { + if err != nil || node == nil { + return false, err + } + if fs.HasPathPrefix(path, nodepath) { + files = append(files, fileNode{nodepath, node}) + } + if node.Type == "dir" && !fs.HasPathPrefix(nodepath, path) { + return false, walker.ErrSkipNode + } + return false, nil + }) + return files, err +} + +func runWebServer(ctx context.Context, opts ServeOptions, gopts GlobalOptions, args []string) error { + if len(args) > 0 { + return errors.Fatal("this command does not accept additional arguments") + } + + repo, err := OpenRepository(ctx, gopts) + if err != nil { + return err + } + + if !gopts.NoLock { + var lock *restic.Lock + lock, ctx, err = lockRepo(ctx, repo) + defer unlockRepo(lock) + if err != nil { + return err + } + } + + err = repo.LoadIndex(ctx) + if err != nil { + return err + } + + funcMap := template.FuncMap{ + "FormatTime": func(time time.Time) string { return time.Format(TimeFormat) }, + } + indexPage := template.Must(template.New("index").Funcs(funcMap).Parse(indexPageTpl)) + treePage := template.Must(template.New("tree").Funcs(funcMap).Parse(treePageTpl)) + + http.HandleFunc("/tree/", func(w http.ResponseWriter, r *http.Request) { + snapshotID, curPath, _ := strings.Cut(r.URL.Path[6:], "/") + curPath = "/" + strings.Trim(curPath, "/") + _ = r.ParseForm() + + sn, err := restic.FindSnapshot(ctx, repo.Backend(), repo, snapshotID) + if err != nil { + http.Error(w, "Snapshot not found: "+err.Error(), http.StatusNotFound) + return + } + + files, err := listNodes(ctx, repo, *sn.Tree, curPath) + if err != nil || len(files) == 0 { + http.Error(w, "Path not found in snapshot", http.StatusNotFound) + return + } + + if r.Form.Get("action") == "dump" { + var tree restic.Tree + for _, file := range files { + for _, name := range r.Form["name"] { + if name == file.Node.Name { + tree.Nodes = append(tree.Nodes, file.Node) + } + } + } + if len(tree.Nodes) > 0 { + filename := strings.ReplaceAll(strings.Trim(snapshotID+curPath, "/"), "/", "_") + ".tar.gz" + w.Header().Set("Content-Disposition", "attachment; filename=\""+filename+"\"") + // For now it's hardcoded to tar because it's the only format that supports all node types correctly + if err := dump.New("tar", repo, w).DumpTree(ctx, &tree, "/"); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + } + + if len(files) == 1 && files[0].Node.Type == "file" { + if err := dump.New("zip", repo, w).WriteNode(ctx, files[0].Node); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + var rows []treePageRow + for _, item := range files { + if item.Path != curPath { + rows = append(rows, treePageRow{"/tree/" + snapshotID + item.Path, item.Node.Name, item.Node.Type, item.Node.Size, item.Node.ModTime}) + } + } + sort.SliceStable(rows, func(i, j int) bool { + return strings.ToLower(rows[i].Name) < strings.ToLower(rows[j].Name) + }) + sort.SliceStable(rows, func(i, j int) bool { + return rows[i].Type == "dir" && rows[j].Type != "dir" + }) + parent := "/tree/" + snapshotID + curPath + "/.." + if curPath == "/" { + parent = "/" + } + if err := treePage.Execute(w, treePageData{snapshotID + ": " + curPath, parent, rows}); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + }) + + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + var rows []indexPageRow + for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &restic.SnapshotFilter{}, nil) { + rows = append(rows, indexPageRow{"/tree/" + sn.ID().Str() + "/", sn.ID().Str(), sn.Time, sn.Hostname, sn.Tags, sn.Paths}) + } + sort.Slice(rows, func(i, j int) bool { + return rows[i].Time.After(rows[j].Time) + }) + if err := indexPage.Execute(w, indexPageData{"Snapshots", rows}); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + }) + + http.HandleFunc("/style.css", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "max-age=300") + _, _ = w.Write([]byte(stylesheetTxt)) + }) + + Printf("Now serving the repository at http://%s\n", opts.Listen) + Printf("When finished, quit with Ctrl-c here.\n") + + return http.ListenAndServe(opts.Listen, nil) +} + +type indexPageRow struct { + Link string + ID string + Time time.Time + Host string + Tags []string + Paths []string +} + +type indexPageData struct { + Title string + Rows []indexPageRow +} + +type treePageRow struct { + Link string + Name string + Type string + Size uint64 + Time time.Time +} + +type treePageData struct { + Title string + Parent string + Rows []treePageRow +} + +const indexPageTpl = ` + + +{{.Title}} :: restic + + +

{{.Title}}

+ + + +{{range .Rows}} + +{{end}} + +
IDTimeHostTagsPaths
{{.ID}}{{.Time | FormatTime}}{{.Host}}{{.Tags}}{{.Paths}}
+ +` + +const treePageTpl = ` + + +{{.Title}} :: restic + + +

{{.Title}}

+
+ + + +{{if .Parent}}{{end}} +{{range .Rows}} + +{{end}} + + + + +
NameTypeSizeDate modified
..parent
{{.Name}}{{.Type}}{{.Size}}{{.Time | FormatTime}}
+
+ +` + +const stylesheetTxt = ` +h1,h2,h3 {text-align:center; margin: 0.5em;} +table {margin: 0 auto;border-collapse: collapse; } +thead th {text-align: left; font-weight: bold;} +tbody.content tr:hover {background: #eee;} +tbody.content a.file:before {content: '\1F4C4'} +tbody.content a.dir:before {content: '\1F4C1'} +tbody.actions td {padding:.5em;} +table, td, tr, th { border: 1px solid black; padding: .1em .5em;} +` From e977c9f7982359ea2b5e2d1456685ef061acfe6b Mon Sep 17 00:00:00 2001 From: Alex Duchesne Date: Tue, 2 Apr 2024 18:24:08 -0400 Subject: [PATCH 2/7] fixed compilation issues --- cmd/restic/cmd_serve.go | 47 ++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/cmd/restic/cmd_serve.go b/cmd/restic/cmd_serve.go index 80309b539..13421d3f4 100644 --- a/cmd/restic/cmd_serve.go +++ b/cmd/restic/cmd_serve.go @@ -48,17 +48,19 @@ type fileNode struct { func listNodes(ctx context.Context, repo restic.Repository, tree restic.ID, path string) ([]fileNode, error) { var files []fileNode - err := walker.Walk(ctx, repo, tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { - if err != nil || node == nil { - return false, err - } - if fs.HasPathPrefix(path, nodepath) { - files = append(files, fileNode{nodepath, node}) - } - if node.Type == "dir" && !fs.HasPathPrefix(nodepath, path) { - return false, walker.ErrSkipNode - } - return false, nil + err := walker.Walk(ctx, repo, tree, walker.WalkVisitor{ + ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error { + if err != nil || node == nil { + return err + } + if fs.HasPathPrefix(path, nodepath) { + files = append(files, fileNode{nodepath, node}) + } + if node.Type == "dir" && !fs.HasPathPrefix(nodepath, path) { + return walker.ErrSkipNode + } + return nil + }, }) return files, err } @@ -68,21 +70,18 @@ func runWebServer(ctx context.Context, opts ServeOptions, gopts GlobalOptions, a return errors.Fatal("this command does not accept additional arguments") } - repo, err := OpenRepository(ctx, gopts) + ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, gopts.NoLock) if err != nil { return err } + defer unlock() - if !gopts.NoLock { - var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) - defer unlockRepo(lock) - if err != nil { - return err - } + snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile) + if err != nil { + return err } - - err = repo.LoadIndex(ctx) + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + err = repo.LoadIndex(ctx, bar) if err != nil { return err } @@ -98,7 +97,7 @@ func runWebServer(ctx context.Context, opts ServeOptions, gopts GlobalOptions, a curPath = "/" + strings.Trim(curPath, "/") _ = r.ParseForm() - sn, err := restic.FindSnapshot(ctx, repo.Backend(), repo, snapshotID) + sn, _, err := restic.FindSnapshot(ctx, snapshotLister, repo, snapshotID) if err != nil { http.Error(w, "Snapshot not found: "+err.Error(), http.StatusNotFound) return @@ -164,7 +163,7 @@ func runWebServer(ctx context.Context, opts ServeOptions, gopts GlobalOptions, a return } var rows []indexPageRow - for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &restic.SnapshotFilter{}, nil) { + for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &restic.SnapshotFilter{}, nil) { rows = append(rows, indexPageRow{"/tree/" + sn.ID().Str() + "/", sn.ID().Str(), sn.Time, sn.Hostname, sn.Tags, sn.Paths}) } sort.Slice(rows, func(i, j int) bool { @@ -255,7 +254,7 @@ const treePageTpl = ` ` - + const stylesheetTxt = ` h1,h2,h3 {text-align:center; margin: 0.5em;} table {margin: 0 auto;border-collapse: collapse; } From 5fc118f36d83fb7f5fa4e2fe3eb42c6d8d3868fb Mon Sep 17 00:00:00 2001 From: Alex Duchesne Date: Tue, 2 Apr 2024 18:53:38 -0400 Subject: [PATCH 3/7] improved readability --- cmd/restic/cmd_serve.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/cmd/restic/cmd_serve.go b/cmd/restic/cmd_serve.go index 13421d3f4..372115d20 100644 --- a/cmd/restic/cmd_serve.go +++ b/cmd/restic/cmd_serve.go @@ -80,6 +80,7 @@ func runWebServer(ctx context.Context, opts ServeOptions, gopts GlobalOptions, a if err != nil { return err } + bar := newIndexProgress(gopts.Quiet, gopts.JSON) err = repo.LoadIndex(ctx, bar) if err != nil { @@ -139,7 +140,13 @@ func runWebServer(ctx context.Context, opts ServeOptions, gopts GlobalOptions, a var rows []treePageRow for _, item := range files { if item.Path != curPath { - rows = append(rows, treePageRow{"/tree/" + snapshotID + item.Path, item.Node.Name, item.Node.Type, item.Node.Size, item.Node.ModTime}) + rows = append(rows, treePageRow{ + Link: "/tree/" + snapshotID + item.Path, + Name: item.Node.Name, + Type: item.Node.Type, + Size: item.Node.Size, + Time: item.Node.ModTime, + }) } } sort.SliceStable(rows, func(i, j int) bool { @@ -164,7 +171,14 @@ func runWebServer(ctx context.Context, opts ServeOptions, gopts GlobalOptions, a } var rows []indexPageRow for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &restic.SnapshotFilter{}, nil) { - rows = append(rows, indexPageRow{"/tree/" + sn.ID().Str() + "/", sn.ID().Str(), sn.Time, sn.Hostname, sn.Tags, sn.Paths}) + rows = append(rows, indexPageRow{ + Link: "/tree/" + sn.ID().Str() + "/", + ID: sn.ID().Str(), + Time: sn.Time, + Host: sn.Hostname, + Tags: sn.Tags, + Paths: sn.Paths, + }) } sort.Slice(rows, func(i, j int) bool { return rows[i].Time.After(rows[j].Time) From 993eb7042294f5d44f73313455c0cb1b6566fc9f Mon Sep 17 00:00:00 2001 From: Alex Duchesne Date: Sat, 6 Apr 2024 13:07:02 -0400 Subject: [PATCH 4/7] satisfy the linter --- cmd/restic/cmd_serve.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/restic/cmd_serve.go b/cmd/restic/cmd_serve.go index 372115d20..4865a0b58 100644 --- a/cmd/restic/cmd_serve.go +++ b/cmd/restic/cmd_serve.go @@ -49,7 +49,7 @@ type fileNode struct { func listNodes(ctx context.Context, repo restic.Repository, tree restic.ID, path string) ([]fileNode, error) { var files []fileNode err := walker.Walk(ctx, repo, tree, walker.WalkVisitor{ - ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error { + ProcessNode: func(_ restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil || node == nil { return err } @@ -188,7 +188,7 @@ func runWebServer(ctx context.Context, opts ServeOptions, gopts GlobalOptions, a } }) - http.HandleFunc("/style.css", func(w http.ResponseWriter, r *http.Request) { + http.HandleFunc("/style.css", func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Cache-Control", "max-age=300") _, _ = w.Write([]byte(stylesheetTxt)) }) From 8a5ac6dc1302673c4a368834e5ffb05f481707d0 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Thu, 25 Apr 2024 21:40:46 +0200 Subject: [PATCH 5/7] Rework server --- cmd/restic/cmd_serve.go | 221 +--------------------------- internal/server/assets/fs.go | 6 + internal/server/assets/index.html | 34 +++++ internal/server/assets/style.css | 40 ++++++ internal/server/assets/tree.html | 51 +++++++ internal/server/server.go | 231 ++++++++++++++++++++++++++++++ 6 files changed, 365 insertions(+), 218 deletions(-) create mode 100644 internal/server/assets/fs.go create mode 100644 internal/server/assets/index.html create mode 100644 internal/server/assets/style.css create mode 100644 internal/server/assets/tree.html create mode 100644 internal/server/server.go diff --git a/cmd/restic/cmd_serve.go b/cmd/restic/cmd_serve.go index 4865a0b58..d00ce201e 100644 --- a/cmd/restic/cmd_serve.go +++ b/cmd/restic/cmd_serve.go @@ -2,19 +2,13 @@ package main import ( "context" - "html/template" "net/http" - "sort" - "strings" - "time" "github.com/spf13/cobra" - "github.com/restic/restic/internal/dump" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/walker" + "github.com/restic/restic/internal/server" ) var cmdServe = &cobra.Command{ @@ -41,30 +35,6 @@ func init() { cmdFlags.StringVarP(&serveOptions.Listen, "listen", "l", "localhost:3080", "set the listen host name and `address`") } -type fileNode struct { - Path string - Node *restic.Node -} - -func listNodes(ctx context.Context, repo restic.Repository, tree restic.ID, path string) ([]fileNode, error) { - var files []fileNode - err := walker.Walk(ctx, repo, tree, walker.WalkVisitor{ - ProcessNode: func(_ restic.ID, nodepath string, node *restic.Node, err error) error { - if err != nil || node == nil { - return err - } - if fs.HasPathPrefix(path, nodepath) { - files = append(files, fileNode{nodepath, node}) - } - if node.Type == "dir" && !fs.HasPathPrefix(nodepath, path) { - return walker.ErrSkipNode - } - return nil - }, - }) - return files, err -} - func runWebServer(ctx context.Context, opts ServeOptions, gopts GlobalOptions, args []string) error { if len(args) > 0 { return errors.Fatal("this command does not accept additional arguments") @@ -87,195 +57,10 @@ func runWebServer(ctx context.Context, opts ServeOptions, gopts GlobalOptions, a return err } - funcMap := template.FuncMap{ - "FormatTime": func(time time.Time) string { return time.Format(TimeFormat) }, - } - indexPage := template.Must(template.New("index").Funcs(funcMap).Parse(indexPageTpl)) - treePage := template.Must(template.New("tree").Funcs(funcMap).Parse(treePageTpl)) - - http.HandleFunc("/tree/", func(w http.ResponseWriter, r *http.Request) { - snapshotID, curPath, _ := strings.Cut(r.URL.Path[6:], "/") - curPath = "/" + strings.Trim(curPath, "/") - _ = r.ParseForm() - - sn, _, err := restic.FindSnapshot(ctx, snapshotLister, repo, snapshotID) - if err != nil { - http.Error(w, "Snapshot not found: "+err.Error(), http.StatusNotFound) - return - } - - files, err := listNodes(ctx, repo, *sn.Tree, curPath) - if err != nil || len(files) == 0 { - http.Error(w, "Path not found in snapshot", http.StatusNotFound) - return - } - - if r.Form.Get("action") == "dump" { - var tree restic.Tree - for _, file := range files { - for _, name := range r.Form["name"] { - if name == file.Node.Name { - tree.Nodes = append(tree.Nodes, file.Node) - } - } - } - if len(tree.Nodes) > 0 { - filename := strings.ReplaceAll(strings.Trim(snapshotID+curPath, "/"), "/", "_") + ".tar.gz" - w.Header().Set("Content-Disposition", "attachment; filename=\""+filename+"\"") - // For now it's hardcoded to tar because it's the only format that supports all node types correctly - if err := dump.New("tar", repo, w).DumpTree(ctx, &tree, "/"); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - } - - if len(files) == 1 && files[0].Node.Type == "file" { - if err := dump.New("zip", repo, w).WriteNode(ctx, files[0].Node); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - var rows []treePageRow - for _, item := range files { - if item.Path != curPath { - rows = append(rows, treePageRow{ - Link: "/tree/" + snapshotID + item.Path, - Name: item.Node.Name, - Type: item.Node.Type, - Size: item.Node.Size, - Time: item.Node.ModTime, - }) - } - } - sort.SliceStable(rows, func(i, j int) bool { - return strings.ToLower(rows[i].Name) < strings.ToLower(rows[j].Name) - }) - sort.SliceStable(rows, func(i, j int) bool { - return rows[i].Type == "dir" && rows[j].Type != "dir" - }) - parent := "/tree/" + snapshotID + curPath + "/.." - if curPath == "/" { - parent = "/" - } - if err := treePage.Execute(w, treePageData{snapshotID + ": " + curPath, parent, rows}); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - }) - - http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/" { - http.NotFound(w, r) - return - } - var rows []indexPageRow - for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &restic.SnapshotFilter{}, nil) { - rows = append(rows, indexPageRow{ - Link: "/tree/" + sn.ID().Str() + "/", - ID: sn.ID().Str(), - Time: sn.Time, - Host: sn.Hostname, - Tags: sn.Tags, - Paths: sn.Paths, - }) - } - sort.Slice(rows, func(i, j int) bool { - return rows[i].Time.After(rows[j].Time) - }) - if err := indexPage.Execute(w, indexPageData{"Snapshots", rows}); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - }) - - http.HandleFunc("/style.css", func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Cache-Control", "max-age=300") - _, _ = w.Write([]byte(stylesheetTxt)) - }) + srv := server.New(repo, snapshotLister, TimeFormat) Printf("Now serving the repository at http://%s\n", opts.Listen) Printf("When finished, quit with Ctrl-c here.\n") - return http.ListenAndServe(opts.Listen, nil) + return http.ListenAndServe(opts.Listen, srv) } - -type indexPageRow struct { - Link string - ID string - Time time.Time - Host string - Tags []string - Paths []string -} - -type indexPageData struct { - Title string - Rows []indexPageRow -} - -type treePageRow struct { - Link string - Name string - Type string - Size uint64 - Time time.Time -} - -type treePageData struct { - Title string - Parent string - Rows []treePageRow -} - -const indexPageTpl = ` - - -{{.Title}} :: restic - - -

{{.Title}}

- - - -{{range .Rows}} - -{{end}} - -
IDTimeHostTagsPaths
{{.ID}}{{.Time | FormatTime}}{{.Host}}{{.Tags}}{{.Paths}}
- -` - -const treePageTpl = ` - - -{{.Title}} :: restic - - -

{{.Title}}

-
- - - -{{if .Parent}}{{end}} -{{range .Rows}} - -{{end}} - - - - -
NameTypeSizeDate modified
..parent
{{.Name}}{{.Type}}{{.Size}}{{.Time | FormatTime}}
-
- -` - -const stylesheetTxt = ` -h1,h2,h3 {text-align:center; margin: 0.5em;} -table {margin: 0 auto;border-collapse: collapse; } -thead th {text-align: left; font-weight: bold;} -tbody.content tr:hover {background: #eee;} -tbody.content a.file:before {content: '\1F4C4'} -tbody.content a.dir:before {content: '\1F4C1'} -tbody.actions td {padding:.5em;} -table, td, tr, th { border: 1px solid black; padding: .1em .5em;} -` diff --git a/internal/server/assets/fs.go b/internal/server/assets/fs.go new file mode 100644 index 000000000..ed1942d08 --- /dev/null +++ b/internal/server/assets/fs.go @@ -0,0 +1,6 @@ +package assets + +import "embed" + +//go:embed * +var FS embed.FS diff --git a/internal/server/assets/index.html b/internal/server/assets/index.html new file mode 100644 index 000000000..c40677252 --- /dev/null +++ b/internal/server/assets/index.html @@ -0,0 +1,34 @@ + + + + + {{.Title}} :: restic + + + +

{{.Title}}

+ + + + + + + + + + + + {{range .Rows}} + + + + + + + + {{end}} + +
IDTimeHostTagsPaths
{{.ID}}{{.Time | FormatTime}}{{.Host}}{{.Tags}}{{.Paths}}
+ + + \ No newline at end of file diff --git a/internal/server/assets/style.css b/internal/server/assets/style.css new file mode 100644 index 000000000..79a87bcb4 --- /dev/null +++ b/internal/server/assets/style.css @@ -0,0 +1,40 @@ +h1, +h2, +h3 { + text-align: center; + margin: 0.5em; +} + +table { + margin: 0 auto; + border-collapse: collapse; +} + +thead th { + text-align: left; + font-weight: bold; +} + +tbody.content tr:hover { + background: #eee; +} + +tbody.content a.file:before { + content: '\1F4C4' +} + +tbody.content a.dir:before { + content: '\1F4C1' +} + +tbody.actions td { + padding: .5em; +} + +table, +td, +tr, +th { + border: 1px solid black; + padding: .1em .5em; +} \ No newline at end of file diff --git a/internal/server/assets/tree.html b/internal/server/assets/tree.html new file mode 100644 index 000000000..174613df4 --- /dev/null +++ b/internal/server/assets/tree.html @@ -0,0 +1,51 @@ + + + + + {{.Title}} :: restic + + + +

{{.Title}}

+
+ + + + + + + + + + + + {{if .Parent}} + + + + + {{end}} + {{range .Rows}} + + + + + + + + + {{end}} + + + + + + +
+ NameTypeSizeDate modified
..parent +
{{.Name}}{{.Type}}{{.Size}}{{.Time | FormatTime}}
+
+ + + \ No newline at end of file diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 000000000..77d02e7cd --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,231 @@ +// Package server contains an HTTP server which can serve content from a repo. +package server + +import ( + "context" + "fmt" + "io/fs" + "net/http" + "sort" + "strings" + "text/template" + "time" + + "github.com/restic/restic/internal/dump" + rfs "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/server/assets" + "github.com/restic/restic/internal/walker" +) + +// New returns a new HTTP server. +func New(repo restic.Repository, snapshotLister restic.Lister, timeFormat string) http.Handler { + funcs := template.FuncMap{ + "FormatTime": func(time time.Time) string { return time.Format(timeFormat) }, + } + + templates := template.Must(template.New("").Funcs(funcs).ParseFS(assets.FS, "*.html")) + + mux := http.NewServeMux() + + indexPage := templates.Lookup("index.html") + if indexPage == nil { + panic("index.html not found") + } + + treePage := templates.Lookup("tree.html") + if treePage == nil { + panic("tree.html not found") + } + + mux.HandleFunc("/tree/", func(rw http.ResponseWriter, req *http.Request) { + snapshotID, curPath, _ := strings.Cut(req.URL.Path[6:], "/") + curPath = "/" + strings.Trim(curPath, "/") + _ = req.ParseForm() + + sn, _, err := restic.FindSnapshot(req.Context(), snapshotLister, repo, snapshotID) + if err != nil { + http.Error(rw, "Snapshot not found: "+err.Error(), http.StatusNotFound) + return + } + + files, err := listNodes(req.Context(), repo, *sn.Tree, curPath) + if err != nil || len(files) == 0 { + http.Error(rw, "Path not found in snapshot", http.StatusNotFound) + return + } + + if req.Form.Get("action") == "dump" { + var tree restic.Tree + for _, file := range files { + for _, name := range req.Form["name"] { + if name == file.Node.Name { + tree.Nodes = append(tree.Nodes, file.Node) + } + } + } + if len(tree.Nodes) > 0 { + filename := strings.ReplaceAll(strings.Trim(snapshotID+curPath, "/"), "/", "_") + ".tar.gz" + rw.Header().Set("Content-Disposition", "attachment; filename=\""+filename+"\"") + // For now it's hardcoded to tar because it's the only format that supports all node types correctly + if err := dump.New("tar", repo, rw).DumpTree(req.Context(), &tree, "/"); err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + } + return + } + } + + if len(files) == 1 && files[0].Node.Type == "file" { + if err := dump.New("zip", repo, rw).WriteNode(req.Context(), files[0].Node); err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + } + return + } + + var rows []treePageRow + for _, item := range files { + if item.Path != curPath { + rows = append(rows, treePageRow{ + Link: "/tree/" + snapshotID + item.Path, + Name: item.Node.Name, + Type: item.Node.Type, + Size: item.Node.Size, + Time: item.Node.ModTime, + }) + } + } + sort.SliceStable(rows, func(i, j int) bool { + return strings.ToLower(rows[i].Name) < strings.ToLower(rows[j].Name) + }) + sort.SliceStable(rows, func(i, j int) bool { + return rows[i].Type == "dir" && rows[j].Type != "dir" + }) + parent := "/tree/" + snapshotID + curPath + "/.." + if curPath == "/" { + parent = "/" + } + if err := treePage.Execute(rw, treePageData{snapshotID + ": " + curPath, parent, rows}); err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + } + }) + + http.HandleFunc("/", func(rw http.ResponseWriter, req *http.Request) { + if req.URL.Path != "/" { + http.NotFound(rw, req) + return + } + var rows []indexPageRow + for sn := range findFilteredSnapshots(req.Context(), snapshotLister, repo, &restic.SnapshotFilter{}, nil) { + rows = append(rows, indexPageRow{ + Link: "/tree/" + sn.ID().Str() + "/", + ID: sn.ID().Str(), + Time: sn.Time, + Host: sn.Hostname, + Tags: sn.Tags, + Paths: sn.Paths, + }) + } + sort.Slice(rows, func(i, j int) bool { + return rows[i].Time.After(rows[j].Time) + }) + if err := indexPage.Execute(rw, indexPageData{"Snapshots", rows}); err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + } + }) + + http.HandleFunc("/style.css", func(rw http.ResponseWriter, _ *http.Request) { + rw.Header().Set("Cache-Control", "max-age=300") + buf, err := fs.ReadFile(assets.FS, "style.css") + if err == nil { + rw.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(rw, "error: %v", err) + return + } + + _, _ = rw.Write(buf) + }) + + return mux +} + +type fileNode struct { + Path string + Node *restic.Node +} + +func listNodes(ctx context.Context, repo restic.Repository, tree restic.ID, path string) ([]fileNode, error) { + var files []fileNode + err := walker.Walk(ctx, repo, tree, walker.WalkVisitor{ + ProcessNode: func(_ restic.ID, nodepath string, node *restic.Node, err error) error { + if err != nil || node == nil { + return err + } + if rfs.HasPathPrefix(path, nodepath) { + files = append(files, fileNode{nodepath, node}) + } + if node.Type == "dir" && !rfs.HasPathPrefix(nodepath, path) { + return walker.ErrSkipNode + } + return nil + }, + }) + return files, err +} + +type indexPageRow struct { + Link string + ID string + Time time.Time + Host string + Tags []string + Paths []string +} + +type indexPageData struct { + Title string + Rows []indexPageRow +} + +type treePageRow struct { + Link string + Name string + Type string + Size uint64 + Time time.Time +} + +type treePageData struct { + Title string + Parent string + Rows []treePageRow +} + +// findFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. +func findFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, f *restic.SnapshotFilter, snapshotIDs []string) <-chan *restic.Snapshot { + out := make(chan *restic.Snapshot) + go func() { + defer close(out) + be, err := restic.MemorizeList(ctx, be, restic.SnapshotFile) + if err != nil { + // Warnf("could not load snapshots: %v\n", err) + return + } + + err = f.FindAll(ctx, be, loader, snapshotIDs, func(id string, sn *restic.Snapshot, err error) error { + if err != nil { + // Warnf("Ignoring %q: %v\n", id, err) + } else { + select { + case <-ctx.Done(): + return ctx.Err() + case out <- sn: + } + } + return nil + }) + if err != nil { + // Warnf("could not load snapshots: %v\n", err) + } + }() + return out +} From d463996ce9dd24710763f32d77b3347b474a2ed5 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Fri, 26 Apr 2024 19:21:32 +0200 Subject: [PATCH 6/7] serve: Properly shut down the server --- cmd/restic/cmd_serve.go | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/cmd/restic/cmd_serve.go b/cmd/restic/cmd_serve.go index d00ce201e..d101ec2c6 100644 --- a/cmd/restic/cmd_serve.go +++ b/cmd/restic/cmd_serve.go @@ -2,7 +2,10 @@ package main import ( "context" + "fmt" + "net" "net/http" + "time" "github.com/spf13/cobra" @@ -35,6 +38,8 @@ func init() { cmdFlags.StringVarP(&serveOptions.Listen, "listen", "l", "localhost:3080", "set the listen host name and `address`") } +const serverShutdownTimeout = 30 * time.Second + func runWebServer(ctx context.Context, opts ServeOptions, gopts GlobalOptions, args []string) error { if len(args) > 0 { return errors.Fatal("this command does not accept additional arguments") @@ -57,10 +62,42 @@ func runWebServer(ctx context.Context, opts ServeOptions, gopts GlobalOptions, a return err } - srv := server.New(repo, snapshotLister, TimeFormat) + srv := http.Server{ + BaseContext: func(l net.Listener) context.Context { + // just return the global context + return ctx + }, + Handler: server.New(repo, snapshotLister, TimeFormat), + } + + listener, err := net.Listen("tcp", opts.Listen) + if err != nil { + return fmt.Errorf("start listener: %v", err) + } + + // wait until context is cancelled, then close listener + go func() { + <-ctx.Done() + Printf("gracefully shutting down server\n") + + ctxTimeout, cancel := context.WithTimeout(context.Background(), serverShutdownTimeout) + defer cancel() + + _ = srv.Shutdown(ctxTimeout) + }() Printf("Now serving the repository at http://%s\n", opts.Listen) Printf("When finished, quit with Ctrl-c here.\n") - return http.ListenAndServe(opts.Listen, srv) + err = srv.Serve(listener) + + if errors.Is(err, http.ErrServerClosed) { + err = nil + } + + if err != nil { + return fmt.Errorf("serve: %v", err) + } + + return nil } From 303dda646faafa6f648d6d56eb55ec7caeca82db Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Fri, 26 Apr 2024 19:36:47 +0200 Subject: [PATCH 7/7] Fix server --- internal/server/server.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/internal/server/server.go b/internal/server/server.go index 77d02e7cd..ab6938144 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -4,7 +4,6 @@ package server import ( "context" "fmt" - "io/fs" "net/http" "sort" "strings" @@ -109,11 +108,12 @@ func New(repo restic.Repository, snapshotLister restic.Lister, timeFormat string } }) - http.HandleFunc("/", func(rw http.ResponseWriter, req *http.Request) { + mux.HandleFunc("/", func(rw http.ResponseWriter, req *http.Request) { if req.URL.Path != "/" { http.NotFound(rw, req) return } + var rows []indexPageRow for sn := range findFilteredSnapshots(req.Context(), snapshotLister, repo, &restic.SnapshotFilter{}, nil) { rows = append(rows, indexPageRow{ @@ -125,23 +125,29 @@ func New(repo restic.Repository, snapshotLister restic.Lister, timeFormat string Paths: sn.Paths, }) } + sort.Slice(rows, func(i, j int) bool { return rows[i].Time.After(rows[j].Time) }) + if err := indexPage.Execute(rw, indexPageData{"Snapshots", rows}); err != nil { http.Error(rw, err.Error(), http.StatusInternalServerError) } }) - http.HandleFunc("/style.css", func(rw http.ResponseWriter, _ *http.Request) { - rw.Header().Set("Cache-Control", "max-age=300") - buf, err := fs.ReadFile(assets.FS, "style.css") - if err == nil { + mux.HandleFunc("/style.css", func(rw http.ResponseWriter, req *http.Request) { + buf, err := assets.FS.ReadFile("style.css") + if err != nil { rw.WriteHeader(http.StatusInternalServerError) - fmt.Fprintf(rw, "error: %v", err) + + fmt.Fprintf(rw, "error reading embedded style.css: %v\n", err) + return } + rw.Header().Set("Cache-Control", "max-age=300") + rw.Header().Set("Content-Type", "text/css") + _, _ = rw.Write(buf) })