From c2348ba768fc10fd6aaba1fd294892a1361e0ade Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 20 Feb 2016 22:05:48 +0100 Subject: [PATCH] Add REST backend This is a port of the original work by @bchapuis in https://github.com/restic/restic/pull/253 --- doc/REST_backend.md | 56 +++++ src/restic/backend/rest/backend_test.go | 87 ++++++++ src/restic/backend/rest/config.go | 29 +++ src/restic/backend/rest/config_test.go | 41 ++++ src/restic/backend/rest/rest.go | 258 ++++++++++++++++++++++++ src/restic/backend/rest/rest_test.go | 54 +++++ src/restic/test/backend.go | 1 + 7 files changed, 526 insertions(+) create mode 100644 doc/REST_backend.md create mode 100644 src/restic/backend/rest/backend_test.go create mode 100644 src/restic/backend/rest/config.go create mode 100644 src/restic/backend/rest/config_test.go create mode 100644 src/restic/backend/rest/rest.go create mode 100644 src/restic/backend/rest/rest_test.go diff --git a/doc/REST_backend.md b/doc/REST_backend.md new file mode 100644 index 000000000..2f793c58a --- /dev/null +++ b/doc/REST_backend.md @@ -0,0 +1,56 @@ +REST Backend +============ + +Restic can interact with HTTP Backend that respects the following REST API. + +## HEAD /config + +Returns "200 OK" if the repository has a configuration, +an HTTP error otherwise. + +## GET /config + +Returns the content of the configuration file if the repository has a configuration, +an HTTP error otherwise. + +Response format: binary/octet-stream + +## POST /config + +Returns "200 OK" if the configuration of the request body has been saved, +an HTTP error otherwise. + +## GET /{type}/ + +Returns a JSON array containing the names of all the blobs stored for a given type. + +Response format: JSON + +## HEAD /{type}/{name} + +Returns "200 OK" if the blob with the given name and type is stored in the repository, +"404 not found" otherwise. If the blob exists, the HTTP header `Content-Length` +is set to the file size. + +## GET /{type}/{name} + +Returns the content of the blob with the given name and type if it is stored in the repository, +"404 not found" otherwise. + +If the request specifies a partial read with a Range header field, +then the status code of the response is 206 instead of 200 +and the response only contains the specified range. + +Response format: binary/octet-stream + +## POST /{type}/{name} + +Saves the content of the request body as a blob with the given name and type, +an HTTP error otherwise. + +Request format: binary/octet-stream + +## DELETE /{type}/{name} + +Returns "200 OK" if the blob with the given name and type has been deleted from the repository, +an HTTP error otherwise. diff --git a/src/restic/backend/rest/backend_test.go b/src/restic/backend/rest/backend_test.go new file mode 100644 index 000000000..4274bfcb1 --- /dev/null +++ b/src/restic/backend/rest/backend_test.go @@ -0,0 +1,87 @@ +// DO NOT EDIT, AUTOMATICALLY GENERATED +package rest_test + +import ( + "testing" + + "restic/backend/test" +) + +var SkipMessage string + +func TestRestBackendCreate(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestCreate(t) +} + +func TestRestBackendOpen(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestOpen(t) +} + +func TestRestBackendCreateWithConfig(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestCreateWithConfig(t) +} + +func TestRestBackendLocation(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestLocation(t) +} + +func TestRestBackendConfig(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestConfig(t) +} + +func TestRestBackendLoad(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestLoad(t) +} + +func TestRestBackendSave(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSave(t) +} + +func TestRestBackendSaveFilenames(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSaveFilenames(t) +} + +func TestRestBackendBackend(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestBackend(t) +} + +func TestRestBackendDelete(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestDelete(t) +} + +func TestRestBackendCleanup(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestCleanup(t) +} diff --git a/src/restic/backend/rest/config.go b/src/restic/backend/rest/config.go new file mode 100644 index 000000000..c4459344c --- /dev/null +++ b/src/restic/backend/rest/config.go @@ -0,0 +1,29 @@ +package rest + +import ( + "errors" + "net/url" + "strings" +) + +// Config contains all configuration necessary to connect to a REST server. +type Config struct { + URL *url.URL +} + +// ParseConfig parses the string s and extracts the REST server URL. +func ParseConfig(s string) (interface{}, error) { + if !strings.HasPrefix(s, "rest:") { + return nil, errors.New("invalid REST backend specification") + } + + s = s[5:] + u, err := url.Parse(s) + + if err != nil { + return nil, err + } + + cfg := Config{URL: u} + return cfg, nil +} diff --git a/src/restic/backend/rest/config_test.go b/src/restic/backend/rest/config_test.go new file mode 100644 index 000000000..937204a57 --- /dev/null +++ b/src/restic/backend/rest/config_test.go @@ -0,0 +1,41 @@ +package rest + +import ( + "net/url" + "reflect" + "testing" +) + +func parseURL(s string) *url.URL { + u, err := url.Parse(s) + if err != nil { + panic(err) + } + + return u +} + +var configTests = []struct { + s string + cfg Config +}{ + {"rest:http://localhost:1234", Config{ + URL: parseURL("http://localhost:1234"), + }}, +} + +func TestParseConfig(t *testing.T) { + for i, test := range configTests { + cfg, err := ParseConfig(test.s) + if err != nil { + t.Errorf("test %d:%s failed: %v", i, test.s, err) + continue + } + + if !reflect.DeepEqual(cfg, test.cfg) { + t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v", + i, test.s, test.cfg, cfg) + continue + } + } +} diff --git a/src/restic/backend/rest/rest.go b/src/restic/backend/rest/rest.go new file mode 100644 index 000000000..7cfa3c84d --- /dev/null +++ b/src/restic/backend/rest/rest.go @@ -0,0 +1,258 @@ +package rest + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path" + + "restic/backend" +) + +const connLimit = 10 + +// restPath returns the path to the given resource. +func restPath(url *url.URL, h backend.Handle) string { + p := url.Path + if p == "" { + p = "/" + } + + var dir string + + switch h.Type { + case backend.Config: + dir = "" + case backend.Data: + dir = backend.Paths.Data + case backend.Snapshot: + dir = backend.Paths.Snapshots + case backend.Index: + dir = backend.Paths.Index + case backend.Lock: + dir = backend.Paths.Locks + case backend.Key: + dir = backend.Paths.Keys + default: + dir = string(h.Type) + } + + return path.Join(p, dir, h.Name) +} + +type restBackend struct { + url *url.URL + connChan chan struct{} + client *http.Client +} + +// Open opens the REST backend with the given config. +func Open(cfg Config) (backend.Backend, error) { + connChan := make(chan struct{}, connLimit) + for i := 0; i < connLimit; i++ { + connChan <- struct{}{} + } + tr := &http.Transport{} + client := http.Client{Transport: tr} + + return &restBackend{url: cfg.URL, connChan: connChan, client: &client}, nil +} + +// Location returns this backend's location (the server's URL). +func (b *restBackend) Location() string { + return b.url.String() +} + +// Load returns the data stored in the backend for h at the given offset +// and saves it in p. Load has the same semantics as io.ReaderAt. +func (b *restBackend) Load(h backend.Handle, p []byte, off int64) (n int, err error) { + if err := h.Valid(); err != nil { + return 0, err + } + + req, err := http.NewRequest("GET", restPath(b.url, h), nil) + if err != nil { + return 0, err + } + req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p)))) + client := *b.client + + <-b.connChan + resp, err := client.Do(req) + b.connChan <- struct{}{} + + if resp != nil { + defer func() { + e := resp.Body.Close() + + if err == nil { + err = e + } + }() + } + + if err != nil { + return 0, err + } + if resp.StatusCode != 206 { + return 0, errors.New("blob not found") + } + + return io.ReadFull(resp.Body, p) +} + +// Save stores data in the backend at the handle. +func (b *restBackend) Save(h backend.Handle, p []byte) (err error) { + if err := h.Valid(); err != nil { + return err + } + + client := *b.client + + <-b.connChan + resp, err := client.Post(restPath(b.url, h), "binary/octet-stream", bytes.NewReader(p)) + b.connChan <- struct{}{} + + if resp != nil { + defer func() { + e := resp.Body.Close() + + if err == nil { + err = e + } + }() + } + + if err != nil { + return err + } + + if resp.StatusCode != 200 { + return errors.New("blob not saved") + } + + return nil +} + +// Stat returns information about a blob. +func (b *restBackend) Stat(h backend.Handle) (backend.BlobInfo, error) { + if err := h.Valid(); err != nil { + return backend.BlobInfo{}, err + } + + client := *b.client + <-b.connChan + resp, err := client.Head(restPath(b.url, h)) + b.connChan <- struct{}{} + if err != nil { + return backend.BlobInfo{}, err + } + + if err = resp.Body.Close(); err != nil { + return backend.BlobInfo{}, err + } + + if resp.StatusCode != 200 { + return backend.BlobInfo{}, errors.New("blob not saved") + } + + if resp.ContentLength < 0 { + return backend.BlobInfo{}, errors.New("negative content length") + } + + bi := backend.BlobInfo{ + Size: resp.ContentLength, + } + + return bi, nil +} + +// Test returns true if a blob of the given type and name exists in the backend. +func (b *restBackend) Test(t backend.Type, name string) (bool, error) { + _, err := b.Stat(backend.Handle{Type: t, Name: name}) + if err != nil { + return false, nil + } + + return true, nil +} + +// Remove removes the blob with the given name and type. +func (b *restBackend) Remove(t backend.Type, name string) error { + h := backend.Handle{Type: t, Name: name} + if err := h.Valid(); err != nil { + return err + } + + req, err := http.NewRequest("DELETE", restPath(b.url, h), nil) + if err != nil { + return err + } + client := *b.client + + <-b.connChan + resp, err := client.Do(req) + b.connChan <- struct{}{} + + if err != nil { + return err + } + + if resp.StatusCode != 200 { + return errors.New("blob not removed") + } + + return resp.Body.Close() +} + +// List returns a channel that yields all names of blobs of type t. A +// goroutine is started for this. If the channel done is closed, sending +// stops. +func (b *restBackend) List(t backend.Type, done <-chan struct{}) <-chan string { + ch := make(chan string) + + client := *b.client + <-b.connChan + resp, err := client.Get(restPath(b.url, backend.Handle{Type: t})) + b.connChan <- struct{}{} + + if resp != nil { + defer resp.Body.Close() + } + + if err != nil { + close(ch) + return ch + } + + dec := json.NewDecoder(resp.Body) + var list []string + if err = dec.Decode(&list); err != nil { + close(ch) + return ch + } + + go func() { + defer close(ch) + for _, m := range list { + select { + case ch <- m: + case <-done: + return + } + } + }() + + return ch +} + +// Close closes all open files. +func (b *restBackend) Close() error { + // this does not need to do anything, all open files are closed within the + // same function. + return nil +} diff --git a/src/restic/backend/rest/rest_test.go b/src/restic/backend/rest/rest_test.go new file mode 100644 index 000000000..bbcfff9b0 --- /dev/null +++ b/src/restic/backend/rest/rest_test.go @@ -0,0 +1,54 @@ +package rest_test + +import ( + "errors" + "fmt" + "net/url" + "os" + + "restic/backend" + "restic/backend/rest" + "restic/backend/test" + . "restic/test" +) + +//go:generate go run ../test/generate_backend_tests.go + +func init() { + if TestRESTServer == "" { + SkipMessage = "REST test server not available" + return + } + + url, err := url.Parse(TestRESTServer) + if err != nil { + fmt.Fprintf(os.Stderr, "invalid url: %v\n", err) + return + } + + cfg := rest.Config{ + URL: url, + } + + test.CreateFn = func() (backend.Backend, error) { + be, err := rest.Open(cfg) + if err != nil { + return nil, err + } + + exists, err := be.Test(backend.Config, "") + if err != nil { + return nil, err + } + + if exists { + return nil, errors.New("config already exists") + } + + return be, nil + } + + test.OpenFn = func() (backend.Backend, error) { + return rest.Open(cfg) + } +} diff --git a/src/restic/test/backend.go b/src/restic/test/backend.go index ca9ee45b0..5516cecdf 100644 --- a/src/restic/test/backend.go +++ b/src/restic/test/backend.go @@ -23,6 +23,7 @@ var ( TestWalkerPath = getStringVar("RESTIC_TEST_PATH", ".") BenchArchiveDirectory = getStringVar("RESTIC_BENCH_DIR", ".") TestS3Server = getStringVar("RESTIC_TEST_S3_SERVER", "") + TestRESTServer = getStringVar("RESTIC_TEST_REST_SERVER", "") ) func getStringVar(name, defaultValue string) string {