mirror of https://github.com/restic/restic.git
Add REST backend
This is a port of the original work by @bchapuis in https://github.com/restic/restic/pull/253
This commit is contained in:
Alexander Neumann
parent
75d69639e6
commit
c2348ba768
|
|
@ -0,0 +1,56 @@
|
|||
REST Backend
|
||||
============
|
||||
|
||||
Restic can interact with HTTP Backend that respects the following REST API.
|
||||
|
||||
## HEAD /config
|
||||
|
||||
Returns "200 OK" if the repository has a configuration,
|
||||
an HTTP error otherwise.
|
||||
|
||||
## GET /config
|
||||
|
||||
Returns the content of the configuration file if the repository has a configuration,
|
||||
an HTTP error otherwise.
|
||||
|
||||
Response format: binary/octet-stream
|
||||
|
||||
## POST /config
|
||||
|
||||
Returns "200 OK" if the configuration of the request body has been saved,
|
||||
an HTTP error otherwise.
|
||||
|
||||
## GET /{type}/
|
||||
|
||||
Returns a JSON array containing the names of all the blobs stored for a given type.
|
||||
|
||||
Response format: JSON
|
||||
|
||||
## HEAD /{type}/{name}
|
||||
|
||||
Returns "200 OK" if the blob with the given name and type is stored in the repository,
|
||||
"404 not found" otherwise. If the blob exists, the HTTP header `Content-Length`
|
||||
is set to the file size.
|
||||
|
||||
## GET /{type}/{name}
|
||||
|
||||
Returns the content of the blob with the given name and type if it is stored in the repository,
|
||||
"404 not found" otherwise.
|
||||
|
||||
If the request specifies a partial read with a Range header field,
|
||||
then the status code of the response is 206 instead of 200
|
||||
and the response only contains the specified range.
|
||||
|
||||
Response format: binary/octet-stream
|
||||
|
||||
## POST /{type}/{name}
|
||||
|
||||
Saves the content of the request body as a blob with the given name and type,
|
||||
an HTTP error otherwise.
|
||||
|
||||
Request format: binary/octet-stream
|
||||
|
||||
## DELETE /{type}/{name}
|
||||
|
||||
Returns "200 OK" if the blob with the given name and type has been deleted from the repository,
|
||||
an HTTP error otherwise.
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
// DO NOT EDIT, AUTOMATICALLY GENERATED
|
||||
package rest_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"restic/backend/test"
|
||||
)
|
||||
|
||||
var SkipMessage string
|
||||
|
||||
func TestRestBackendCreate(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestCreate(t)
|
||||
}
|
||||
|
||||
func TestRestBackendOpen(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestOpen(t)
|
||||
}
|
||||
|
||||
func TestRestBackendCreateWithConfig(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestCreateWithConfig(t)
|
||||
}
|
||||
|
||||
func TestRestBackendLocation(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestLocation(t)
|
||||
}
|
||||
|
||||
func TestRestBackendConfig(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestConfig(t)
|
||||
}
|
||||
|
||||
func TestRestBackendLoad(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestLoad(t)
|
||||
}
|
||||
|
||||
func TestRestBackendSave(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestSave(t)
|
||||
}
|
||||
|
||||
func TestRestBackendSaveFilenames(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestSaveFilenames(t)
|
||||
}
|
||||
|
||||
func TestRestBackendBackend(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestBackend(t)
|
||||
}
|
||||
|
||||
func TestRestBackendDelete(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestDelete(t)
|
||||
}
|
||||
|
||||
func TestRestBackendCleanup(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestCleanup(t)
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
package rest
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Config contains all configuration necessary to connect to a REST server.
|
||||
type Config struct {
|
||||
URL *url.URL
|
||||
}
|
||||
|
||||
// ParseConfig parses the string s and extracts the REST server URL.
|
||||
func ParseConfig(s string) (interface{}, error) {
|
||||
if !strings.HasPrefix(s, "rest:") {
|
||||
return nil, errors.New("invalid REST backend specification")
|
||||
}
|
||||
|
||||
s = s[5:]
|
||||
u, err := url.Parse(s)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg := Config{URL: u}
|
||||
return cfg, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
package rest
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func parseURL(s string) *url.URL {
|
||||
u, err := url.Parse(s)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return u
|
||||
}
|
||||
|
||||
var configTests = []struct {
|
||||
s string
|
||||
cfg Config
|
||||
}{
|
||||
{"rest:http://localhost:1234", Config{
|
||||
URL: parseURL("http://localhost:1234"),
|
||||
}},
|
||||
}
|
||||
|
||||
func TestParseConfig(t *testing.T) {
|
||||
for i, test := range configTests {
|
||||
cfg, err := ParseConfig(test.s)
|
||||
if err != nil {
|
||||
t.Errorf("test %d:%s failed: %v", i, test.s, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(cfg, test.cfg) {
|
||||
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
|
||||
i, test.s, test.cfg, cfg)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,258 @@
|
|||
package rest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
|
||||
"restic/backend"
|
||||
)
|
||||
|
||||
const connLimit = 10
|
||||
|
||||
// restPath returns the path to the given resource.
|
||||
func restPath(url *url.URL, h backend.Handle) string {
|
||||
p := url.Path
|
||||
if p == "" {
|
||||
p = "/"
|
||||
}
|
||||
|
||||
var dir string
|
||||
|
||||
switch h.Type {
|
||||
case backend.Config:
|
||||
dir = ""
|
||||
case backend.Data:
|
||||
dir = backend.Paths.Data
|
||||
case backend.Snapshot:
|
||||
dir = backend.Paths.Snapshots
|
||||
case backend.Index:
|
||||
dir = backend.Paths.Index
|
||||
case backend.Lock:
|
||||
dir = backend.Paths.Locks
|
||||
case backend.Key:
|
||||
dir = backend.Paths.Keys
|
||||
default:
|
||||
dir = string(h.Type)
|
||||
}
|
||||
|
||||
return path.Join(p, dir, h.Name)
|
||||
}
|
||||
|
||||
type restBackend struct {
|
||||
url *url.URL
|
||||
connChan chan struct{}
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Open opens the REST backend with the given config.
|
||||
func Open(cfg Config) (backend.Backend, error) {
|
||||
connChan := make(chan struct{}, connLimit)
|
||||
for i := 0; i < connLimit; i++ {
|
||||
connChan <- struct{}{}
|
||||
}
|
||||
tr := &http.Transport{}
|
||||
client := http.Client{Transport: tr}
|
||||
|
||||
return &restBackend{url: cfg.URL, connChan: connChan, client: &client}, nil
|
||||
}
|
||||
|
||||
// Location returns this backend's location (the server's URL).
|
||||
func (b *restBackend) Location() string {
|
||||
return b.url.String()
|
||||
}
|
||||
|
||||
// Load returns the data stored in the backend for h at the given offset
|
||||
// and saves it in p. Load has the same semantics as io.ReaderAt.
|
||||
func (b *restBackend) Load(h backend.Handle, p []byte, off int64) (n int, err error) {
|
||||
if err := h.Valid(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("GET", restPath(b.url, h), nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))))
|
||||
client := *b.client
|
||||
|
||||
<-b.connChan
|
||||
resp, err := client.Do(req)
|
||||
b.connChan <- struct{}{}
|
||||
|
||||
if resp != nil {
|
||||
defer func() {
|
||||
e := resp.Body.Close()
|
||||
|
||||
if err == nil {
|
||||
err = e
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if resp.StatusCode != 206 {
|
||||
return 0, errors.New("blob not found")
|
||||
}
|
||||
|
||||
return io.ReadFull(resp.Body, p)
|
||||
}
|
||||
|
||||
// Save stores data in the backend at the handle.
|
||||
func (b *restBackend) Save(h backend.Handle, p []byte) (err error) {
|
||||
if err := h.Valid(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
client := *b.client
|
||||
|
||||
<-b.connChan
|
||||
resp, err := client.Post(restPath(b.url, h), "binary/octet-stream", bytes.NewReader(p))
|
||||
b.connChan <- struct{}{}
|
||||
|
||||
if resp != nil {
|
||||
defer func() {
|
||||
e := resp.Body.Close()
|
||||
|
||||
if err == nil {
|
||||
err = e
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return errors.New("blob not saved")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stat returns information about a blob.
|
||||
func (b *restBackend) Stat(h backend.Handle) (backend.BlobInfo, error) {
|
||||
if err := h.Valid(); err != nil {
|
||||
return backend.BlobInfo{}, err
|
||||
}
|
||||
|
||||
client := *b.client
|
||||
<-b.connChan
|
||||
resp, err := client.Head(restPath(b.url, h))
|
||||
b.connChan <- struct{}{}
|
||||
if err != nil {
|
||||
return backend.BlobInfo{}, err
|
||||
}
|
||||
|
||||
if err = resp.Body.Close(); err != nil {
|
||||
return backend.BlobInfo{}, err
|
||||
}
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return backend.BlobInfo{}, errors.New("blob not saved")
|
||||
}
|
||||
|
||||
if resp.ContentLength < 0 {
|
||||
return backend.BlobInfo{}, errors.New("negative content length")
|
||||
}
|
||||
|
||||
bi := backend.BlobInfo{
|
||||
Size: resp.ContentLength,
|
||||
}
|
||||
|
||||
return bi, nil
|
||||
}
|
||||
|
||||
// Test returns true if a blob of the given type and name exists in the backend.
|
||||
func (b *restBackend) Test(t backend.Type, name string) (bool, error) {
|
||||
_, err := b.Stat(backend.Handle{Type: t, Name: name})
|
||||
if err != nil {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Remove removes the blob with the given name and type.
|
||||
func (b *restBackend) Remove(t backend.Type, name string) error {
|
||||
h := backend.Handle{Type: t, Name: name}
|
||||
if err := h.Valid(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("DELETE", restPath(b.url, h), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client := *b.client
|
||||
|
||||
<-b.connChan
|
||||
resp, err := client.Do(req)
|
||||
b.connChan <- struct{}{}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return errors.New("blob not removed")
|
||||
}
|
||||
|
||||
return resp.Body.Close()
|
||||
}
|
||||
|
||||
// List returns a channel that yields all names of blobs of type t. A
|
||||
// goroutine is started for this. If the channel done is closed, sending
|
||||
// stops.
|
||||
func (b *restBackend) List(t backend.Type, done <-chan struct{}) <-chan string {
|
||||
ch := make(chan string)
|
||||
|
||||
client := *b.client
|
||||
<-b.connChan
|
||||
resp, err := client.Get(restPath(b.url, backend.Handle{Type: t}))
|
||||
b.connChan <- struct{}{}
|
||||
|
||||
if resp != nil {
|
||||
defer resp.Body.Close()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
dec := json.NewDecoder(resp.Body)
|
||||
var list []string
|
||||
if err = dec.Decode(&list); err != nil {
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
for _, m := range list {
|
||||
select {
|
||||
case ch <- m:
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
// Close closes all open files.
|
||||
func (b *restBackend) Close() error {
|
||||
// this does not need to do anything, all open files are closed within the
|
||||
// same function.
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
package rest_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
|
||||
"restic/backend"
|
||||
"restic/backend/rest"
|
||||
"restic/backend/test"
|
||||
. "restic/test"
|
||||
)
|
||||
|
||||
//go:generate go run ../test/generate_backend_tests.go
|
||||
|
||||
func init() {
|
||||
if TestRESTServer == "" {
|
||||
SkipMessage = "REST test server not available"
|
||||
return
|
||||
}
|
||||
|
||||
url, err := url.Parse(TestRESTServer)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "invalid url: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
cfg := rest.Config{
|
||||
URL: url,
|
||||
}
|
||||
|
||||
test.CreateFn = func() (backend.Backend, error) {
|
||||
be, err := rest.Open(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
exists, err := be.Test(backend.Config, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if exists {
|
||||
return nil, errors.New("config already exists")
|
||||
}
|
||||
|
||||
return be, nil
|
||||
}
|
||||
|
||||
test.OpenFn = func() (backend.Backend, error) {
|
||||
return rest.Open(cfg)
|
||||
}
|
||||
}
|
||||
|
|
@ -23,6 +23,7 @@ var (
|
|||
TestWalkerPath = getStringVar("RESTIC_TEST_PATH", ".")
|
||||
BenchArchiveDirectory = getStringVar("RESTIC_BENCH_DIR", ".")
|
||||
TestS3Server = getStringVar("RESTIC_TEST_S3_SERVER", "")
|
||||
TestRESTServer = getStringVar("RESTIC_TEST_REST_SERVER", "")
|
||||
)
|
||||
|
||||
func getStringVar(name, defaultValue string) string {
|
||||
|
|
|
|||
Loading…
Reference in New Issue