diff --git a/backend/interface.go b/backend/interface.go index 9fff4bf1e..fb0927c6e 100644 --- a/backend/interface.go +++ b/backend/interface.go @@ -1,7 +1,5 @@ package backend -import "io" - // Type is the type of a Blob. type Type string @@ -21,10 +19,6 @@ type Backend interface { // repository. Location() string - // Create creates a new Blob. The data is available only after Finalize() - // has been called on the returned Blob. - Create() (Blob, error) - // Test a boolean value whether a Blob with the name and type exists. Test(t Type, name string) (bool, error) @@ -65,14 +59,3 @@ type Deleter interface { type BlobInfo struct { Size int64 } - -// Blob is old. -type Blob interface { - io.Writer - - // Finalize moves the data blob to the final location for type and name. - Finalize(t Type, name string) error - - // Size returns the number of bytes written to the backend so far. - Size() uint -} diff --git a/backend/local/backend_test.go b/backend/local/backend_test.go index 5617ab74a..b2d5e7b0f 100644 --- a/backend/local/backend_test.go +++ b/backend/local/backend_test.go @@ -51,13 +51,6 @@ func TestLocalBackendLoad(t *testing.T) { test.TestLoad(t) } -func TestLocalBackendWrite(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestWrite(t) -} - func TestLocalBackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/local/local.go b/backend/local/local.go index e45221f91..9a6409a75 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -14,8 +14,7 @@ import ( "github.com/restic/restic/debug" ) -var ErrWrongData = errors.New("wrong data returned by backend, checksum does not match") - +// Local is a backend in a local directory. type Local struct { p string mu sync.Mutex @@ -80,93 +79,6 @@ func (b *Local) Location() string { return b.p } -// Return temp directory in correct directory for this backend. -func (b *Local) tempFile() (*os.File, error) { - return ioutil.TempFile(filepath.Join(b.p, backend.Paths.Temp), "temp-") -} - -type localBlob struct { - f *os.File - size uint - final bool - basedir string -} - -func (lb *localBlob) Write(p []byte) (int, error) { - if lb.final { - return 0, errors.New("blob already closed") - } - - n, err := lb.f.Write(p) - lb.size += uint(n) - return n, err -} - -func (lb *localBlob) Size() uint { - return lb.size -} - -func (lb *localBlob) Finalize(t backend.Type, name string) error { - if lb.final { - return errors.New("Already finalized") - } - - lb.final = true - - err := lb.f.Close() - if err != nil { - return fmt.Errorf("local: file.Close: %v", err) - } - - f := filename(lb.basedir, t, name) - - // create directories if necessary, ignore errors - if t == backend.Data { - os.MkdirAll(filepath.Dir(f), backend.Modes.Dir) - } - - // test if new path already exists - if _, err := os.Stat(f); err == nil { - return fmt.Errorf("Close(): file %v already exists", f) - } - - if err := os.Rename(lb.f.Name(), f); err != nil { - return err - } - - // set mode to read-only - fi, err := os.Stat(f) - if err != nil { - return err - } - - return setNewFileMode(f, fi) -} - -// Create creates a new Blob. The data is available only after Finalize() -// has been called on the returned Blob. -func (b *Local) Create() (backend.Blob, error) { - // TODO: make sure that tempfile is removed upon error - - // create tempfile in backend - file, err := b.tempFile() - if err != nil { - return nil, err - } - - blob := localBlob{ - f: file, - basedir: b.p, - } - - b.mu.Lock() - open, _ := b.open["blobs"] - b.open["blobs"] = append(open, file) - b.mu.Unlock() - - return &blob, nil -} - // Construct path for given Type and name. func filename(base string, t backend.Type, name string) string { if t == backend.Config { diff --git a/backend/mem/backend_test.go b/backend/mem/backend_test.go index 9b1a602a1..31f86e4fc 100644 --- a/backend/mem/backend_test.go +++ b/backend/mem/backend_test.go @@ -51,13 +51,6 @@ func TestMemBackendLoad(t *testing.T) { test.TestLoad(t) } -func TestMemBackendWrite(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestWrite(t) -} - func TestMemBackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/mem/mem_backend.go b/backend/mem/mem_backend.go index 91831dc5e..4de0afe82 100644 --- a/backend/mem/mem_backend.go +++ b/backend/mem/mem_backend.go @@ -37,10 +37,6 @@ func New() *MemoryBackend { return memTest(be, t, name) } - be.MockBackend.CreateFn = func() (backend.Blob, error) { - return memCreate(be) - } - be.MockBackend.LoadFn = func(h backend.Handle, p []byte, off int64) (int, error) { return memLoad(be, h, p, off) } @@ -127,12 +123,6 @@ func (e *tempMemEntry) Finalize(t backend.Type, name string) error { return e.be.insert(t, name, e.data.Bytes()) } -func memCreate(be *MemoryBackend) (backend.Blob, error) { - blob := &tempMemEntry{be: be} - debug.Log("MemoryBackend.Create", "create new blob %p", blob) - return blob, nil -} - func memLoad(be *MemoryBackend, h backend.Handle, p []byte, off int64) (int, error) { if err := h.Valid(); err != nil { return 0, err @@ -179,6 +169,10 @@ func memSave(be *MemoryBackend, h backend.Handle, p []byte) error { h.Name = "" } + if _, ok := be.data[entry{h.Type, h.Name}]; ok { + return errors.New("file already exists") + } + debug.Log("MemoryBackend.Save", "save %v bytes at %v", len(p), h) buf := make([]byte, len(p)) copy(buf, p) diff --git a/backend/mock_backend.go b/backend/mock_backend.go index 9e692883f..e685ba556 100644 --- a/backend/mock_backend.go +++ b/backend/mock_backend.go @@ -6,7 +6,6 @@ import "errors" // should only be used for tests. type MockBackend struct { CloseFn func() error - CreateFn func() (Blob, error) LoadFn func(h Handle, p []byte, off int64) (int, error) SaveFn func(h Handle, p []byte) error StatFn func(h Handle) (BlobInfo, error) @@ -33,14 +32,6 @@ func (m *MockBackend) Location() string { return m.LocationFn() } -func (m *MockBackend) Create() (Blob, error) { - if m.CreateFn == nil { - return nil, errors.New("not implemented") - } - - return m.CreateFn() -} - func (m *MockBackend) Load(h Handle, p []byte, off int64) (int, error) { if m.LoadFn == nil { return 0, errors.New("not implemented") diff --git a/backend/s3/backend_test.go b/backend/s3/backend_test.go index 4bd4d6bee..8db0a31f9 100644 --- a/backend/s3/backend_test.go +++ b/backend/s3/backend_test.go @@ -51,13 +51,6 @@ func TestS3BackendLoad(t *testing.T) { test.TestLoad(t) } -func TestS3BackendWrite(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestWrite(t) -} - func TestS3BackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 9fd5c6152..a8f69c4c9 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -2,7 +2,6 @@ package s3 import ( "bytes" - "errors" "io" "strings" @@ -23,6 +22,7 @@ func s3path(t backend.Type, name string) string { return backendPrefix + "/" + string(t) + "/" + name } +// S3Backend is a backend which stores the data on an S3 endpoint. type S3Backend struct { client minio.CloudStorageClient connChan chan struct{} @@ -68,83 +68,6 @@ func (be *S3Backend) Location() string { return be.bucketname } -type s3Blob struct { - b *S3Backend - buf *bytes.Buffer - final bool -} - -func (bb *s3Blob) Write(p []byte) (int, error) { - if bb.final { - return 0, errors.New("blob already closed") - } - - n, err := bb.buf.Write(p) - return n, err -} - -func (bb *s3Blob) Read(p []byte) (int, error) { - return bb.buf.Read(p) -} - -func (bb *s3Blob) Close() error { - bb.final = true - bb.buf.Reset() - return nil -} - -func (bb *s3Blob) Size() uint { - return uint(bb.buf.Len()) -} - -func (bb *s3Blob) Finalize(t backend.Type, name string) error { - debug.Log("s3.blob.Finalize()", "bucket %v, finalize %v, %d bytes", bb.b.bucketname, name, bb.buf.Len()) - if bb.final { - return errors.New("Already finalized") - } - - bb.final = true - - path := s3path(t, name) - - // Check key does not already exist - _, err := bb.b.client.StatObject(bb.b.bucketname, path) - if err == nil { - debug.Log("s3.blob.Finalize()", "%v already exists", name) - return errors.New("key already exists") - } - - expectedBytes := bb.buf.Len() - - <-bb.b.connChan - debug.Log("s3.Finalize", "PutObject(%v, %v, %v, %v)", - bb.b.bucketname, path, int64(bb.buf.Len()), "binary/octet-stream") - n, err := bb.b.client.PutObject(bb.b.bucketname, path, bb.buf, "binary/octet-stream") - debug.Log("s3.Finalize", "finalized %v -> n %v, err %#v", path, n, err) - bb.b.connChan <- struct{}{} - - if err != nil { - return err - } - - if n != int64(expectedBytes) { - return errors.New("could not store all bytes") - } - - return nil -} - -// Create creates a new Blob. The data is available only after Finalize() -// has been called on the returned Blob. -func (be *S3Backend) Create() (backend.Blob, error) { - blob := s3Blob{ - b: be, - buf: &bytes.Buffer{}, - } - - return &blob, nil -} - // Load returns the data stored in the backend for h at the given offset // and saves it in p. Load has the same semantics as io.ReaderAt. func (be S3Backend) Load(h backend.Handle, p []byte, off int64) (int, error) { diff --git a/backend/sftp/backend_test.go b/backend/sftp/backend_test.go index ca268b3a3..afab17e60 100644 --- a/backend/sftp/backend_test.go +++ b/backend/sftp/backend_test.go @@ -51,13 +51,6 @@ func TestSftpBackendLoad(t *testing.T) { test.TestLoad(t) } -func TestSftpBackendWrite(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestWrite(t) -} - func TestSftpBackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index a7ad4955e..94cc91132 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -22,6 +22,7 @@ const ( tempfileRandomSuffixLength = 10 ) +// SFTP is a backend in a directory accessed via SFTP. type SFTP struct { c *sftp.Client p string @@ -253,64 +254,7 @@ func (r *SFTP) renameFile(oldname string, t backend.Type, name string) error { return r.c.Chmod(filename, fi.Mode()&os.FileMode(^uint32(0222))) } -type sftpBlob struct { - f *sftp.File - tempname string - size uint - closed bool - backend *SFTP -} - -func (sb *sftpBlob) Finalize(t backend.Type, name string) error { - if sb.closed { - return errors.New("Close() called on closed file") - } - sb.closed = true - - err := sb.f.Close() - if err != nil { - return fmt.Errorf("sftp: file.Close: %v", err) - } - - // rename file - err = sb.backend.renameFile(sb.tempname, t, name) - if err != nil { - return fmt.Errorf("sftp: renameFile: %v", err) - } - - return nil -} - -func (sb *sftpBlob) Write(p []byte) (int, error) { - n, err := sb.f.Write(p) - sb.size += uint(n) - return n, err -} - -func (sb *sftpBlob) Size() uint { - return sb.size -} - -// Create creates a new Blob. The data is available only after Finalize() -// has been called on the returned Blob. -func (r *SFTP) Create() (backend.Blob, error) { - // TODO: make sure that tempfile is removed upon error - - // create tempfile in backend - filename, file, err := r.tempFile() - if err != nil { - return nil, errors.Annotate(err, "create tempfile") - } - - blob := sftpBlob{ - f: file, - tempname: filename, - backend: r, - } - - return &blob, nil -} - +// Join joins the given paths and cleans them afterwards. func Join(parts ...string) string { return filepath.Clean(strings.Join(parts, "/")) } @@ -515,16 +459,16 @@ func (r *SFTP) List(t backend.Type, done <-chan struct{}) <-chan string { } // Close closes the sftp connection and terminates the underlying command. -func (s *SFTP) Close() error { - if s == nil { +func (r *SFTP) Close() error { + if r == nil { return nil } - s.c.Close() + r.c.Close() - if err := s.cmd.Process.Kill(); err != nil { + if err := r.cmd.Process.Kill(); err != nil { return err } - return s.cmd.Wait() + return r.cmd.Wait() } diff --git a/backend/test/backend_test.go b/backend/test/backend_test.go index e9c19680d..c1bee84c7 100644 --- a/backend/test/backend_test.go +++ b/backend/test/backend_test.go @@ -51,13 +51,6 @@ func TestTestBackendLoad(t *testing.T) { test.TestLoad(t) } -func TestTestBackendWrite(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestWrite(t) -} - func TestTestBackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/test/tests.go b/backend/test/tests.go index b3557d10d..f1b4b8107 100644 --- a/backend/test/tests.go +++ b/backend/test/tests.go @@ -156,19 +156,9 @@ func TestConfig(t testing.TB) { t.Fatalf("did not get expected error for non-existing config") } - blob, err := b.Create() + err = b.Save(backend.Handle{Type: backend.Config}, []byte(testString)) if err != nil { - t.Fatalf("Create() error: %v", err) - } - - _, err = blob.Write([]byte(testString)) - if err != nil { - t.Fatalf("Write() error: %v", err) - } - - err = blob.Finalize(backend.Config, "") - if err != nil { - t.Fatalf("Finalize() error: %v", err) + t.Fatalf("Save() error: %v", err) } // try accessing the config with different names, should all return the @@ -205,12 +195,11 @@ func TestLoad(t testing.TB) { data := Random(23, length) id := backend.Hash(data) - blob, err := b.Create() - OK(t, err) - - _, err = blob.Write([]byte(data)) - OK(t, err) - OK(t, blob.Finalize(backend.Data, id.String())) + handle := backend.Handle{Type: backend.Data, Name: id.String()} + err = b.Save(handle, data) + if err != nil { + t.Fatalf("Save() error: %v", err) + } for i := 0; i < 500; i++ { l := rand.Intn(length + 2000) @@ -229,8 +218,7 @@ func TestLoad(t testing.TB) { } buf := make([]byte, l) - h := backend.Handle{Type: backend.Data, Name: id.String()} - n, err := b.Load(h, buf, int64(o)) + n, err := b.Load(handle, buf, int64(o)) // if we requested data beyond the end of the file, ignore // ErrUnexpectedEOF error @@ -260,63 +248,6 @@ func TestLoad(t testing.TB) { OK(t, b.Remove(backend.Data, id.String())) } -// TestWrite tests writing data to the backend. -func TestWrite(t testing.TB) { - b := open(t) - defer close(t) - - length := rand.Intn(1<<23) + 2000 - - data := Random(23, length) - id := backend.Hash(data) - - for i := 0; i < 10; i++ { - blob, err := b.Create() - OK(t, err) - - o := 0 - for o < len(data) { - l := rand.Intn(len(data) - o) - if len(data)-o < 20 { - l = len(data) - o - } - - n, err := blob.Write(data[o : o+l]) - OK(t, err) - if n != l { - t.Fatalf("wrong number of bytes written, want %v, got %v", l, n) - } - - o += l - } - - name := fmt.Sprintf("%s-%d", id, i) - OK(t, blob.Finalize(backend.Data, name)) - - buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Data, Name: name}, nil) - OK(t, err) - if len(buf) != len(data) { - t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf)) - } - - if !bytes.Equal(buf, data) { - t.Fatalf("data not equal") - } - - fi, err := b.Stat(backend.Handle{Type: backend.Data, Name: name}) - OK(t, err) - - if fi.Size != int64(len(data)) { - t.Fatalf("Stat() returned different size, want %q, got %d", len(data), fi.Size) - } - - err = b.Remove(backend.Data, name) - if err != nil { - t.Fatalf("error removing item: %v", err) - } - } -} - // TestSave tests saving data in the backend. func TestSave(t testing.TB) { b := open(t) @@ -415,13 +346,8 @@ var testStrings = []struct { func store(t testing.TB, b backend.Backend, tpe backend.Type, data []byte) { id := backend.Hash(data) - - blob, err := b.Create() + err := b.Save(backend.Handle{Name: id.String(), Type: tpe}, data) OK(t, err) - - _, err = blob.Write([]byte(data)) - OK(t, err) - OK(t, blob.Finalize(tpe, id.String())) } func read(t testing.TB, rd io.Reader, expectedData []byte) { @@ -492,12 +418,7 @@ func TestBackend(t testing.TB) { test := testStrings[0] // create blob - blob, err := b.Create() - OK(t, err) - - _, err = blob.Write([]byte(test.data)) - OK(t, err) - err = blob.Finalize(tpe, test.id) + err := b.Save(backend.Handle{Type: tpe, Name: test.id}, []byte(test.data)) Assert(t, err != nil, "expected error, got %v", err) // remove and recreate @@ -510,13 +431,9 @@ func TestBackend(t testing.TB) { Assert(t, ok == false, "removed blob still present") // create blob - blob, err = b.Create() + err = b.Save(backend.Handle{Type: tpe, Name: test.id}, []byte(test.data)) OK(t, err) - _, err = io.Copy(blob, bytes.NewReader([]byte(test.data))) - OK(t, err) - OK(t, blob.Finalize(tpe, test.id)) - // list items IDs := backend.IDs{} diff --git a/backend/utils_test.go b/backend/utils_test.go index 426f866de..98a0106ef 100644 --- a/backend/utils_test.go +++ b/backend/utils_test.go @@ -20,14 +20,9 @@ func TestLoadAll(t *testing.T) { data := Random(23+i, rand.Intn(MiB)+500*KiB) id := backend.Hash(data) - - blob, err := b.Create() + err := b.Save(backend.Handle{Name: id.String(), Type: backend.Data}, data) OK(t, err) - _, err = blob.Write([]byte(data)) - OK(t, err) - OK(t, blob.Finalize(backend.Data, id.String())) - buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Data, Name: id.String()}, nil) OK(t, err)