clitools
/
restic
mirror of https://github.com/restic/restic.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Remove Create() everywhere

This commit is contained in:
Alexander Neumann 2016-01-24 20:23:50 +01:00
parent ea29ad6f96
commit 1547d3b656
13 changed files with 25 additions and 401 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
backend/interface.go
View File

@ -1,7 +1,5 @@
package backend
import "io"
// Type is the type of a Blob.
type Type string
@ -21,10 +19,6 @@ type Backend interface {
// repository.
Location() string
// Create creates a new Blob. The data is available only after Finalize()
// has been called on the returned Blob.
Create() (Blob, error)
// Test a boolean value whether a Blob with the name and type exists.
Test(t Type, name string) (bool, error)
@ -65,14 +59,3 @@ type Deleter interface {
type BlobInfo struct {
Size int64
}
// Blob is old.
type Blob interface {
io.Writer
// Finalize moves the data blob to the final location for type and name.
Finalize(t Type, name string) error
// Size returns the number of bytes written to the backend so far.
Size() uint
}

7
backend/local/backend_test.go
View File

@ -51,13 +51,6 @@ func TestLocalBackendLoad(t *testing.T) {
test.TestLoad(t)
}
func TestLocalBackendWrite(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestWrite(t)
}
func TestLocalBackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)

90
backend/local/local.go
View File

@ -14,8 +14,7 @@ import (
"github.com/restic/restic/debug"
)
var ErrWrongData = errors.New("wrong data returned by backend, checksum does not match")
// Local is a backend in a local directory.
type Local struct {
p string
mu sync.Mutex
@ -80,93 +79,6 @@ func (b *Local) Location() string {
return b.p
}
// Return temp directory in correct directory for this backend.
func (b *Local) tempFile() (*os.File, error) {
return ioutil.TempFile(filepath.Join(b.p, backend.Paths.Temp), "temp-")
}
type localBlob struct {
f *os.File
size uint
final bool
basedir string
}
func (lb *localBlob) Write(p []byte) (int, error) {
if lb.final {
return 0, errors.New("blob already closed")
}
n, err := lb.f.Write(p)
lb.size += uint(n)
return n, err
}
func (lb *localBlob) Size() uint {
return lb.size
}
func (lb *localBlob) Finalize(t backend.Type, name string) error {
if lb.final {
return errors.New("Already finalized")
}
lb.final = true
err := lb.f.Close()
if err != nil {
return fmt.Errorf("local: file.Close: %v", err)
}
f := filename(lb.basedir, t, name)
// create directories if necessary, ignore errors
if t == backend.Data {
os.MkdirAll(filepath.Dir(f), backend.Modes.Dir)
}
// test if new path already exists
if _, err := os.Stat(f); err == nil {
return fmt.Errorf("Close(): file %v already exists", f)
}
if err := os.Rename(lb.f.Name(), f); err != nil {
return err
}
// set mode to read-only
fi, err := os.Stat(f)
if err != nil {
return err
}
return setNewFileMode(f, fi)
}
// Create creates a new Blob. The data is available only after Finalize()
// has been called on the returned Blob.
func (b *Local) Create() (backend.Blob, error) {
// TODO: make sure that tempfile is removed upon error
// create tempfile in backend
file, err := b.tempFile()
if err != nil {
return nil, err
}
blob := localBlob{
f: file,
basedir: b.p,
}
b.mu.Lock()
open, _ := b.open["blobs"]
b.open["blobs"] = append(open, file)
b.mu.Unlock()
return &blob, nil
}
// Construct path for given Type and name.
func filename(base string, t backend.Type, name string) string {
if t == backend.Config {

7
backend/mem/backend_test.go
View File

@ -51,13 +51,6 @@ func TestMemBackendLoad(t *testing.T) {
test.TestLoad(t)
}
func TestMemBackendWrite(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestWrite(t)
}
func TestMemBackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)

14
backend/mem/mem_backend.go
View File

@ -37,10 +37,6 @@ func New() *MemoryBackend {
return memTest(be, t, name)
}
be.MockBackend.CreateFn = func() (backend.Blob, error) {
return memCreate(be)
}
be.MockBackend.LoadFn = func(h backend.Handle, p []byte, off int64) (int, error) {
return memLoad(be, h, p, off)
}
@ -127,12 +123,6 @@ func (e *tempMemEntry) Finalize(t backend.Type, name string) error {
return e.be.insert(t, name, e.data.Bytes())
}
func memCreate(be *MemoryBackend) (backend.Blob, error) {
blob := &tempMemEntry{be: be}
debug.Log("MemoryBackend.Create", "create new blob %p", blob)
return blob, nil
}
func memLoad(be *MemoryBackend, h backend.Handle, p []byte, off int64) (int, error) {
if err := h.Valid(); err != nil {
return 0, err
@ -179,6 +169,10 @@ func memSave(be *MemoryBackend, h backend.Handle, p []byte) error {
h.Name = ""
}
if _, ok := be.data[entry{h.Type, h.Name}]; ok {
return errors.New("file already exists")
}
debug.Log("MemoryBackend.Save", "save %v bytes at %v", len(p), h)
buf := make([]byte, len(p))
copy(buf, p)

9
backend/mock_backend.go
View File

@ -6,7 +6,6 @@ import "errors"
// should only be used for tests.
type MockBackend struct {
CloseFn func() error
CreateFn func() (Blob, error)
LoadFn func(h Handle, p []byte, off int64) (int, error)
SaveFn func(h Handle, p []byte) error
StatFn func(h Handle) (BlobInfo, error)
@ -33,14 +32,6 @@ func (m *MockBackend) Location() string {
return m.LocationFn()
}
func (m *MockBackend) Create() (Blob, error) {
if m.CreateFn == nil {
return nil, errors.New("not implemented")
}
return m.CreateFn()
}
func (m *MockBackend) Load(h Handle, p []byte, off int64) (int, error) {
if m.LoadFn == nil {
return 0, errors.New("not implemented")

7
backend/s3/backend_test.go
View File

@ -51,13 +51,6 @@ func TestS3BackendLoad(t *testing.T) {
test.TestLoad(t)
}
func TestS3BackendWrite(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestWrite(t)
}
func TestS3BackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)

79
backend/s3/s3.go
View File

@ -2,7 +2,6 @@ package s3
import (
"bytes"
"errors"
"io"
"strings"
@ -23,6 +22,7 @@ func s3path(t backend.Type, name string) string {
return backendPrefix + "/" + string(t) + "/" + name
}
// S3Backend is a backend which stores the data on an S3 endpoint.
type S3Backend struct {
client minio.CloudStorageClient
connChan chan struct{}
@ -68,83 +68,6 @@ func (be *S3Backend) Location() string {
return be.bucketname
}
type s3Blob struct {
b *S3Backend
buf *bytes.Buffer
final bool
}
func (bb *s3Blob) Write(p []byte) (int, error) {
if bb.final {
return 0, errors.New("blob already closed")
}
n, err := bb.buf.Write(p)
return n, err
}
func (bb *s3Blob) Read(p []byte) (int, error) {
return bb.buf.Read(p)
}
func (bb *s3Blob) Close() error {
bb.final = true
bb.buf.Reset()
return nil
}
func (bb *s3Blob) Size() uint {
return uint(bb.buf.Len())
}
func (bb *s3Blob) Finalize(t backend.Type, name string) error {
debug.Log("s3.blob.Finalize()", "bucket %v, finalize %v, %d bytes", bb.b.bucketname, name, bb.buf.Len())
if bb.final {
return errors.New("Already finalized")
}
bb.final = true
path := s3path(t, name)
// Check key does not already exist
_, err := bb.b.client.StatObject(bb.b.bucketname, path)
if err == nil {
debug.Log("s3.blob.Finalize()", "%v already exists", name)
return errors.New("key already exists")
}
expectedBytes := bb.buf.Len()
<-bb.b.connChan
debug.Log("s3.Finalize", "PutObject(%v, %v, %v, %v)",
bb.b.bucketname, path, int64(bb.buf.Len()), "binary/octet-stream")
n, err := bb.b.client.PutObject(bb.b.bucketname, path, bb.buf, "binary/octet-stream")
debug.Log("s3.Finalize", "finalized %v -> n %v, err %#v", path, n, err)
bb.b.connChan <- struct{}{}
if err != nil {
return err
}
if n != int64(expectedBytes) {
return errors.New("could not store all bytes")
}
return nil
}
// Create creates a new Blob. The data is available only after Finalize()
// has been called on the returned Blob.
func (be *S3Backend) Create() (backend.Blob, error) {
blob := s3Blob{
b: be,
buf: &bytes.Buffer{},
}
return &blob, nil
}
// Load returns the data stored in the backend for h at the given offset
// and saves it in p. Load has the same semantics as io.ReaderAt.
func (be S3Backend) Load(h backend.Handle, p []byte, off int64) (int, error) {

7
backend/sftp/backend_test.go
View File

@ -51,13 +51,6 @@ func TestSftpBackendLoad(t *testing.T) {
test.TestLoad(t)
}
func TestSftpBackendWrite(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestWrite(t)
}
func TestSftpBackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)

70
backend/sftp/sftp.go
View File

@ -22,6 +22,7 @@ const (
tempfileRandomSuffixLength = 10
)
// SFTP is a backend in a directory accessed via SFTP.
type SFTP struct {
c *sftp.Client
p string
@ -253,64 +254,7 @@ func (r *SFTP) renameFile(oldname string, t backend.Type, name string) error {
return r.c.Chmod(filename, fi.Mode()&os.FileMode(^uint32(0222)))
}
type sftpBlob struct {
f *sftp.File
tempname string
size uint
closed bool
backend *SFTP
}
func (sb *sftpBlob) Finalize(t backend.Type, name string) error {
if sb.closed {
return errors.New("Close() called on closed file")
}
sb.closed = true
err := sb.f.Close()
if err != nil {
return fmt.Errorf("sftp: file.Close: %v", err)
}
// rename file
err = sb.backend.renameFile(sb.tempname, t, name)
if err != nil {
return fmt.Errorf("sftp: renameFile: %v", err)
}
return nil
}
func (sb *sftpBlob) Write(p []byte) (int, error) {
n, err := sb.f.Write(p)
sb.size += uint(n)
return n, err
}
func (sb *sftpBlob) Size() uint {
return sb.size
}
// Create creates a new Blob. The data is available only after Finalize()
// has been called on the returned Blob.
func (r *SFTP) Create() (backend.Blob, error) {
// TODO: make sure that tempfile is removed upon error
// create tempfile in backend
filename, file, err := r.tempFile()
if err != nil {
return nil, errors.Annotate(err, "create tempfile")
}
blob := sftpBlob{
f: file,
tempname: filename,
backend: r,
}
return &blob, nil
}
// Join joins the given paths and cleans them afterwards.
func Join(parts ...string) string {
return filepath.Clean(strings.Join(parts, "/"))
}
@ -515,16 +459,16 @@ func (r *SFTP) List(t backend.Type, done <-chan struct{}) <-chan string {
}
// Close closes the sftp connection and terminates the underlying command.
func (s *SFTP) Close() error {
if s == nil {
func (r *SFTP) Close() error {
if r == nil {
return nil
}
s.c.Close()
r.c.Close()
if err := s.cmd.Process.Kill(); err != nil {
if err := r.cmd.Process.Kill(); err != nil {
return err
}
return s.cmd.Wait()
return r.cmd.Wait()
}

7
backend/test/backend_test.go
View File

@ -51,13 +51,6 @@ func TestTestBackendLoad(t *testing.T) {
test.TestLoad(t)
}
func TestTestBackendWrite(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestWrite(t)
}
func TestTestBackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)

105
backend/test/tests.go
View File

@ -156,19 +156,9 @@ func TestConfig(t testing.TB) {
t.Fatalf("did not get expected error for non-existing config")
}
blob, err := b.Create()
err = b.Save(backend.Handle{Type: backend.Config}, []byte(testString))
if err != nil {
t.Fatalf("Create() error: %v", err)
}
_, err = blob.Write([]byte(testString))
if err != nil {
t.Fatalf("Write() error: %v", err)
}
err = blob.Finalize(backend.Config, "")
if err != nil {
t.Fatalf("Finalize() error: %v", err)
t.Fatalf("Save() error: %v", err)
}
// try accessing the config with different names, should all return the
@ -205,12 +195,11 @@ func TestLoad(t testing.TB) {
data := Random(23, length)
id := backend.Hash(data)
blob, err := b.Create()
OK(t, err)
_, err = blob.Write([]byte(data))
OK(t, err)
OK(t, blob.Finalize(backend.Data, id.String()))
handle := backend.Handle{Type: backend.Data, Name: id.String()}
err = b.Save(handle, data)
if err != nil {
t.Fatalf("Save() error: %v", err)
}
for i := 0; i < 500; i++ {
l := rand.Intn(length + 2000)
@ -229,8 +218,7 @@ func TestLoad(t testing.TB) {
}
buf := make([]byte, l)
h := backend.Handle{Type: backend.Data, Name: id.String()}
n, err := b.Load(h, buf, int64(o))
n, err := b.Load(handle, buf, int64(o))
// if we requested data beyond the end of the file, ignore
// ErrUnexpectedEOF error
@ -260,63 +248,6 @@ func TestLoad(t testing.TB) {
OK(t, b.Remove(backend.Data, id.String()))
}
// TestWrite tests writing data to the backend.
func TestWrite(t testing.TB) {
b := open(t)
defer close(t)
length := rand.Intn(1<<23) + 2000
data := Random(23, length)
id := backend.Hash(data)
for i := 0; i < 10; i++ {
blob, err := b.Create()
OK(t, err)
o := 0
for o < len(data) {
l := rand.Intn(len(data) - o)
if len(data)-o < 20 {
l = len(data) - o
}
n, err := blob.Write(data[o : o+l])
OK(t, err)
if n != l {
t.Fatalf("wrong number of bytes written, want %v, got %v", l, n)
}
o += l
}
name := fmt.Sprintf("%s-%d", id, i)
OK(t, blob.Finalize(backend.Data, name))
buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Data, Name: name}, nil)
OK(t, err)
if len(buf) != len(data) {
t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf))
}
if !bytes.Equal(buf, data) {
t.Fatalf("data not equal")
}
fi, err := b.Stat(backend.Handle{Type: backend.Data, Name: name})
OK(t, err)
if fi.Size != int64(len(data)) {
t.Fatalf("Stat() returned different size, want %q, got %d", len(data), fi.Size)
}
err = b.Remove(backend.Data, name)
if err != nil {
t.Fatalf("error removing item: %v", err)
}
}
}
// TestSave tests saving data in the backend.
func TestSave(t testing.TB) {
b := open(t)
@ -415,13 +346,8 @@ var testStrings = []struct {
func store(t testing.TB, b backend.Backend, tpe backend.Type, data []byte) {
id := backend.Hash(data)
blob, err := b.Create()
err := b.Save(backend.Handle{Name: id.String(), Type: tpe}, data)
OK(t, err)
_, err = blob.Write([]byte(data))
OK(t, err)
OK(t, blob.Finalize(tpe, id.String()))
}
func read(t testing.TB, rd io.Reader, expectedData []byte) {
@ -492,12 +418,7 @@ func TestBackend(t testing.TB) {
test := testStrings[0]
// create blob
blob, err := b.Create()
OK(t, err)
_, err = blob.Write([]byte(test.data))
OK(t, err)
err = blob.Finalize(tpe, test.id)
err := b.Save(backend.Handle{Type: tpe, Name: test.id}, []byte(test.data))
Assert(t, err != nil, "expected error, got %v", err)
// remove and recreate
@ -510,13 +431,9 @@ func TestBackend(t testing.TB) {
Assert(t, ok == false, "removed blob still present")
// create blob
blob, err = b.Create()
err = b.Save(backend.Handle{Type: tpe, Name: test.id}, []byte(test.data))
OK(t, err)
_, err = io.Copy(blob, bytes.NewReader([]byte(test.data)))
OK(t, err)
OK(t, blob.Finalize(tpe, test.id))
// list items
IDs := backend.IDs{}

7
backend/utils_test.go
View File

@ -20,14 +20,9 @@ func TestLoadAll(t *testing.T) {
data := Random(23+i, rand.Intn(MiB)+500*KiB)
id := backend.Hash(data)
blob, err := b.Create()
err := b.Save(backend.Handle{Name: id.String(), Type: backend.Data}, data)
OK(t, err)
_, err = blob.Write([]byte(data))
OK(t, err)
OK(t, blob.Finalize(backend.Data, id.String()))
buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Data, Name: id.String()}, nil)
OK(t, err)