Merge pull request #999 from restic/backend-use-semaphore

backends: Use new semaphore

src/restic/backend/b2/b2.go

@@ -53,6 +53,11 @@ func Open(cfg Config) (restic.Backend, error) {
 		return nil, errors.Wrap(err, "Bucket")
 	}
 
+	sem, err := backend.NewSemaphore(cfg.Connections)
+	if err != nil {
+		return nil, err
+	}
+
 	be := &b2Backend{
 		client: client,
 		bucket: bucket,
@@ -61,7 +66,7 @@ func Open(cfg Config) (restic.Backend, error) {
 			Join: path.Join,
 			Path: cfg.Prefix,
 		},
-		sem: backend.NewSemaphore(cfg.Connections),
+		sem: sem,
 	}
 
 	return be, nil
@@ -88,6 +93,11 @@ func Create(cfg Config) (restic.Backend, error) {
 		return nil, errors.Wrap(err, "NewBucket")
 	}
 
+	sem, err := backend.NewSemaphore(cfg.Connections)
+	if err != nil {
+		return nil, err
+	}
+
 	be := &b2Backend{
 		client: client,
 		bucket: bucket,
@@ -96,7 +106,7 @@ func Create(cfg Config) (restic.Backend, error) {
 			Join: path.Join,
 			Path: cfg.Prefix,
 		},
-		sem: backend.NewSemaphore(cfg.Connections),
+		sem: sem,
 	}
 
 	present, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})

src/restic/backend/b2/config.go

@@ -17,7 +17,7 @@ type Config struct {
 	Bucket    string
 	Prefix    string
 
-	Connections int `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
+	Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
 }
 
 // NewConfig returns a new config with default options applied.

src/restic/backend/location/location_test.go

@@ -120,9 +120,10 @@ var parseTests = []struct {
 		"s3://eu-central-1/bucketname",
 		Location{Scheme: "s3",
 			Config: s3.Config{
-				Endpoint: "eu-central-1",
-				Bucket:   "bucketname",
-				Prefix:   "restic",
+				Endpoint:    "eu-central-1",
+				Bucket:      "bucketname",
+				Prefix:      "restic",
+				Connections: 20,
 			},
 		},
 	},
@@ -130,9 +131,10 @@ var parseTests = []struct {
 		"s3://hostname.foo/bucketname",
 		Location{Scheme: "s3",
 			Config: s3.Config{
-				Endpoint: "hostname.foo",
-				Bucket:   "bucketname",
-				Prefix:   "restic",
+				Endpoint:    "hostname.foo",
+				Bucket:      "bucketname",
+				Prefix:      "restic",
+				Connections: 20,
 			},
 		},
 	},
@@ -140,9 +142,10 @@ var parseTests = []struct {
 		"s3://hostname.foo/bucketname/prefix/directory",
 		Location{Scheme: "s3",
 			Config: s3.Config{
-				Endpoint: "hostname.foo",
-				Bucket:   "bucketname",
-				Prefix:   "prefix/directory",
+				Endpoint:    "hostname.foo",
+				Bucket:      "bucketname",
+				Prefix:      "prefix/directory",
+				Connections: 20,
 			},
 		},
 	},
@@ -150,9 +153,10 @@ var parseTests = []struct {
 		"s3:eu-central-1/repo",
 		Location{Scheme: "s3",
 			Config: s3.Config{
-				Endpoint: "eu-central-1",
-				Bucket:   "repo",
-				Prefix:   "restic",
+				Endpoint:    "eu-central-1",
+				Bucket:      "repo",
+				Prefix:      "restic",
+				Connections: 20,
 			},
 		},
 	},
@@ -160,9 +164,10 @@ var parseTests = []struct {
 		"s3:eu-central-1/repo/prefix/directory",
 		Location{Scheme: "s3",
 			Config: s3.Config{
-				Endpoint: "eu-central-1",
-				Bucket:   "repo",
-				Prefix:   "prefix/directory",
+				Endpoint:    "eu-central-1",
+				Bucket:      "repo",
+				Prefix:      "prefix/directory",
+				Connections: 20,
 			},
 		},
 	},
@@ -170,9 +175,10 @@ var parseTests = []struct {
 		"s3:https://hostname.foo/repo",
 		Location{Scheme: "s3",
 			Config: s3.Config{
-				Endpoint: "hostname.foo",
-				Bucket:   "repo",
-				Prefix:   "restic",
+				Endpoint:    "hostname.foo",
+				Bucket:      "repo",
+				Prefix:      "restic",
+				Connections: 20,
 			},
 		},
 	},
@@ -180,9 +186,10 @@ var parseTests = []struct {
 		"s3:https://hostname.foo/repo/prefix/directory",
 		Location{Scheme: "s3",
 			Config: s3.Config{
-				Endpoint: "hostname.foo",
-				Bucket:   "repo",
-				Prefix:   "prefix/directory",
+				Endpoint:    "hostname.foo",
+				Bucket:      "repo",
+				Prefix:      "prefix/directory",
+				Connections: 20,
 			},
 		},
 	},
@@ -190,10 +197,11 @@ var parseTests = []struct {
 		"s3:http://hostname.foo/repo",
 		Location{Scheme: "s3",
 			Config: s3.Config{
-				Endpoint: "hostname.foo",
-				Bucket:   "repo",
-				Prefix:   "restic",
-				UseHTTP:  true,
+				Endpoint:    "hostname.foo",
+				Bucket:      "repo",
+				Prefix:      "restic",
+				UseHTTP:     true,
+				Connections: 20,
 			},
 		},
 	},
@@ -201,8 +209,9 @@ var parseTests = []struct {
 		"swift:container17:/",
 		Location{Scheme: "swift",
 			Config: swift.Config{
-				Container: "container17",
-				Prefix:    "",
+				Container:   "container17",
+				Prefix:      "",
+				Connections: 20,
 			},
 		},
 	},
@@ -210,8 +219,9 @@ var parseTests = []struct {
 		"swift:container17:/prefix97",
 		Location{Scheme: "swift",
 			Config: swift.Config{
-				Container: "container17",
-				Prefix:    "prefix97",
+				Container:   "container17",
+				Prefix:      "prefix97",
+				Connections: 20,
 			},
 		},
 	},
@@ -219,7 +229,8 @@ var parseTests = []struct {
 		"rest:http://hostname.foo:1234/",
 		Location{Scheme: "rest",
 			Config: rest.Config{
-				URL: parseURL("http://hostname.foo:1234/"),
+				URL:         parseURL("http://hostname.foo:1234/"),
+				Connections: 20,
 			},
 		},
 	},

src/restic/backend/rest/config.go

@@ -5,11 +5,24 @@ import (
 	"strings"
 
 	"restic/errors"
+	"restic/options"
 )
 
 // Config contains all configuration necessary to connect to a REST server.
 type Config struct {
-	URL *url.URL
+	URL         *url.URL
+	Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 20)"`
+}
+
+func init() {
+	options.Register("rest", Config{})
+}
+
+// NewConfig returns a new Config with the default values filled in.
+func NewConfig() Config {
+	return Config{
+		Connections: 20,
+	}
 }
 
 // ParseConfig parses the string s and extracts the REST server URL.
@@ -25,6 +38,7 @@ func ParseConfig(s string) (interface{}, error) {
 		return nil, errors.Wrap(err, "url.Parse")
 	}
 
-	cfg := Config{URL: u}
+	cfg := NewConfig()
+	cfg.URL = u
 	return cfg, nil
 }

src/restic/backend/rest/config_test.go

@@ -20,7 +20,8 @@ var configTests = []struct {
 	cfg Config
 }{
 	{"rest:http://localhost:1234", Config{
-		URL: parseURL("http://localhost:1234"),
+		URL:         parseURL("http://localhost:1234"),
+		Connections: 20,
 	}},
 }
 

src/restic/backend/rest/rest.go

@@ -26,21 +26,21 @@ const connLimit = 40
 var _ restic.Backend = &restBackend{}
 
 type restBackend struct {
-	url      *url.URL
-	connChan chan struct{}
-	client   *http.Client
+	url    *url.URL
+	sem    *backend.Semaphore
+	client *http.Client
 	backend.Layout
 }
 
 // Open opens the REST backend with the given config.
 func Open(cfg Config) (restic.Backend, error) {
-	connChan := make(chan struct{}, connLimit)
-	for i := 0; i < connLimit; i++ {
-		connChan <- struct{}{}
-	}
-
 	client := &http.Client{Transport: backend.Transport()}
 
+	sem, err := backend.NewSemaphore(cfg.Connections)
+	if err != nil {
+		return nil, err
+	}
+
 	// use url without trailing slash for layout
 	url := cfg.URL.String()
 	if url[len(url)-1] == '/' {
@@ -48,10 +48,10 @@ func Open(cfg Config) (restic.Backend, error) {
 	}
 
 	be := &restBackend{
-		url:      cfg.URL,
-		connChan: connChan,
-		client:   client,
-		Layout:   &backend.RESTLayout{URL: url, Join: path.Join},
+		url:    cfg.URL,
+		client: client,
+		Layout: &backend.RESTLayout{URL: url, Join: path.Join},
+		sem:    sem,
 	}
 
 	return be, nil
@@ -114,9 +114,9 @@ func (b *restBackend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (
 	// backend.Closer, which has a noop method.
 	rd = backend.Closer{Reader: rd}
 
-	<-b.connChan
+	b.sem.GetToken()
 	resp, err := ctxhttp.Post(ctx, b.client, b.Filename(h), "binary/octet-stream", rd)
-	b.connChan <- struct{}{}
+	b.sem.ReleaseToken()
 
 	if resp != nil {
 		defer func() {
@@ -169,9 +169,9 @@ func (b *restBackend) Load(ctx context.Context, h restic.Handle, length int, off
 	req.Header.Add("Range", byteRange)
 	debug.Log("Load(%v) send range %v", h, byteRange)
 
-	<-b.connChan
+	b.sem.GetToken()
 	resp, err := ctxhttp.Do(ctx, b.client, req)
-	b.connChan <- struct{}{}
+	b.sem.ReleaseToken()
 
 	if err != nil {
 		if resp != nil {
@@ -195,9 +195,9 @@ func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInf
 		return restic.FileInfo{}, err
 	}
 
-	<-b.connChan
+	b.sem.GetToken()
 	resp, err := ctxhttp.Head(ctx, b.client, b.Filename(h))
-	b.connChan <- struct{}{}
+	b.sem.ReleaseToken()
 	if err != nil {
 		return restic.FileInfo{}, errors.Wrap(err, "client.Head")
 	}
@@ -242,9 +242,9 @@ func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error {
 	if err != nil {
 		return errors.Wrap(err, "http.NewRequest")
 	}
-	<-b.connChan
+	b.sem.GetToken()
 	resp, err := ctxhttp.Do(ctx, b.client, req)
-	b.connChan <- struct{}{}
+	b.sem.ReleaseToken()
 
 	if err != nil {
 		return errors.Wrap(err, "client.Do")
@@ -273,9 +273,9 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType) <-chan string
 		url += "/"
 	}
 
-	<-b.connChan
+	b.sem.GetToken()
 	resp, err := ctxhttp.Get(ctx, b.client, url)
-	b.connChan <- struct{}{}
+	b.sem.ReleaseToken()
 
 	if resp != nil {
 		defer func() {

src/restic/backend/rest/rest_test.go

@@ -76,9 +76,8 @@ func newTestSuite(ctx context.Context, t testing.TB) *test.Suite {
 				t.Fatal(err)
 			}
 
-			cfg := rest.Config{
-				URL: url,
-			}
+			cfg := rest.NewConfig()
+			cfg.URL = url
 			return cfg, nil
 		},
 

src/restic/backend/s3/config.go

@@ -18,6 +18,15 @@ type Config struct {
 	Bucket        string
 	Prefix        string
 	Layout        string `option:"layout" help:"use this backend layout (default: auto-detect)"`
+
+	Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 20)"`
+}
+
+// NewConfig returns a new Config with the default values filled in.
+func NewConfig() Config {
+	return Config{
+		Connections: 20,
+	}
 }
 
 func init() {
@@ -70,10 +79,10 @@ func createConfig(endpoint string, p []string, useHTTP bool) (interface{}, error
 	default:
 		prefix = path.Clean(p[1])
 	}
-	return Config{
-		Endpoint: endpoint,
-		UseHTTP:  useHTTP,
-		Bucket:   p[0],
-		Prefix:   prefix,
-	}, nil
+	cfg := NewConfig()
+	cfg.Endpoint = endpoint
+	cfg.UseHTTP = useHTTP
+	cfg.Bucket = p[0]
+	cfg.Prefix = prefix
+	return cfg, nil
 }

src/restic/backend/s3/config_test.go

@@ -7,78 +7,92 @@ var configTests = []struct {
 	cfg Config
 }{
 	{"s3://eu-central-1/bucketname", Config{
-		Endpoint: "eu-central-1",
-		Bucket:   "bucketname",
-		Prefix:   "restic",
+		Endpoint:    "eu-central-1",
+		Bucket:      "bucketname",
+		Prefix:      "restic",
+		Connections: 20,
 	}},
 	{"s3://eu-central-1/bucketname/", Config{
-		Endpoint: "eu-central-1",
-		Bucket:   "bucketname",
-		Prefix:   "restic",
+		Endpoint:    "eu-central-1",
+		Bucket:      "bucketname",
+		Prefix:      "restic",
+		Connections: 20,
 	}},
 	{"s3://eu-central-1/bucketname/prefix/directory", Config{
-		Endpoint: "eu-central-1",
-		Bucket:   "bucketname",
-		Prefix:   "prefix/directory",
+		Endpoint:    "eu-central-1",
+		Bucket:      "bucketname",
+		Prefix:      "prefix/directory",
+		Connections: 20,
 	}},
 	{"s3://eu-central-1/bucketname/prefix/directory/", Config{
-		Endpoint: "eu-central-1",
-		Bucket:   "bucketname",
-		Prefix:   "prefix/directory",
+		Endpoint:    "eu-central-1",
+		Bucket:      "bucketname",
+		Prefix:      "prefix/directory",
+		Connections: 20,
 	}},
 	{"s3:eu-central-1/foobar", Config{
-		Endpoint: "eu-central-1",
-		Bucket:   "foobar",
-		Prefix:   "restic",
+		Endpoint:    "eu-central-1",
+		Bucket:      "foobar",
+		Prefix:      "restic",
+		Connections: 20,
 	}},
 	{"s3:eu-central-1/foobar/", Config{
-		Endpoint: "eu-central-1",
-		Bucket:   "foobar",
-		Prefix:   "restic",
+		Endpoint:    "eu-central-1",
+		Bucket:      "foobar",
+		Prefix:      "restic",
+		Connections: 20,
 	}},
 	{"s3:eu-central-1/foobar/prefix/directory", Config{
-		Endpoint: "eu-central-1",
-		Bucket:   "foobar",
-		Prefix:   "prefix/directory",
+		Endpoint:    "eu-central-1",
+		Bucket:      "foobar",
+		Prefix:      "prefix/directory",
+		Connections: 20,
 	}},
 	{"s3:eu-central-1/foobar/prefix/directory/", Config{
-		Endpoint: "eu-central-1",
-		Bucket:   "foobar",
-		Prefix:   "prefix/directory",
+		Endpoint:    "eu-central-1",
+		Bucket:      "foobar",
+		Prefix:      "prefix/directory",
+		Connections: 20,
 	}},
 	{"s3:https://hostname:9999/foobar", Config{
-		Endpoint: "hostname:9999",
-		Bucket:   "foobar",
-		Prefix:   "restic",
+		Endpoint:    "hostname:9999",
+		Bucket:      "foobar",
+		Prefix:      "restic",
+		Connections: 20,
 	}},
 	{"s3:https://hostname:9999/foobar/", Config{
-		Endpoint: "hostname:9999",
-		Bucket:   "foobar",
-		Prefix:   "restic",
+		Endpoint:    "hostname:9999",
+		Bucket:      "foobar",
+		Prefix:      "restic",
+		Connections: 20,
 	}},
 	{"s3:http://hostname:9999/foobar", Config{
-		Endpoint: "hostname:9999",
-		Bucket:   "foobar",
-		Prefix:   "restic",
-		UseHTTP:  true,
+		Endpoint:    "hostname:9999",
+		Bucket:      "foobar",
+		Prefix:      "restic",
+		UseHTTP:     true,
+		Connections: 20,
 	}},
 	{"s3:http://hostname:9999/foobar/", Config{
-		Endpoint: "hostname:9999",
-		Bucket:   "foobar",
-		Prefix:   "restic",
-		UseHTTP:  true,
+		Endpoint:    "hostname:9999",
+		Bucket:      "foobar",
+		Prefix:      "restic",
+		UseHTTP:     true,
+		Connections: 20,
 	}},
 	{"s3:http://hostname:9999/bucket/prefix/directory", Config{
-		Endpoint: "hostname:9999",
-		Bucket:   "bucket",
-		Prefix:   "prefix/directory",
-		UseHTTP:  true,
+		Endpoint:    "hostname:9999",
+		Bucket:      "bucket",
+		Prefix:      "prefix/directory",
+		UseHTTP:     true,
+		Connections: 20,
 	}},
 	{"s3:http://hostname:9999/bucket/prefix/directory/", Config{
-		Endpoint: "hostname:9999",
-		Bucket:   "bucket",
-		Prefix:   "prefix/directory",
-		UseHTTP:  true,
+		Endpoint:    "hostname:9999",
+		Bucket:      "bucket",
+		Prefix:      "prefix/directory",
+		UseHTTP:     true,
+		Connections: 20,
 	}},
 }
 

src/restic/backend/s3/s3.go

@@ -24,7 +24,7 @@ const connLimit = 10
 // s3 is a backend which stores the data on an S3 endpoint.
 type s3 struct {
 	client       *minio.Client
-	connChan     chan struct{}
+	sem          *backend.Semaphore
 	bucketname   string
 	prefix       string
 	cacheMutex   sync.RWMutex
@@ -47,8 +47,14 @@ func Open(cfg Config) (restic.Backend, error) {
 		return nil, errors.Wrap(err, "minio.New")
 	}
 
+	sem, err := backend.NewSemaphore(cfg.Connections)
+	if err != nil {
+		return nil, err
+	}
+
 	be := &s3{
 		client:       client,
+		sem:          sem,
 		bucketname:   cfg.Bucket,
 		prefix:       cfg.Prefix,
 		cacheObjSize: make(map[string]int64),
@@ -63,8 +69,6 @@ func Open(cfg Config) (restic.Backend, error) {
 
 	be.Layout = l
 
-	be.createConnections()
-
 	found, err := client.BucketExists(cfg.Bucket)
 	if err != nil {
 		debug.Log("BucketExists(%v) returned err %v", cfg.Bucket, err)
@@ -82,13 +86,6 @@ func Open(cfg Config) (restic.Backend, error) {
 	return be, nil
 }
 
-func (be *s3) createConnections() {
-	be.connChan = make(chan struct{}, connLimit)
-	for i := 0; i < connLimit; i++ {
-		be.connChan <- struct{}{}
-	}
-}
-
 // IsNotExist returns true if the error is caused by a not existing file.
 func (be *s3) IsNotExist(err error) bool {
 	debug.Log("IsNotExist(%T, %#v)", err, err)
@@ -226,7 +223,7 @@ func (be *s3) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err erro
 		return errors.New("key already exists")
 	}
 
-	<-be.connChan
+	be.sem.GetToken()
 
 	// wrap the reader so that net/http client cannot close the reader, return
 	// the token instead.
@@ -238,11 +235,10 @@ func (be *s3) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err erro
 	}
 
 	debug.Log("PutObject(%v, %v)", be.bucketname, objName)
-	coreClient := minio.Core{be.client}
+	coreClient := minio.Core{Client: be.client}
 	info, err := coreClient.PutObject(be.bucketname, objName, size, rd, nil, nil, nil)
 
-	// return token
-	be.connChan <- struct{}{}
+	be.sem.ReleaseToken()
 	debug.Log("%v -> %v bytes, err %#v", objName, info.Size, err)
 
 	return errors.Wrap(err, "client.PutObject")
@@ -279,8 +275,7 @@ func (be *s3) Load(ctx context.Context, h restic.Handle, length int, offset int6
 
 	objName := be.Filename(h)
 
-	// get token for connection
-	<-be.connChan
+	be.sem.GetToken()
 
 	byteRange := fmt.Sprintf("bytes=%d-", offset)
 	if length > 0 {
@@ -290,11 +285,10 @@ func (be *s3) Load(ctx context.Context, h restic.Handle, length int, offset int6
 	headers.Add("Range", byteRange)
 	debug.Log("Load(%v) send range %v", h, byteRange)
 
-	coreClient := minio.Core{be.client}
+	coreClient := minio.Core{Client: be.client}
 	rd, _, err := coreClient.GetObject(be.bucketname, objName, headers)
 	if err != nil {
-		// return token
-		be.connChan <- struct{}{}
+		be.sem.ReleaseToken()
 		return nil, err
 	}
 
@@ -302,8 +296,7 @@ func (be *s3) Load(ctx context.Context, h restic.Handle, length int, offset int6
 		ReadCloser: rd,
 		f: func() {
 			debug.Log("Close()")
-			// return token
-			be.connChan <- struct{}{}
+			be.sem.ReleaseToken()
 		},
 	}
 

src/restic/backend/s3/s3_test.go

@@ -114,14 +114,13 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
 			key, secret := newRandomCredentials(t)
 			cfg.stopServer = runMinio(ctx, t, cfg.tempdir, key, secret)
 
-			cfg.Config = s3.Config{
-				Endpoint: "localhost:9000",
-				Bucket:   "restictestbucket",
-				Prefix:   fmt.Sprintf("test-%d", time.Now().UnixNano()),
-				UseHTTP:  true,
-				KeyID:    key,
-				Secret:   secret,
-			}
+			cfg.Config = s3.NewConfig()
+			cfg.Config.Endpoint = "localhost:9000"
+			cfg.Config.Bucket = "restictestbucket"
+			cfg.Config.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
+			cfg.Config.UseHTTP = true
+			cfg.Config.KeyID = key
+			cfg.Config.Secret = secret
 			return cfg, nil
 		},
 

src/restic/backend/semaphore.go

@@ -1,15 +1,20 @@
 package backend
 
+import "restic/errors"
+
 // Semaphore limits access to a restricted resource.
 type Semaphore struct {
 	ch chan struct{}
 }
 
 // NewSemaphore returns a new semaphore with capacity n.
-func NewSemaphore(n int) *Semaphore {
+func NewSemaphore(n uint) (*Semaphore, error) {
+	if n <= 0 {
+		return nil, errors.New("must be a positive number")
+	}
 	return &Semaphore{
 		ch: make(chan struct{}, n),
-	}
+	}, nil
 }
 
 // GetToken blocks until a Token is available.

src/restic/backend/swift/config.go

@@ -3,6 +3,7 @@ package swift
 import (
 	"os"
 	"restic/errors"
+	"restic/options"
 	"strings"
 )
 
@@ -24,6 +25,19 @@ type Config struct {
 	Container              string
 	Prefix                 string
 	DefaultContainerPolicy string
+
+	Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 20)"`
+}
+
+func init() {
+	options.Register("swift", Config{})
+}
+
+// NewConfig returns a new config with the default values filled in.
+func NewConfig() Config {
+	return Config{
+		Connections: 20,
+	}
 }
 
 // ParseConfig parses the string s and extract swift's container name and prefix.
@@ -47,10 +61,9 @@ func ParseConfig(s string) (interface{}, error) {
 	}
 	prefix = prefix[1:]
 
-	cfg := Config{
-		Container: container,
-		Prefix:    prefix,
-	}
+	cfg := NewConfig()
+	cfg.Container = container
+	cfg.Prefix = prefix
 
 	return cfg, nil
 }

src/restic/backend/swift/config_test.go

@@ -6,9 +6,28 @@ var configTests = []struct {
 	s   string
 	cfg Config
 }{
-	{"swift:cnt1:/", Config{Container: "cnt1", Prefix: ""}},
-	{"swift:cnt2:/prefix", Config{Container: "cnt2", Prefix: "prefix"}},
-	{"swift:cnt3:/prefix/longer", Config{Container: "cnt3", Prefix: "prefix/longer"}},
+	{
+		"swift:cnt1:/",
+		Config{
+			Container:   "cnt1",
+			Prefix:      "",
+			Connections: 20,
+		},
+	},
+	{
+		"swift:cnt2:/prefix",
+		Config{Container: "cnt2",
+			Prefix:      "prefix",
+			Connections: 20,
+		},
+	},
+	{
+		"swift:cnt3:/prefix/longer",
+		Config{Container: "cnt3",
+			Prefix:      "prefix/longer",
+			Connections: 20,
+		},
+	},
 }
 
 func TestParseConfig(t *testing.T) {

src/restic/backend/swift/swift.go

@@ -22,7 +22,7 @@ const connLimit = 10
 // beSwift is a backend which stores the data on a swift endpoint.
 type beSwift struct {
 	conn      *swift.Connection
-	connChan  chan struct{}
+	sem       *backend.Semaphore
 	container string // Container name
 	prefix    string // Prefix of object names in the container
 	backend.Layout
@@ -36,6 +36,11 @@ var _ restic.Backend = &beSwift{}
 func Open(cfg Config) (restic.Backend, error) {
 	debug.Log("config %#v", cfg)
 
+	sem, err := backend.NewSemaphore(cfg.Connections)
+	if err != nil {
+		return nil, err
+	}
+
 	be := &beSwift{
 		conn: &swift.Connection{
 			UserName:       cfg.UserName,
@@ -54,6 +59,7 @@ func Open(cfg Config) (restic.Backend, error) {
 
 			Transport: backend.Transport(),
 		},
+		sem:       sem,
 		container: cfg.Container,
 		prefix:    cfg.Prefix,
 		Layout: &backend.DefaultLayout{
@@ -61,7 +67,6 @@ func Open(cfg Config) (restic.Backend, error) {
 			Join: path.Join,
 		},
 	}
-	be.createConnections()
 
 	// Authenticate if needed
 	if !be.conn.Authenticated() {
@@ -98,13 +103,6 @@ func Open(cfg Config) (restic.Backend, error) {
 	return be, nil
 }
 
-func (be *beSwift) createConnections() {
-	be.connChan = make(chan struct{}, connLimit)
-	for i := 0; i < connLimit; i++ {
-		be.connChan <- struct{}{}
-	}
-}
-
 func (be *beSwift) createContainer(policy string) error {
 	var h swift.Headers
 	if policy != "" {
@@ -140,9 +138,9 @@ func (be *beSwift) Load(ctx context.Context, h restic.Handle, length int, offset
 
 	objName := be.Filename(h)
 
-	<-be.connChan
+	be.sem.GetToken()
 	defer func() {
-		be.connChan <- struct{}{}
+		be.sem.ReleaseToken()
 	}()
 
 	headers := swift.Headers{}
@@ -190,9 +188,9 @@ func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err
 		return errors.Wrap(err, "conn.Object")
 	}
 
-	<-be.connChan
+	be.sem.GetToken()
 	defer func() {
-		be.connChan <- struct{}{}
+		be.sem.ReleaseToken()
 	}()
 
 	encoding := "binary/octet-stream"