From 86b5d8ffaac04301a28e23348460fe8b13616368 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 3 Jan 2020 19:37:43 +0000 Subject: [PATCH] s3: add bucket-lookup parameter to select path or dns style bucket lookup This is to enable restic working with Alibaba cloud Fixes #2528 --- changelog/unreleased/issue-2528 | 20 ++++++++++++++++ doc/030_preparing_a_new_repo.rst | 40 ++++++++++++++++++++++++++++++++ internal/backend/s3/config.go | 7 +++--- internal/backend/s3/s3.go | 19 +++++++++++++-- 4 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 changelog/unreleased/issue-2528 diff --git a/changelog/unreleased/issue-2528 b/changelog/unreleased/issue-2528 new file mode 100644 index 000000000..680e03840 --- /dev/null +++ b/changelog/unreleased/issue-2528 @@ -0,0 +1,20 @@ +Enhancement: support Alibaba/Aliyun OSS with S3 backend + +We've added a new flag to the s3 backend `s3.bucket-lookup` which can +be set to these 3 values: + +- `auto` - existing behaviour +- `dns` - use DNS style bucket access +- `path` - use path style bucket access + +To make the s3 backend work with Alibaba/Aliyun OSS you must set +`s3.bucket-lookup` to `dns` and set the `s3.region` parameter. For +example: + + restic -o s3.bucket-lookup=dns -o s3.region=oss-eu-west-1 -r s3:https://oss-eu-west-1.aliyuncs.com/bucketname init + +Note that s3.region must be set otherwise the minio SDK tries to look +it up and it seems that Alibaba doesn't support that properly. + +https://github.com/restic/restic/issues/2528 +https://github.com/restic/restic/pull/2535 diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index d7907645a..f470cf383 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -299,6 +299,46 @@ this command. Please note that knowledge of your password is required to access the repository. Losing your password means that your data is irrecoverably lost. +Alibaba Cloud (Aliyun) Object Storage System (OSS) +************************************************** + +`Alibaba OSS `__ is an +encrypted, secure, cost-effective, and easy-to-use object storage +service that enables you to store, back up, and archive large amounts +of data in the cloud. + +Alibaba OSS is S3 compatible so it can be used as a storage provider +for a restic repository with a couple of extra parameters. + +- Determine the correct `Alibaba OSS region endpoint `__ - this will be something like ``oss-eu-west-1.aliyuncs.com`` +- You'll need the region name too - this will be something like ``oss-eu-west-1`` + +You must first setup the following environment variables with the +credentials of your Alibaba OSS account. + +.. code-block:: console + + $ export AWS_ACCESS_KEY_ID= + $ export AWS_SECRET_ACCESS_KEY= + +Now you can easily initialize restic to use Alibaba OSS as a backend with +this command. + +.. code-block:: console + + $ ./restic -o s3.bucket-lookup=dns -o s3.region= -r s3:https:/// init + enter password for new backend: + enter password again: + created restic backend xxxxxxxxxx at s3:https:/// + Please note that knowledge of your password is required to access + the repository. Losing your password means that your data is irrecoverably lost. + +For example with an actual endpoint: + +.. code-block:: console + + $ restic -o s3.bucket-lookup=dns -o s3.region=oss-eu-west-1 -r s3:https://oss-eu-west-1.aliyuncs.com/bucketname init + OpenStack Swift *************** diff --git a/internal/backend/s3/config.go b/internal/backend/s3/config.go index 499e05094..93de42152 100644 --- a/internal/backend/s3/config.go +++ b/internal/backend/s3/config.go @@ -20,9 +20,10 @@ type Config struct { Layout string `option:"layout" help:"use this backend layout (default: auto-detect)"` StorageClass string `option:"storage-class" help:"set S3 storage class (STANDARD, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or REDUCED_REDUNDANCY)"` - Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"` - MaxRetries uint `option:"retries" help:"set the number of retries attempted"` - Region string `option:"region" help:"set region"` + Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"` + MaxRetries uint `option:"retries" help:"set the number of retries attempted"` + Region string `option:"region" help:"set region"` + BucketLookup string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'."` } // NewConfig returns a new Config with the default values filled in. diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index d1243bfd5..3c40f7c49 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -2,6 +2,7 @@ package s3 import ( "context" + "fmt" "io" "io/ioutil" "net/http" @@ -66,12 +67,26 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro }, }, }) - client, err := minio.New(cfg.Endpoint, &minio.Options{ + + options := &minio.Options{ Creds: creds, Secure: !cfg.UseHTTP, Region: cfg.Region, Transport: rt, - }) + } + + switch strings.ToLower(cfg.BucketLookup) { + case "", "auto": + options.BucketLookup = minio.BucketLookupAuto + case "dns": + options.BucketLookup = minio.BucketLookupDNS + case "path": + options.BucketLookup = minio.BucketLookupPath + default: + return nil, fmt.Errorf(`bad bucket-lookup style %q must be "auto", "path" or "dns"`, cfg.BucketLookup) + } + + client, err := minio.New(cfg.Endpoint, options) if err != nil { return nil, errors.Wrap(err, "minio.New") }