From 375a9b7940ea644bb2ead17b8bbaa4513feefe40 Mon Sep 17 00:00:00 2001 From: Aneesh N Date: Mon, 30 Jan 2023 15:42:07 -0700 Subject: [PATCH 01/25] Add smb changes (#6) * backend/smb: Add SMB backend and testcases Add new SMB storage backend for restic. Added test cases for testing SMB backend. --------- Co-authored-by: Aneesh Nireshwalia Co-authored-by: Srigovind Nayak --- .github/workflows/tests.yml | 84 +++- cmd/restic/global.go | 81 +++- go.mod | 3 + go.sum | 7 + internal/backend/location/location.go | 2 + internal/backend/smb/config.go | 96 +++++ internal/backend/smb/config_test.go | 51 +++ internal/backend/smb/conpool.go | 230 +++++++++++ internal/backend/smb/smb.go | 565 ++++++++++++++++++++++++++ internal/backend/smb/smb_test.go | 77 ++++ internal/test/vars.go | 1 + 11 files changed, 1195 insertions(+), 2 deletions(-) create mode 100644 internal/backend/smb/config.go create mode 100644 internal/backend/smb/config_test.go create mode 100644 internal/backend/smb/conpool.go create mode 100644 internal/backend/smb/smb.go create mode 100644 internal/backend/smb/smb_test.go diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9c9555543..e515d2781 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,9 +1,10 @@ name: test on: # run tests on push to master, but not when other branches are pushed to + workflow_dispatch: {} push: branches: - - master + - 'feature/smb-test-setup' # run tests for all pull requests pull_request: @@ -26,6 +27,7 @@ jobs: go: 1.19.x os: macOS-latest test_fuse: false + test_smb: false - job_name: Linux go: 1.19.x @@ -82,6 +84,65 @@ jobs: chmod 755 $HOME/bin/rclone rm -rf rclone* + echo "install samba" + user="smbuser" + pass="mGoWwqvgdnwtmh07" + + if [ "$RUNNER_OS" == "macOS" ]; then + #NONINTERACTIVE=1 brew install samba + else + sudo apt-get update + sudo apt-get install samba -y + + echo "Allow Samba in firewall" + sudo ufw allow 'Samba' + + echo "modifying samba config" + echo '' | sudo tee -a /etc/samba/smb.conf + echo ' interfaces = 127.0.0.0/8 eth0' | sudo tee -a /etc/samba/smb.conf + echo ' bind interfaces only = yes' | sudo tee -a /etc/samba/smb.conf + echo '' | sudo tee -a /etc/samba/smb.conf + echo "[$user]" | sudo tee -a /etc/samba/smb.conf + echo ' comment = Samba on Ubuntu' | sudo tee -a /etc/samba/smb.conf + echo " path = /samba/$user" | sudo tee -a /etc/samba/smb.conf + echo ' browseable = yes' | sudo tee -a /etc/samba/smb.conf + echo ' read only = no' | sudo tee -a /etc/samba/smb.conf + echo ' force create mode = 0660' | sudo tee -a /etc/samba/smb.conf + echo ' force directory mode = 2770' | sudo tee -a /etc/samba/smb.conf + echo " valid users = $user" | sudo tee -a /etc/samba/smb.conf + + echo "restart services" + sudo systemctl restart smbd + sudo systemctl restart nmbd + + echo "create samba share directory" + sudo mkdir /samba + + echo "change sambashare group" + sudo chgrp sambashare /samba + + echo "add samba user" + sudo id -u "$user" &>/dev/null || sudo useradd -M -d "/samba/$user" -s /usr/sbin/nologin -G sambashare "$user" + + echo "create samba share user directory" + sudo mkdir "/samba/$user" + + echo "change samba share user directory ownership" + sudo chown "$user":sambashare "/samba/$user" + + echo "modify permissions on samba share user directory" + sudo chmod 2770 "/samba/$user" + + echo "change smb password" + (echo "$pass"; echo "$pass") | sudo smbpasswd -a "$user" + + echo "enable samba user" + sudo smbpasswd -e "$user" + + echo "restart services" + sudo systemctl restart smbd + sudo systemctl restart nmbd + fi # add $HOME/bin to path ($GOBIN was already added to the path by setup-go@v3) echo $HOME/bin >> $GITHUB_PATH if: matrix.os == 'ubuntu-latest' || matrix.os == 'macOS-latest' @@ -121,6 +182,26 @@ jobs: Invoke-WebRequest https://github.com/restic/test-assets/raw/master/libiconv-1.8-1-bin.zip -OutFile libiconv.zip unzip libiconv.zip + # Create new smbshare + $user="smbuser" + $pass="mGoWwqvgdnwtmh07" + $SecurePassword = $pass | ConvertTo-SecureString -AsPlainText -Force + + echo "Create user" + New-LocalUser $user -Password $SecurePassword -FullName "SMB User" -Description "Account used for smb access." + + echo "Making user admin" + Add-LocalGroupMember -Group "Administrators" -Member "$user" + + $path="C:\$user" + mkdir $path + + echo "Create share" + New-SmbShare -Name $user -Path $path -FullAccess "Administrators" -EncryptData $True + + echo "Grant access to share" + Grant-SmbShareAccess -Name $user -AccountName $user -AccessRight Full -Force + # add $USERPROFILE/tar/bin to path echo $Env:USERPROFILE\tar\bin >> $Env:GITHUB_PATH if: matrix.os == 'windows-latest' @@ -135,6 +216,7 @@ jobs: - name: Run local Tests env: RESTIC_TEST_FUSE: ${{ matrix.test_fuse }} + RESTIC_TEST_SMB: ${{ matrix.test_smb }} run: | go test -cover ${{matrix.test_opts}} ./... diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 8a4edf407..5b44041f2 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -25,6 +25,7 @@ import ( "github.com/restic/restic/internal/backend/retry" "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/sftp" + "github.com/restic/restic/internal/backend/smb" "github.com/restic/restic/internal/backend/swift" "github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/debug" @@ -683,6 +684,80 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro debug.Log("opening rest repository at %#v", cfg) return cfg, nil + case "smb": + cfg := loc.Config.(smb.Config) + if err := opts.Apply(loc.Scheme, &cfg); err != nil { + return nil, err + } + if cfg.User == "" { + cfg.User = os.Getenv("RESTIC_SMB_USER") + } + + if cfg.Password.String() == "" { + cfg.Password = options.NewSecretString(os.Getenv("RESTIC_SMB_PASSWORD")) + } + + if cfg.Domain == "" { + cfg.Domain = os.Getenv("RESTIC_SMB_DOMAIN") + } + if cfg.Domain == "" { + cfg.Domain = smb.DefaultDomain + } + + //0 is an acceptable value for timeout, hence using -1 as the default unset value. + if cfg.IdleTimeout == nil { + it := os.Getenv("RESTIC_SMB_IDLETIMEOUTSECS") + if it == "" { + timeout := smb.DefaultIdleTimeout + cfg.IdleTimeout = &timeout + } else { + t, err := strconv.Atoi(it) + if err != nil { + return nil, err + } + timeout := (time.Duration(int64(t) * int64(time.Second))) + cfg.IdleTimeout = &timeout + } + } + + if cfg.Connections == 0 { + c := os.Getenv("RESTIC_SMB_CONNECTIONS") + if c == "" { + cfg.Connections = smb.DefaultConnections + } else { + con, err := strconv.Atoi(c) + if err != nil { + return nil, err + } + cfg.Connections = uint(con) + } + } + + if cfg.RequireMessageSigning == nil { + v := os.Getenv("RESTIC_SMB_REQUIRE_MESSAGESIGNING") + rms := strings.ToLower(v) == "true" + cfg.RequireMessageSigning = &rms + } + + if cfg.ClientGuid == "" { + c := os.Getenv("RESTIC_SMB_CLIENTGUID") + cfg.ClientGuid = c + } + + if cfg.Dialect == 0 { + d := os.Getenv("RESTIC_SMB_DIALECT") + if d != "" { + v, err := strconv.Atoi(d) + if err != nil { + return nil, err + } + cfg.Dialect = uint16(v) + } + } + + debug.Log("opening smb repository at %#v", cfg) + return cfg, nil + } return nil, errors.Fatalf("invalid backend: %q", loc.Scheme) @@ -717,6 +792,8 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio be, err = local.Open(ctx, cfg.(local.Config)) case "sftp": be, err = sftp.Open(ctx, cfg.(sftp.Config)) + case "smb": + be, err = smb.Open(ctx, cfg.(smb.Config)) case "s3": be, err = s3.Open(ctx, cfg.(s3.Config), rt) case "gs": @@ -748,7 +825,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio } } - if loc.Scheme == "local" || loc.Scheme == "sftp" { + if loc.Scheme == "local" || loc.Scheme == "sftp" || loc.Scheme == "smb" { // wrap the backend in a LimitBackend so that the throughput is limited be = limiter.LimitBackend(be, lim) } @@ -789,6 +866,8 @@ func create(ctx context.Context, s string, opts options.Options) (restic.Backend return local.Create(ctx, cfg.(local.Config)) case "sftp": return sftp.Create(ctx, cfg.(sftp.Config)) + case "smb": + return smb.Create(ctx, cfg.(smb.Config)) case "s3": return s3.Create(ctx, cfg.(s3.Config), rt) case "gs": diff --git a/go.mod b/go.mod index a172c6992..f6fae1dcc 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/go-ole/go-ole v1.2.6 github.com/google/go-cmp v0.5.9 github.com/hashicorp/golang-lru/v2 v2.0.1 + github.com/hirochachacha/go-smb2 v1.1.0 github.com/juju/ratelimit v1.0.2 github.com/klauspost/compress v1.15.15 github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5 @@ -24,6 +25,7 @@ require ( github.com/restic/chunker v0.4.0 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 + github.com/valyala/fastrand v1.1.0 golang.org/x/crypto v0.5.0 golang.org/x/net v0.5.0 golang.org/x/oauth2 v0.4.0 @@ -44,6 +46,7 @@ require ( github.com/dnaeon/go-vcr v1.2.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/felixge/fgprof v0.9.3 // indirect + github.com/geoffgarside/ber v1.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b // indirect diff --git a/go.sum b/go.sum index 08069a411..1a12f025c 100644 --- a/go.sum +++ b/go.sum @@ -51,6 +51,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= +github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= +github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= @@ -95,6 +97,8 @@ github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1Yu github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI= +github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -165,12 +169,15 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= +github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= +github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= diff --git a/internal/backend/location/location.go b/internal/backend/location/location.go index a732233cc..c9bbf981f 100644 --- a/internal/backend/location/location.go +++ b/internal/backend/location/location.go @@ -12,6 +12,7 @@ import ( "github.com/restic/restic/internal/backend/rest" "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/sftp" + "github.com/restic/restic/internal/backend/smb" "github.com/restic/restic/internal/backend/swift" "github.com/restic/restic/internal/errors" ) @@ -41,6 +42,7 @@ var parsers = []parser{ {"swift", swift.ParseConfig, noPassword}, {"rest", rest.ParseConfig, rest.StripPassword}, {"rclone", rclone.ParseConfig, noPassword}, + {"smb", smb.ParseConfig, noPassword}, } // noPassword returns the repository location unchanged (there's no sensitive information there) diff --git a/internal/backend/smb/config.go b/internal/backend/smb/config.go new file mode 100644 index 000000000..9d63d3b96 --- /dev/null +++ b/internal/backend/smb/config.go @@ -0,0 +1,96 @@ +package smb + +import ( + "path" + "strconv" + "strings" + "time" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/options" +) + +// Config contains all configuration necessary to connect to an SMB server +type Config struct { + Address string + Port int + ShareName string + Path string + + Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"` + Connections uint `option:"connections" help:"set a limit for the number of concurrent operations (default: 2)"` + IdleTimeout *time.Duration `option:"idle-timeout" help:"Max time in seconds before closing idle connections. If no connections have been returned to the connection pool in the time given, the connection pool will be emptied. Set to 0 to keep connections indefinitely.(default: 60)"` + RequireMessageSigning *bool `option:"require-message-signing" help:"Mandates message signing otherwise does not allow the connection. If this is false, messaging signing is just enabled and not enforced. (default: false)"` + Dialect uint16 `option:"dialect" help:"Force a specific dialect to be used. SMB311:785, SMB302:770, SMB300:768, SMB210:528, SMB202:514, SMB2:767. If unspecfied (0), following dialects are tried in order - SMB311, SMB302, SMB300, SMB210, SMB202 (default: 0)"` + ClientGuid string `option:"client-guid" help:"A 16-byte GUID to uniquely identify a client. If not specific a random GUID is used. (default: \"\")"` + + User string `option:"user"` + Password options.SecretString `option:"password"` + Domain string `option:"domain"` +} + +const ( + DefaultSmbPort int = 445 + DefaultDomain string = "WORKGROUP" + DefaultConnections uint = 2 + DefaultIdleTimeout time.Duration = 60 * time.Second +) + +// NewConfig returns a new Config with the default values filled in. +func NewConfig() Config { + return Config{ + Port: DefaultSmbPort, + } +} + +func init() { + options.Register("smb", Config{}) +} + +// ParseConfig parses the string s and extracts the s3 config. The two +// supported configuration formats are smb://address:port/sharename/directory and +// smb://address/sharename/directory in which case default port 445 is used. +// If no prefix is given the prefix "restic" will be used. +func ParseConfig(s string) (interface{}, error) { + switch { + case strings.HasPrefix(s, "smb://"): + s = s[6:] + case strings.HasPrefix(s, "smb:"): + s = s[4:] + default: + return nil, errors.New("smb: invalid format") + } + // use the first entry of the path as the endpoint and the + // remainder as bucket name and prefix + fullAddress, rest, _ := strings.Cut(s, "/") + address, portString, hasPort := strings.Cut(fullAddress, ":") + var port int + if !hasPort { + port = DefaultSmbPort + } else { + var err error + port, err = strconv.Atoi(portString) + if err != nil { + return nil, err + } + } + sharename, directory, _ := strings.Cut(rest, "/") + return createConfig(address, port, sharename, directory) +} + +func createConfig(address string, port int, sharename string, directory string) (interface{}, error) { + if address == "" { + return nil, errors.New("smb: invalid format, address not found") + } + + if directory != "" { + directory = path.Clean(directory) + } + + cfg := NewConfig() + cfg.Address = address + cfg.Port = port + cfg.ShareName = sharename + cfg.Path = directory + return cfg, nil +} diff --git a/internal/backend/smb/config_test.go b/internal/backend/smb/config_test.go new file mode 100644 index 000000000..678bb8db0 --- /dev/null +++ b/internal/backend/smb/config_test.go @@ -0,0 +1,51 @@ +package smb + +import ( + "strings" + "testing" +) + +var configTests = []struct { + s string + cfg Config +}{ + {"smb://shareaddress/sharename/directory", Config{ + Address: "shareaddress", + Port: DefaultSmbPort, + ShareName: "sharename", + Path: "directory", + }}, + {"smb://shareaddress:456/sharename/directory", Config{ + Address: "shareaddress", + Port: 456, + ShareName: "sharename", + Path: "directory", + }}, +} + +func TestParseConfig(t *testing.T) { + for i, test := range configTests { + cfg, err := ParseConfig(test.s) + if err != nil { + t.Errorf("test %d:%s failed: %v", i, test.s, err) + continue + } + + if cfg != test.cfg { + t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v", + i, test.s, test.cfg, cfg) + continue + } + } +} + +func TestParseError(t *testing.T) { + const prefix = "smb: invalid format," + + for _, s := range []string{"", "/", "//", "/sharename/directory"} { + _, err := ParseConfig("smb://" + s) + if err == nil || !strings.HasPrefix(err.Error(), prefix) { + t.Errorf("expected %q, got %q", prefix, err) + } + } +} diff --git a/internal/backend/smb/conpool.go b/internal/backend/smb/conpool.go new file mode 100644 index 000000000..9855d0557 --- /dev/null +++ b/internal/backend/smb/conpool.go @@ -0,0 +1,230 @@ +package smb + +import ( + "context" + "fmt" + "net" + "strconv" + "sync/atomic" + + "github.com/hirochachacha/go-smb2" + "github.com/restic/restic/internal/debug" +) + +// conn encapsulates a SMB client and corresponding SMB client +type conn struct { + conn *net.Conn + smbSession *smb2.Session + smbShare *smb2.Share + shareName string +} + +// Closes the connection +func (c *conn) close() (err error) { + if c.smbShare != nil { + err = c.smbShare.Umount() + } + sessionLogoffErr := c.smbSession.Logoff() + if err != nil { + return err + } + return sessionLogoffErr +} + +// True if it's closed +func (c *conn) closed() bool { + var nopErr error + if c.smbShare != nil { + // stat the current directory + _, nopErr = c.smbShare.Stat(".") + } else { + // list the shares + _, nopErr = c.smbSession.ListSharenames() + } + return nopErr == nil +} + +// Show that we are using a SMB session +// +// Call removeSession() when done +func (b *Backend) addSession() { + atomic.AddInt32(&b.sessions, 1) +} + +// Show the SMB session is no longer in use +func (b *Backend) removeSession() { + atomic.AddInt32(&b.sessions, -1) +} + +// getSessions shows whether there are any sessions in use +func (b *Backend) getSessions() int32 { + return atomic.LoadInt32(&b.sessions) +} + +// dial starts a client connection to the given SMB server. It is a +// convenience function that connects to the given network address, +// initiates the SMB handshake, and then sets up a Client. +func (b *Backend) dial(ctx context.Context, network, addr string) (*conn, error) { + dialer := net.Dialer{} + tconn, err := dialer.Dial(network, addr) + if err != nil { + return nil, err + } + var clientId [16]byte + if b.ClientGuid != "" { + copy(clientId[:], []byte(b.ClientGuid)) + } + + rms := b.RequireMessageSigning != nil + if rms { + rms = *b.RequireMessageSigning + } + d := &smb2.Dialer{ + Negotiator: smb2.Negotiator{ + RequireMessageSigning: rms, + SpecifiedDialect: b.Dialect, + ClientGuid: clientId, + }, + Initiator: &smb2.NTLMInitiator{ + User: b.User, + Password: b.Password.Unwrap(), + Domain: b.Domain, + }, + } + + session, err := d.DialContext(ctx, tconn) + if err != nil { + return nil, err + } + + return &conn{ + smbSession: session, + conn: &tconn, + }, nil +} + +// Open a new connection to the SMB server. +func (b *Backend) newConnection(share string) (c *conn, err error) { + // As we are pooling these connections we need to decouple + // them from the current context + ctx := context.Background() + + c, err = b.dial(ctx, "tcp", b.Address+":"+strconv.Itoa(b.Port)) + if err != nil { + return nil, fmt.Errorf("couldn't connect SMB: %w", err) + } + + if share != "" { + // mount the specified share as well if user requested + c.smbShare, err = c.smbSession.Mount(share) + if err != nil { + _ = c.smbSession.Logoff() + return nil, fmt.Errorf("couldn't initialize SMB: %w", err) + } + c.smbShare = c.smbShare.WithContext(ctx) + } + + return c, nil +} + +// Ensure the specified share is mounted or the session is unmounted +func (c *conn) mountShare(share string) (err error) { + if c.shareName == share { + return nil + } + if c.smbShare != nil { + err = c.smbShare.Umount() + c.smbShare = nil + } + if err != nil { + return + } + if share != "" { + c.smbShare, err = c.smbSession.Mount(share) + if err != nil { + return + } + } + c.shareName = share + return nil +} + +// Get a SMB connection from the pool, or open a new one +func (b *Backend) getConnection(ctx context.Context, share string) (c *conn, err error) { + b.poolMu.Lock() + for len(b.pool) > 0 { + c = b.pool[0] + b.pool = b.pool[1:] + err = c.mountShare(share) + if err == nil { + break + } + debug.Log("Discarding unusable SMB connection: %v", err) + c = nil + } + b.poolMu.Unlock() + if c != nil { + return c, nil + } + c, err = b.newConnection(share) + return c, err +} + +// Return a SMB connection to the pool +// +// It nils the pointed to connection out so it can't be reused +func (b *Backend) putConnection(pc **conn) { + c := *pc + *pc = nil + + var nopErr error + if c.smbShare != nil { + // stat the current directory + _, nopErr = c.smbShare.Stat(".") + } else { + // list the shares + _, nopErr = c.smbSession.ListSharenames() + } + if nopErr != nil { + debug.Log("Connection failed, closing: %v", nopErr) + _ = c.close() + return + } + + b.poolMu.Lock() + b.pool = append(b.pool, c) + if b.Config.IdleTimeout != nil && *b.Config.IdleTimeout > 0 { + b.drain.Reset(*b.Config.IdleTimeout) // nudge on the pool emptying timer + } + b.poolMu.Unlock() +} + +// Drain the pool of any connections +func (b *Backend) drainPool() (err error) { + b.poolMu.Lock() + defer b.poolMu.Unlock() + if sessions := b.getSessions(); sessions != 0 { + debug.Log("Not closing %d unused connections as %d sessions active", len(b.pool), sessions) + if b.Config.IdleTimeout != nil && *b.Config.IdleTimeout > 0 { + b.drain.Reset(*b.Config.IdleTimeout) // nudge on the pool emptying timer + } + return nil + } + if b.Config.IdleTimeout != nil && *b.Config.IdleTimeout > 0 { + b.drain.Stop() + } + if len(b.pool) != 0 { + debug.Log("Closing %d unused connections", len(b.pool)) + } + for i, c := range b.pool { + if !c.closed() { + cErr := c.close() + if cErr != nil { + err = cErr + } + } + b.pool[i] = nil + } + b.pool = nil + return err +} diff --git a/internal/backend/smb/smb.go b/internal/backend/smb/smb.go new file mode 100644 index 000000000..d1dad36de --- /dev/null +++ b/internal/backend/smb/smb.go @@ -0,0 +1,565 @@ +package smb + +import ( + "context" + "hash" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strconv" + "sync" + "syscall" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/hirochachacha/go-smb2" + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/backend/sema" + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" + + "github.com/valyala/fastrand" +) + +// Backend stores data on an SMB endpoint. +type Backend struct { + sem sema.Semaphore + Config + layout.Layout + backend.Modes + + sessions int32 + poolMu sync.Mutex + pool []*conn + drain *time.Timer // used to drain the pool when we stop using the connections +} + +// make sure that *Backend implements backend.Backend +var _ restic.Backend = &Backend{} + +const ( + defaultLayout = "default" +) + +func open(ctx context.Context, cfg Config) (*Backend, error) { + + l, err := layout.ParseLayout(ctx, &layout.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path) + if err != nil { + return nil, err + } + + sem, err := sema.New(cfg.Connections) + if err != nil { + return nil, err + } + + b := &Backend{ + Config: cfg, + sem: sem, + Layout: l, + } + + debug.Log("open, config %#v", cfg) + + // set the pool drainer timer going + if b.Config.IdleTimeout != nil && *b.Config.IdleTimeout > 0 { + b.drain = time.AfterFunc(*b.Config.IdleTimeout, func() { _ = b.drainPool() }) + } + + cn, err := b.getConnection(ctx, b.ShareName) + if err != nil { + return nil, err + } + defer b.putConnection(&cn) + + stat, err := cn.smbShare.Stat(l.Filename(restic.Handle{Type: restic.ConfigFile})) + m := backend.DeriveModesFromFileInfo(stat, err) + debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) + + b.Modes = m + + return b, nil +} + +// Open opens the local backend as specified by config. +func Open(ctx context.Context, cfg Config) (*Backend, error) { + debug.Log("open local backend at %v (layout %q)", cfg.Path, cfg.Layout) + return open(ctx, cfg) +} + +// Create creates all the necessary files and directories for a new local +// backend at dir. Afterwards a new config blob should be created. +func Create(ctx context.Context, cfg Config) (*Backend, error) { + debug.Log("create local backend at %v (layout %q)", cfg.Path, cfg.Layout) + + b, err := open(ctx, cfg) + if err != nil { + return nil, err + } + + cn, err := b.getConnection(ctx, cfg.ShareName) + if err != nil { + return b, err + } + defer b.putConnection(&cn) + + // test if config file already exists + _, err = cn.smbShare.Lstat(b.Filename(restic.Handle{Type: restic.ConfigFile})) + if err == nil { + return nil, errors.New("config file already exists") + } + + // create paths for data and refs + for _, d := range b.Paths() { + err := cn.smbShare.MkdirAll(d, b.Modes.Dir) + if err != nil { + return nil, errors.WithStack(err) + } + } + + return b, nil +} + +func (b *Backend) Connections() uint { + return b.Config.Connections +} + +// Location returns this backend's location (the directory name). +func (b *Backend) Location() string { + return b.Join(b.ShareName, b.Path) +} + +// Hasher may return a hash function for calculating a content hash for the backend +func (b *Backend) Hasher() hash.Hash { + return nil +} + +// HasAtomicReplace returns whether Save() can atomically replace files +func (b *Backend) HasAtomicReplace() bool { + return true +} + +// IsNotExist returns true if the error is caused by a non existing file. +func (b *Backend) IsNotExist(err error) bool { + return errors.Is(err, os.ErrNotExist) +} + +// Join combines path components with slashes. +func (be *Backend) Join(p ...string) string { + return path.Join(p...) +} + +// Save stores data in the backend at the handle. +func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) (err error) { + debug.Log("Save %v", h) + if err := h.Valid(); err != nil { + return backoff.Permanent(err) + } + + finalname := b.Filename(h) + dir := filepath.Dir(finalname) + + defer func() { + // Mark non-retriable errors as such + if errors.Is(err, syscall.ENOSPC) || os.IsPermission(err) { + err = backoff.Permanent(err) + } + }() + + b.sem.GetToken() + defer b.sem.ReleaseToken() + + // Create new file with a temporary name. + tmpname := filepath.Base(finalname) + "-tmp-" + + b.addSession() // Show session in use + defer b.removeSession() + + cn, err := b.getConnection(ctx, b.ShareName) + if err != nil { + return err + } + defer b.putConnection(&cn) + + f, err := b.CreateTemp(cn, dir, tmpname) + + if b.IsNotExist(err) { + debug.Log("error %v: creating dir", err) + + // error is caused by a missing directory, try to create it + mkdirErr := cn.smbShare.MkdirAll(dir, b.Modes.Dir) + if mkdirErr != nil { + debug.Log("error creating dir %v: %v", dir, mkdirErr) + } else { + // try again + f, err = b.CreateTemp(cn, dir, tmpname) + } + } + + if err != nil { + return errors.WithStack(err) + } + + defer func(f *smb2.File) { + if err != nil { + _ = f.Close() // Double Close is harmless. + // Remove after Rename is harmless: we embed the final name in the + // temporary's name and no other goroutine will get the same data to + // Save, so the temporary name should never be reused by another + // goroutine. + _ = cn.smbShare.Remove(f.Name()) + } + }(f) + + // save data, then sync + wbytes, err := io.Copy(f, rd) + if err != nil { + return errors.WithStack(err) + } + // sanity check + if wbytes != rd.Length() { + return errors.Errorf("wrote %d bytes instead of the expected %d bytes", wbytes, rd.Length()) + } + + // Ignore error if filesystem does not support fsync. + // In this case the sync call is on the smb client's file. + err = f.Sync() + syncNotSup := err != nil && (errors.Is(err, syscall.ENOTSUP)) + if err != nil && !syncNotSup { + return errors.WithStack(err) + } + + // Close, then rename. Windows doesn't like the reverse order. + if err = f.Close(); err != nil { + return errors.WithStack(err) + } + if err = cn.smbShare.Rename(f.Name(), finalname); err != nil { + return errors.WithStack(err) + } + + // try to mark file as read-only to avoid accidential modifications + // ignore if the operation fails as some filesystems don't allow the chmod call + // e.g. exfat and network file systems with certain mount options + err = cn.setFileReadonly(finalname, b.Modes.File) + if err != nil && !os.IsPermission(err) { + return errors.WithStack(err) + } + + return nil +} + +// set file to readonly +func (cn *conn) setFileReadonly(f string, mode os.FileMode) error { + return cn.smbShare.Chmod(f, mode&^0222) +} + +// Load runs fn with a reader that yields the contents of the file at h at the +// given offset. +func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + return backend.DefaultLoad(ctx, h, length, offset, b.openReader, fn) +} + +func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + debug.Log("Load %v, length %v, offset %v", h, length, offset) + if err := h.Valid(); err != nil { + return nil, backoff.Permanent(err) + } + + if offset < 0 { + return nil, errors.New("offset is negative") + } + + b.addSession() // Show session in use + defer b.removeSession() + cn, err := b.getConnection(ctx, b.ShareName) + if err != nil { + return nil, err + } + defer b.putConnection(&cn) + + b.sem.GetToken() + f, err := cn.smbShare.Open(b.Filename(h)) + if err != nil { + b.sem.ReleaseToken() + return nil, err + } + + if offset > 0 { + _, err = f.Seek(offset, 0) + if err != nil { + b.sem.ReleaseToken() + _ = f.Close() + return nil, err + } + } + + r := b.sem.ReleaseTokenOnClose(f, nil) + + if length > 0 { + return backend.LimitReadCloser(r, int64(length)), nil + } + + return r, nil +} + +// Stat returns information about a blob. +func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { + debug.Log("Stat %v", h) + if err := h.Valid(); err != nil { + return restic.FileInfo{}, backoff.Permanent(err) + } + + b.sem.GetToken() + defer b.sem.ReleaseToken() + + cn, err := b.getConnection(ctx, b.ShareName) + if err != nil { + return restic.FileInfo{}, err + } + defer b.putConnection(&cn) + + fi, err := cn.smbShare.Stat(b.Filename(h)) + if err != nil { + return restic.FileInfo{}, errors.WithStack(err) + } + + return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil +} + +// Remove removes the blob with the given name and type. +func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { + debug.Log("Remove %v", h) + fn := b.Filename(h) + + b.sem.GetToken() + defer b.sem.ReleaseToken() + + cn, err := b.getConnection(ctx, b.ShareName) + if err != nil { + return err + } + defer b.putConnection(&cn) + + // reset read-only flag + err = cn.smbShare.Chmod(fn, 0666) + if err != nil && !os.IsPermission(err) { + return errors.WithStack(err) + } + + return cn.smbShare.Remove(fn) +} + +// List runs fn for each file in the backend which has the type t. When an +// error occurs (or fn returns an error), List stops and returns it. +func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) (err error) { + debug.Log("List %v", t) + + cn, err := b.getConnection(ctx, b.ShareName) + if err != nil { + return err + } + defer b.putConnection(&cn) + + basedir, subdirs := b.Basedir(t) + if subdirs { + err = b.visitDirs(cn, ctx, basedir, fn) + } else { + err = b.visitFiles(cn, ctx, basedir, fn, false) + } + + if b.IsNotExist(err) { + debug.Log("ignoring non-existing directory") + return nil + } + + return err +} + +// The following two functions are like filepath.Walk, but visit only one or +// two levels of directory structure (including dir itself as the first level). +// Also, visitDirs assumes it sees a directory full of directories, while +// visitFiles wants a directory full or regular files. +func (b *Backend) visitDirs(cn *conn, ctx context.Context, dir string, fn func(restic.FileInfo) error) error { + d, err := cn.smbShare.Open(dir) + if err != nil { + return err + } + + sub, err := d.Readdirnames(-1) + if err != nil { + // ignore subsequent errors + _ = d.Close() + return err + } + + err = d.Close() + if err != nil { + return err + } + + for _, f := range sub { + err = b.visitFiles(cn, ctx, filepath.Join(dir, f), fn, true) + if err != nil { + return err + } + } + return ctx.Err() +} + +func (b *Backend) visitFiles(cn *conn, ctx context.Context, dir string, fn func(restic.FileInfo) error, ignoreNotADirectory bool) error { + d, err := cn.smbShare.Open(dir) + if err != nil { + return err + } + + if ignoreNotADirectory { + fi, err := d.Stat() + if err != nil || !fi.IsDir() { + // ignore subsequent errors + _ = d.Close() + return err + } + } + + sub, err := d.Readdir(-1) + if err != nil { + // ignore subsequent errors + _ = d.Close() + return err + } + + err = d.Close() + if err != nil { + return err + } + + for _, fi := range sub { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + err := fn(restic.FileInfo{ + Name: fi.Name(), + Size: fi.Size(), + }) + if err != nil { + return err + } + } + return nil +} + +// Delete removes the repository and all files. +func (b *Backend) Delete(ctx context.Context) error { + debug.Log("Delete()") + cn, err := b.getConnection(ctx, b.ShareName) + if err != nil { + return err + } + defer b.putConnection(&cn) + return cn.smbShare.RemoveAll(b.Location()) +} + +// Close closes all open files. +func (b *Backend) Close() error { + debug.Log("Close()") + err := b.drainPool() + return err +} + +var ( + ErrExist = fs.ErrExist // "file already exists" +) + +// PathError records an error and the operation and file path that caused it. +type PathError = fs.PathError + +const ( + PathSeparator = '/' // OS-specific path separator + PathListSeparator = ';' // OS-specific path list separator +) + +// CreateTemp creates a new temporary file in the directory dir, +// opens the file for reading and writing, and returns the resulting file. +// The filename is generated by taking pattern and adding a random string to the end. +// If pattern includes a "*", the random string replaces the last "*". +// If dir is the empty string, CreateTemp uses the default directory for temporary files, as returned by TempDir. +// Multiple programs or goroutines calling CreateTemp simultaneously will not choose the same file. +// The caller can use the file's Name method to find the pathname of the file. +// It is the caller's responsibility to remove the file when it is no longer needed. +func (b *Backend) CreateTemp(cn *conn, dir, pattern string) (*smb2.File, error) { + if dir == "" { + dir = os.TempDir() + } + + prefix, suffix, err := prefixAndSuffix(pattern) + if err != nil { + return nil, &PathError{Op: "createtemp", Path: pattern, Err: err} + } + prefix = joinPath(dir, prefix) + + try := 0 + for { + name := prefix + nextRandom() + suffix + f, err := cn.smbShare.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) + + if os.IsExist(err) { + if try++; try < 10000 { + continue + } + return nil, &PathError{Op: "createtemp", Path: prefix + "*" + suffix, Err: ErrExist} + } + return f, err + } +} + +var errPatternHasSeparator = errors.New("pattern contains path separator") + +// prefixAndSuffix splits pattern by the last wildcard "*", if applicable, +// returning prefix as the part before "*" and suffix as the part after "*". +func prefixAndSuffix(pattern string) (prefix, suffix string, err error) { + for i := 0; i < len(pattern); i++ { + if IsPathSeparator(pattern[i]) { + return "", "", errPatternHasSeparator + } + } + if pos := lastIndex(pattern, '*'); pos != -1 { + prefix, suffix = pattern[:pos], pattern[pos+1:] + } else { + prefix = pattern + } + return prefix, suffix, nil +} + +// LastIndexByte from the strings package. +func lastIndex(s string, sep byte) int { + for i := len(s) - 1; i >= 0; i-- { + if s[i] == sep { + return i + } + } + return -1 +} + +func nextRandom() string { + return strconv.FormatUint(uint64(fastrand.Uint32()), 10) +} + +func joinPath(dir, name string) string { + if len(dir) > 0 && IsPathSeparator(dir[len(dir)-1]) { + return dir + name + } + return dir + string(PathSeparator) + name +} + +// IsPathSeparator reports whether c is a directory separator character. +func IsPathSeparator(c uint8) bool { + // NOTE: Windows accepts / as path separator. + return c == '\\' || c == '/' +} diff --git a/internal/backend/smb/smb_test.go b/internal/backend/smb/smb_test.go new file mode 100644 index 000000000..86a32b785 --- /dev/null +++ b/internal/backend/smb/smb_test.go @@ -0,0 +1,77 @@ +package smb_test + +import ( + "context" + "testing" + + "github.com/google/uuid" + "github.com/restic/restic/internal/backend/smb" + "github.com/restic/restic/internal/backend/test" + "github.com/restic/restic/internal/options" + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" +) + +func newTestSuite(t testing.TB) *test.Suite { + return &test.Suite{ + // NewConfig returns a config for a new temporary backend that will be used in tests. + NewConfig: func() (interface{}, error) { + + cfg := smb.NewConfig() + cfg.Address = "127.0.0.1" + cfg.User = "smbuser" + cfg.ShareName = cfg.User + cfg.Path = "Repo-" + uuid.New().String() + cfg.Password = options.NewSecretString("mGoWwqvgdnwtmh07") + cfg.Connections = smb.DefaultConnections + timeout := smb.DefaultIdleTimeout + cfg.IdleTimeout = &timeout + cfg.Domain = smb.DefaultDomain + + t.Logf("create new backend at %v", cfg.Address+"/"+cfg.ShareName) + + return cfg, nil + }, + + // CreateFn is a function that creates a temporary repository for the tests. + Create: func(config interface{}) (restic.Backend, error) { + cfg := config.(smb.Config) + return smb.Create(context.TODO(), cfg) + }, + + // OpenFn is a function that opens a previously created temporary repository. + Open: func(config interface{}) (restic.Backend, error) { + cfg := config.(smb.Config) + return smb.Open(context.TODO(), cfg) + }, + + // CleanupFn removes data created during the tests. + Cleanup: func(config interface{}) error { + cfg := config.(smb.Config) + if !rtest.TestCleanupTempDirs { + t.Logf("leaving test backend dir at %v", cfg.Path) + } + + rtest.RemoveAll(t, cfg.Path) + return nil + }, + } +} + +func TestBackendSMB(t *testing.T) { + if !rtest.RunSMBTest { + t.Skip("Skipping smb tests") + } + t.Logf("run tests") + + newTestSuite(t).RunTests(t) +} + +func BenchmarkBackendSMB(t *testing.B) { + if !rtest.RunSMBTest { + t.Skip("Skipping smb tests") + } + t.Logf("run benchmarks") + + newTestSuite(t).RunBenchmarks(t) +} diff --git a/internal/test/vars.go b/internal/test/vars.go index b6b76541e..e2b2497ef 100644 --- a/internal/test/vars.go +++ b/internal/test/vars.go @@ -13,6 +13,7 @@ var ( TestTempDir = getStringVar("RESTIC_TEST_TMPDIR", "") RunIntegrationTest = getBoolVar("RESTIC_TEST_INTEGRATION", true) RunFuseTest = getBoolVar("RESTIC_TEST_FUSE", true) + RunSMBTest = getBoolVar("RESTIC_TEST_SMB", true) TestSFTPPath = getStringVar("RESTIC_TEST_SFTPPATH", "/usr/lib/ssh:/usr/lib/openssh:/usr/libexec") TestWalkerPath = getStringVar("RESTIC_TEST_PATH", ".") BenchArchiveDirectory = getStringVar("RESTIC_BENCH_DIR", ".") From 8a7006d12c805332253373c5d6ab1aab07e7bd56 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Mon, 30 Jan 2023 15:49:15 -0700 Subject: [PATCH 02/25] Update tests and run go fmt --- .github/workflows/tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e515d2781..e5c84a776 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,10 +1,9 @@ name: test on: # run tests on push to master, but not when other branches are pushed to - workflow_dispatch: {} push: branches: - - 'feature/smb-test-setup' + - master # run tests for all pull requests pull_request: From 46c3dc618a00082ac01dd36e361173372b508a32 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Mon, 30 Jan 2023 16:39:49 -0700 Subject: [PATCH 03/25] Add unrelease issue and cleanup configs Removed extra environment variables --- changelog/unreleased/issue-4185 | 7 ++++ cmd/restic/global.go | 51 ----------------------------- internal/backend/smb/config.go | 23 +++++++------ internal/backend/smb/config_test.go | 22 ++++++++----- internal/backend/smb/conpool.go | 16 +++------ internal/backend/smb/smb.go | 4 +-- internal/backend/smb/smb_test.go | 2 +- 7 files changed, 41 insertions(+), 84 deletions(-) create mode 100644 changelog/unreleased/issue-4185 diff --git a/changelog/unreleased/issue-4185 b/changelog/unreleased/issue-4185 new file mode 100644 index 000000000..dd6364e58 --- /dev/null +++ b/changelog/unreleased/issue-4185 @@ -0,0 +1,7 @@ +Enhancement: SMB backend: Add SMB backend + +Restic now supports SMB/CIFS backend. You can now add a SMB repository as `-r smb://://`. + +You can configure the SMB user name (for NTLM authentication) via the environment variable `RESTIC_SMB_USER`, SMB password via the environment variable `RESTIC_SMB_PASSWORD` and optionally SMB domain via the environment variable `RESTIC_SMB_DOMAIN`(default:'WORKGROUP'). + +https://github.com/restic/restic/issues/4185 diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 5b44041f2..73f1ddb6a 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -704,57 +704,6 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro cfg.Domain = smb.DefaultDomain } - //0 is an acceptable value for timeout, hence using -1 as the default unset value. - if cfg.IdleTimeout == nil { - it := os.Getenv("RESTIC_SMB_IDLETIMEOUTSECS") - if it == "" { - timeout := smb.DefaultIdleTimeout - cfg.IdleTimeout = &timeout - } else { - t, err := strconv.Atoi(it) - if err != nil { - return nil, err - } - timeout := (time.Duration(int64(t) * int64(time.Second))) - cfg.IdleTimeout = &timeout - } - } - - if cfg.Connections == 0 { - c := os.Getenv("RESTIC_SMB_CONNECTIONS") - if c == "" { - cfg.Connections = smb.DefaultConnections - } else { - con, err := strconv.Atoi(c) - if err != nil { - return nil, err - } - cfg.Connections = uint(con) - } - } - - if cfg.RequireMessageSigning == nil { - v := os.Getenv("RESTIC_SMB_REQUIRE_MESSAGESIGNING") - rms := strings.ToLower(v) == "true" - cfg.RequireMessageSigning = &rms - } - - if cfg.ClientGuid == "" { - c := os.Getenv("RESTIC_SMB_CLIENTGUID") - cfg.ClientGuid = c - } - - if cfg.Dialect == 0 { - d := os.Getenv("RESTIC_SMB_DIALECT") - if d != "" { - v, err := strconv.Atoi(d) - if err != nil { - return nil, err - } - cfg.Dialect = uint16(v) - } - } - debug.Log("opening smb repository at %#v", cfg) return cfg, nil diff --git a/internal/backend/smb/config.go b/internal/backend/smb/config.go index 9d63d3b96..2302f30fb 100644 --- a/internal/backend/smb/config.go +++ b/internal/backend/smb/config.go @@ -17,16 +17,16 @@ type Config struct { ShareName string Path string - Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"` - Connections uint `option:"connections" help:"set a limit for the number of concurrent operations (default: 2)"` - IdleTimeout *time.Duration `option:"idle-timeout" help:"Max time in seconds before closing idle connections. If no connections have been returned to the connection pool in the time given, the connection pool will be emptied. Set to 0 to keep connections indefinitely.(default: 60)"` - RequireMessageSigning *bool `option:"require-message-signing" help:"Mandates message signing otherwise does not allow the connection. If this is false, messaging signing is just enabled and not enforced. (default: false)"` - Dialect uint16 `option:"dialect" help:"Force a specific dialect to be used. SMB311:785, SMB302:770, SMB300:768, SMB210:528, SMB202:514, SMB2:767. If unspecfied (0), following dialects are tried in order - SMB311, SMB302, SMB300, SMB210, SMB202 (default: 0)"` - ClientGuid string `option:"client-guid" help:"A 16-byte GUID to uniquely identify a client. If not specific a random GUID is used. (default: \"\")"` + User string `option:"user" help:"specify the SMB user for NTLM authentication."` + Password options.SecretString `option:"password" help:"specify the SMB password for NTLM authentication."` + Domain string `option:"domain" help:"specify the domain for authentication."` - User string `option:"user"` - Password options.SecretString `option:"password"` - Domain string `option:"domain"` + Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"` + Connections uint `option:"connections" help:"set a limit for the number of concurrent operations (default: 2)"` + IdleTimeout time.Duration `option:"idle-timeout" help:"Max time in seconds before closing idle connections. If no connections have been returned to the connection pool in the time given, the connection pool will be emptied. Set to 0 to keep connections indefinitely.(default: 60)"` + RequireMessageSigning bool `option:"require-message-signing" help:"Mandates message signing otherwise does not allow the connection. If this is false, messaging signing is just enabled and not enforced. (default: false)"` + Dialect uint16 `option:"dialect" help:"Force a specific dialect to be used. SMB311:785, SMB302:770, SMB300:768, SMB210:528, SMB202:514, SMB2:767. If unspecfied (0), following dialects are tried in order - SMB311, SMB302, SMB300, SMB210, SMB202 (default: 0)"` + ClientGuid string `option:"client-guid" help:"A 16-byte GUID to uniquely identify a client. If not specific a random GUID is used. (default: \"\")"` } const ( @@ -39,7 +39,10 @@ const ( // NewConfig returns a new Config with the default values filled in. func NewConfig() Config { return Config{ - Port: DefaultSmbPort, + Port: DefaultSmbPort, + Domain: DefaultDomain, + IdleTimeout: DefaultIdleTimeout, + Connections: DefaultConnections, } } diff --git a/internal/backend/smb/config_test.go b/internal/backend/smb/config_test.go index 678bb8db0..a24f5b54f 100644 --- a/internal/backend/smb/config_test.go +++ b/internal/backend/smb/config_test.go @@ -10,16 +10,22 @@ var configTests = []struct { cfg Config }{ {"smb://shareaddress/sharename/directory", Config{ - Address: "shareaddress", - Port: DefaultSmbPort, - ShareName: "sharename", - Path: "directory", + Address: "shareaddress", + Port: DefaultSmbPort, + ShareName: "sharename", + Path: "directory", + Domain: DefaultDomain, + Connections: DefaultConnections, + IdleTimeout: DefaultIdleTimeout, }}, {"smb://shareaddress:456/sharename/directory", Config{ - Address: "shareaddress", - Port: 456, - ShareName: "sharename", - Path: "directory", + Address: "shareaddress", + Port: 456, + ShareName: "sharename", + Path: "directory", + Domain: DefaultDomain, + Connections: DefaultConnections, + IdleTimeout: DefaultIdleTimeout, }}, } diff --git a/internal/backend/smb/conpool.go b/internal/backend/smb/conpool.go index 9855d0557..fda2ba183 100644 --- a/internal/backend/smb/conpool.go +++ b/internal/backend/smb/conpool.go @@ -75,13 +75,9 @@ func (b *Backend) dial(ctx context.Context, network, addr string) (*conn, error) copy(clientId[:], []byte(b.ClientGuid)) } - rms := b.RequireMessageSigning != nil - if rms { - rms = *b.RequireMessageSigning - } d := &smb2.Dialer{ Negotiator: smb2.Negotiator{ - RequireMessageSigning: rms, + RequireMessageSigning: b.RequireMessageSigning, SpecifiedDialect: b.Dialect, ClientGuid: clientId, }, @@ -193,9 +189,7 @@ func (b *Backend) putConnection(pc **conn) { b.poolMu.Lock() b.pool = append(b.pool, c) - if b.Config.IdleTimeout != nil && *b.Config.IdleTimeout > 0 { - b.drain.Reset(*b.Config.IdleTimeout) // nudge on the pool emptying timer - } + b.drain.Reset(b.Config.IdleTimeout) // nudge on the pool emptying timer b.poolMu.Unlock() } @@ -205,12 +199,10 @@ func (b *Backend) drainPool() (err error) { defer b.poolMu.Unlock() if sessions := b.getSessions(); sessions != 0 { debug.Log("Not closing %d unused connections as %d sessions active", len(b.pool), sessions) - if b.Config.IdleTimeout != nil && *b.Config.IdleTimeout > 0 { - b.drain.Reset(*b.Config.IdleTimeout) // nudge on the pool emptying timer - } + b.drain.Reset(b.Config.IdleTimeout) // nudge on the pool emptying timer return nil } - if b.Config.IdleTimeout != nil && *b.Config.IdleTimeout > 0 { + if b.Config.IdleTimeout > 0 { b.drain.Stop() } if len(b.pool) != 0 { diff --git a/internal/backend/smb/smb.go b/internal/backend/smb/smb.go index d1dad36de..bbbba65e4 100644 --- a/internal/backend/smb/smb.go +++ b/internal/backend/smb/smb.go @@ -66,8 +66,8 @@ func open(ctx context.Context, cfg Config) (*Backend, error) { debug.Log("open, config %#v", cfg) // set the pool drainer timer going - if b.Config.IdleTimeout != nil && *b.Config.IdleTimeout > 0 { - b.drain = time.AfterFunc(*b.Config.IdleTimeout, func() { _ = b.drainPool() }) + if b.Config.IdleTimeout > 0 { + b.drain = time.AfterFunc(b.Config.IdleTimeout, func() { _ = b.drainPool() }) } cn, err := b.getConnection(ctx, b.ShareName) diff --git a/internal/backend/smb/smb_test.go b/internal/backend/smb/smb_test.go index 86a32b785..719ad3138 100644 --- a/internal/backend/smb/smb_test.go +++ b/internal/backend/smb/smb_test.go @@ -25,7 +25,7 @@ func newTestSuite(t testing.TB) *test.Suite { cfg.Password = options.NewSecretString("mGoWwqvgdnwtmh07") cfg.Connections = smb.DefaultConnections timeout := smb.DefaultIdleTimeout - cfg.IdleTimeout = &timeout + cfg.IdleTimeout = timeout cfg.Domain = smb.DefaultDomain t.Logf("create new backend at %v", cfg.Address+"/"+cfg.ShareName) From bc0327c88463deb4589dd490e32d103230e95963 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Mon, 30 Jan 2023 16:41:57 -0700 Subject: [PATCH 04/25] Tidy go mod --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f6fae1dcc..ae7511074 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/elithrar/simple-scrypt v1.3.0 github.com/go-ole/go-ole v1.2.6 github.com/google/go-cmp v0.5.9 + github.com/google/uuid v1.3.0 github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/hirochachacha/go-smb2 v1.1.0 github.com/juju/ratelimit v1.0.2 @@ -50,7 +51,6 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b // indirect - github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect From 5ff9f58fbb5621a095c72b84d3746461a5ae7944 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Mon, 30 Jan 2023 19:45:37 -0700 Subject: [PATCH 05/25] Add document and cleanup config Add documentation for configuring smb repository. Clean up configuration for smb. Renamed address to host. Add option to configure user in smb repo url as well. Options take highest precendence. --- cmd/restic/global.go | 6 +-- doc/030_preparing_a_new_repo.rst | 28 ++++++++++++ internal/backend/smb/config.go | 70 ++++++++++++++++++++--------- internal/backend/smb/config_test.go | 42 ++++++++++++++--- internal/backend/smb/conpool.go | 2 +- internal/backend/smb/smb_test.go | 4 +- 6 files changed, 118 insertions(+), 34 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 73f1ddb6a..76c7069ce 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -686,9 +686,6 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro return cfg, nil case "smb": cfg := loc.Config.(smb.Config) - if err := opts.Apply(loc.Scheme, &cfg); err != nil { - return nil, err - } if cfg.User == "" { cfg.User = os.Getenv("RESTIC_SMB_USER") } @@ -703,6 +700,9 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro if cfg.Domain == "" { cfg.Domain = smb.DefaultDomain } + if err := opts.Apply(loc.Scheme, &cfg); err != nil { + return nil, err + } debug.Log("opening smb repository at %#v", cfg) return cfg, nil diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index 39a3a0744..aedf3bed8 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -620,6 +620,34 @@ established. .. _other-services: +SMB/CIFS +******** + +In order to backup data to SMB/CIFS, you must specify the host (with port if not default port `445`) as the backend. +You must first setup the following environment variables with the SMB credentials and the domain if it is not the default `WORKGROUP`. + +.. code-block:: console + + $ export RESTIC_SMB_USER= + $ export RESTIC_SMB_PASSWORD= + $ export RESTIC_SMB_DOMAIN= + + +Once the server is configured, the setup of the SFTP repository can +simply be achieved by changing the URL scheme in the ``init`` command: + +.. code-block:: console + + $ restic -r smb://user@host:445/sharename/restic-repo init + enter password for new repository: + enter password again: + created restic repository c7s8ffs329 at smb://host:445/sharename/restic-repo + Please note that knowledge of your password is required to access the repository. + Losing your password means that your data is irrecoverably lost. + +Optionally, you can also pass the ``user``, ``password`` and ``domain`` as options. Configurations specified as options take highest precendence. +You can also specify other smb specific optional configurations like ``dialect``, ``client-guid``, ``require-message-signing``, ``idle-timeout`` and ``connections`` as options. + Other Services via rclone ************************* diff --git a/internal/backend/smb/config.go b/internal/backend/smb/config.go index 2302f30fb..af46e3dc3 100644 --- a/internal/backend/smb/config.go +++ b/internal/backend/smb/config.go @@ -1,6 +1,7 @@ package smb import ( + "net/url" "path" "strconv" "strings" @@ -12,7 +13,7 @@ import ( // Config contains all configuration necessary to connect to an SMB server type Config struct { - Address string + Host string Port int ShareName string Path string @@ -25,7 +26,7 @@ type Config struct { Connections uint `option:"connections" help:"set a limit for the number of concurrent operations (default: 2)"` IdleTimeout time.Duration `option:"idle-timeout" help:"Max time in seconds before closing idle connections. If no connections have been returned to the connection pool in the time given, the connection pool will be emptied. Set to 0 to keep connections indefinitely.(default: 60)"` RequireMessageSigning bool `option:"require-message-signing" help:"Mandates message signing otherwise does not allow the connection. If this is false, messaging signing is just enabled and not enforced. (default: false)"` - Dialect uint16 `option:"dialect" help:"Force a specific dialect to be used. SMB311:785, SMB302:770, SMB300:768, SMB210:528, SMB202:514, SMB2:767. If unspecfied (0), following dialects are tried in order - SMB311, SMB302, SMB300, SMB210, SMB202 (default: 0)"` + Dialect uint16 `option:"dialect" help:"Force a specific dialect to be used. For SMB311 use '785', for SMB302 use '770', for SMB300 use '768', for SMB210 use '528', for SMB202 use '514', for SMB2 use '767'. If unspecfied (0), following dialects are tried in order - SMB311, SMB302, SMB300, SMB210, SMB202 (default: 0)"` ClientGuid string `option:"client-guid" help:"A 16-byte GUID to uniquely identify a client. If not specific a random GUID is used. (default: \"\")"` } @@ -50,40 +51,64 @@ func init() { options.Register("smb", Config{}) } -// ParseConfig parses the string s and extracts the s3 config. The two -// supported configuration formats are smb://address:port/sharename/directory and -// smb://address/sharename/directory in which case default port 445 is used. -// If no prefix is given the prefix "restic" will be used. +// ParseConfig parses the string s and extracts the s3 config. The +// supported configuration format is smb://[user@]host[:port]/sharename/directory. +// User and port are optional. Default port is 445. func ParseConfig(s string) (interface{}, error) { + var repo string switch { case strings.HasPrefix(s, "smb://"): - s = s[6:] + repo = s case strings.HasPrefix(s, "smb:"): - s = s[4:] + repo = "smb://" + s[4:] default: return nil, errors.New("smb: invalid format") } - // use the first entry of the path as the endpoint and the - // remainder as bucket name and prefix - fullAddress, rest, _ := strings.Cut(s, "/") - address, portString, hasPort := strings.Cut(fullAddress, ":") - var port int - if !hasPort { - port = DefaultSmbPort + var user, host, port, dir string + + // parse the "smb://user@host/sharename/directory." url format + url, err := url.Parse(repo) + if err != nil { + return nil, errors.WithStack(err) + } + if url.User != nil { + user = url.User.Username() + //Intentionally not allowing passwords to be set in url as + //it can cause issues when passwords have special characters + //like '@' and it is not recommended to pass passwords in the url. + } + + host = url.Hostname() + if host == "" { + return nil, errors.New("smb: invalid format, host name not found") + } + port = url.Port() + dir = url.Path + if dir == "" { + return nil, errors.Errorf("smb: invalid format, sharename/directory not found") + } + + dir = dir[1:] + + var portNum int + if port == "" { + portNum = DefaultSmbPort } else { var err error - port, err = strconv.Atoi(portString) + portNum, err = strconv.Atoi(port) if err != nil { return nil, err } } - sharename, directory, _ := strings.Cut(rest, "/") - return createConfig(address, port, sharename, directory) + + sharename, directory, _ := strings.Cut(dir, "/") + + return createConfig(user, host, portNum, sharename, directory) } -func createConfig(address string, port int, sharename string, directory string) (interface{}, error) { - if address == "" { - return nil, errors.New("smb: invalid format, address not found") +func createConfig(user string, host string, port int, sharename, directory string) (interface{}, error) { + if host == "" { + return nil, errors.New("smb: invalid format, Host not found") } if directory != "" { @@ -91,7 +116,8 @@ func createConfig(address string, port int, sharename string, directory string) } cfg := NewConfig() - cfg.Address = address + cfg.User = user + cfg.Host = host cfg.Port = port cfg.ShareName = sharename cfg.Path = directory diff --git a/internal/backend/smb/config_test.go b/internal/backend/smb/config_test.go index a24f5b54f..32f02c7bd 100644 --- a/internal/backend/smb/config_test.go +++ b/internal/backend/smb/config_test.go @@ -9,21 +9,51 @@ var configTests = []struct { s string cfg Config }{ - {"smb://shareaddress/sharename/directory", Config{ - Address: "shareaddress", + {"smb://user@host/sharename/directory", Config{ + Host: "host", Port: DefaultSmbPort, + User: "user", + Domain: DefaultDomain, ShareName: "sharename", Path: "directory", - Domain: DefaultDomain, Connections: DefaultConnections, IdleTimeout: DefaultIdleTimeout, }}, - {"smb://shareaddress:456/sharename/directory", Config{ - Address: "shareaddress", + {"smb://user@host:456/sharename/directory", Config{ + Host: "host", Port: 456, + User: "user", + Domain: DefaultDomain, ShareName: "sharename", Path: "directory", + Connections: DefaultConnections, + IdleTimeout: DefaultIdleTimeout, + }}, + {"smb://host/sharename/directory", Config{ + Host: "host", + Port: DefaultSmbPort, Domain: DefaultDomain, + ShareName: "sharename", + Path: "directory", + Connections: DefaultConnections, + IdleTimeout: DefaultIdleTimeout, + }}, + {"smb://host:446/sharename/directory", Config{ + Host: "host", + Port: 446, + Domain: DefaultDomain, + ShareName: "sharename", + Path: "directory", + Connections: DefaultConnections, + IdleTimeout: DefaultIdleTimeout, + }}, + {"smb:user@host:466/sharename/directory", Config{ + Host: "host", + Port: 466, + User: "user", + Domain: DefaultDomain, + ShareName: "sharename", + Path: "directory", Connections: DefaultConnections, IdleTimeout: DefaultIdleTimeout, }}, @@ -48,7 +78,7 @@ func TestParseConfig(t *testing.T) { func TestParseError(t *testing.T) { const prefix = "smb: invalid format," - for _, s := range []string{"", "/", "//", "/sharename/directory"} { + for _, s := range []string{"", "/", "//", "host", "user@host", "user@host:445", "/sharename/directory"} { _, err := ParseConfig("smb://" + s) if err == nil || !strings.HasPrefix(err.Error(), prefix) { t.Errorf("expected %q, got %q", prefix, err) diff --git a/internal/backend/smb/conpool.go b/internal/backend/smb/conpool.go index fda2ba183..540609f7e 100644 --- a/internal/backend/smb/conpool.go +++ b/internal/backend/smb/conpool.go @@ -105,7 +105,7 @@ func (b *Backend) newConnection(share string) (c *conn, err error) { // them from the current context ctx := context.Background() - c, err = b.dial(ctx, "tcp", b.Address+":"+strconv.Itoa(b.Port)) + c, err = b.dial(ctx, "tcp", b.Host+":"+strconv.Itoa(b.Port)) if err != nil { return nil, fmt.Errorf("couldn't connect SMB: %w", err) } diff --git a/internal/backend/smb/smb_test.go b/internal/backend/smb/smb_test.go index 719ad3138..894d13c35 100644 --- a/internal/backend/smb/smb_test.go +++ b/internal/backend/smb/smb_test.go @@ -18,7 +18,7 @@ func newTestSuite(t testing.TB) *test.Suite { NewConfig: func() (interface{}, error) { cfg := smb.NewConfig() - cfg.Address = "127.0.0.1" + cfg.Host = "127.0.0.1" cfg.User = "smbuser" cfg.ShareName = cfg.User cfg.Path = "Repo-" + uuid.New().String() @@ -28,7 +28,7 @@ func newTestSuite(t testing.TB) *test.Suite { cfg.IdleTimeout = timeout cfg.Domain = smb.DefaultDomain - t.Logf("create new backend at %v", cfg.Address+"/"+cfg.ShareName) + t.Logf("create new backend at %v", cfg.Host+"/"+cfg.ShareName) return cfg, nil }, From 11d1b95f4d1b8b6395cc99e5e3e3cc584abb3787 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Mon, 30 Jan 2023 20:33:13 -0700 Subject: [PATCH 06/25] Fix test cases Fix the check for macOS which was failing after commenting brew install. --- .github/workflows/tests.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e5c84a776..d087dcbbe 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -87,9 +87,7 @@ jobs: user="smbuser" pass="mGoWwqvgdnwtmh07" - if [ "$RUNNER_OS" == "macOS" ]; then - #NONINTERACTIVE=1 brew install samba - else + if [ "$RUNNER_OS" != "macOS" ]; then sudo apt-get update sudo apt-get install samba -y From 3f1673d6aa1a4fe3d8a561598ce1ce0687cb28f8 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Mon, 30 Jan 2023 21:03:27 -0700 Subject: [PATCH 07/25] Fix lint issues --- internal/backend/smb/config.go | 10 +++++----- internal/backend/smb/conpool.go | 8 ++++---- internal/backend/smb/smb.go | 28 ++++++++++------------------ 3 files changed, 19 insertions(+), 27 deletions(-) diff --git a/internal/backend/smb/config.go b/internal/backend/smb/config.go index af46e3dc3..6762ef955 100644 --- a/internal/backend/smb/config.go +++ b/internal/backend/smb/config.go @@ -27,14 +27,14 @@ type Config struct { IdleTimeout time.Duration `option:"idle-timeout" help:"Max time in seconds before closing idle connections. If no connections have been returned to the connection pool in the time given, the connection pool will be emptied. Set to 0 to keep connections indefinitely.(default: 60)"` RequireMessageSigning bool `option:"require-message-signing" help:"Mandates message signing otherwise does not allow the connection. If this is false, messaging signing is just enabled and not enforced. (default: false)"` Dialect uint16 `option:"dialect" help:"Force a specific dialect to be used. For SMB311 use '785', for SMB302 use '770', for SMB300 use '768', for SMB210 use '528', for SMB202 use '514', for SMB2 use '767'. If unspecfied (0), following dialects are tried in order - SMB311, SMB302, SMB300, SMB210, SMB202 (default: 0)"` - ClientGuid string `option:"client-guid" help:"A 16-byte GUID to uniquely identify a client. If not specific a random GUID is used. (default: \"\")"` + ClientGUID string `option:"client-guid" help:"A 16-byte GUID to uniquely identify a client. If not specific a random GUID is used. (default: \"\")"` } const ( - DefaultSmbPort int = 445 - DefaultDomain string = "WORKGROUP" - DefaultConnections uint = 2 - DefaultIdleTimeout time.Duration = 60 * time.Second + DefaultSmbPort int = 445 // DefaultSmbPort returns the default port for SMB + DefaultDomain string = "WORKGROUP" // DefaultDomain returns the default domain for SMB + DefaultConnections uint = 2 // DefaultConnections returns the number of concurrent connections for SMB. + DefaultIdleTimeout time.Duration = 60 * time.Second // DefaultIdleTimeout returns the default max time before closing idle connections for SMB. ) // NewConfig returns a new Config with the default values filled in. diff --git a/internal/backend/smb/conpool.go b/internal/backend/smb/conpool.go index 540609f7e..b6cbbf769 100644 --- a/internal/backend/smb/conpool.go +++ b/internal/backend/smb/conpool.go @@ -70,16 +70,16 @@ func (b *Backend) dial(ctx context.Context, network, addr string) (*conn, error) if err != nil { return nil, err } - var clientId [16]byte - if b.ClientGuid != "" { - copy(clientId[:], []byte(b.ClientGuid)) + var clientID [16]byte + if b.ClientGUID != "" { + copy(clientID[:], []byte(b.ClientGUID)) } d := &smb2.Dialer{ Negotiator: smb2.Negotiator{ RequireMessageSigning: b.RequireMessageSigning, SpecifiedDialect: b.Dialect, - ClientGuid: clientId, + ClientGuid: clientID, }, Initiator: &smb2.NTLMInitiator{ User: b.User, diff --git a/internal/backend/smb/smb.go b/internal/backend/smb/smb.go index bbbba65e4..b8f8ab2a1 100644 --- a/internal/backend/smb/smb.go +++ b/internal/backend/smb/smb.go @@ -149,7 +149,7 @@ func (b *Backend) IsNotExist(err error) bool { } // Join combines path components with slashes. -func (be *Backend) Join(p ...string) string { +func (b *Backend) Join(p ...string) string { return path.Join(p...) } @@ -366,9 +366,9 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi basedir, subdirs := b.Basedir(t) if subdirs { - err = b.visitDirs(cn, ctx, basedir, fn) + err = b.visitDirs(ctx, cn, basedir, fn) } else { - err = b.visitFiles(cn, ctx, basedir, fn, false) + err = b.visitFiles(ctx, cn, basedir, fn, false) } if b.IsNotExist(err) { @@ -383,7 +383,7 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi // two levels of directory structure (including dir itself as the first level). // Also, visitDirs assumes it sees a directory full of directories, while // visitFiles wants a directory full or regular files. -func (b *Backend) visitDirs(cn *conn, ctx context.Context, dir string, fn func(restic.FileInfo) error) error { +func (b *Backend) visitDirs(ctx context.Context, cn *conn, dir string, fn func(restic.FileInfo) error) error { d, err := cn.smbShare.Open(dir) if err != nil { return err @@ -402,7 +402,7 @@ func (b *Backend) visitDirs(cn *conn, ctx context.Context, dir string, fn func(r } for _, f := range sub { - err = b.visitFiles(cn, ctx, filepath.Join(dir, f), fn, true) + err = b.visitFiles(ctx, cn, filepath.Join(dir, f), fn, true) if err != nil { return err } @@ -410,7 +410,7 @@ func (b *Backend) visitDirs(cn *conn, ctx context.Context, dir string, fn func(r return ctx.Err() } -func (b *Backend) visitFiles(cn *conn, ctx context.Context, dir string, fn func(restic.FileInfo) error, ignoreNotADirectory bool) error { +func (b *Backend) visitFiles(ctx context.Context, cn *conn, dir string, fn func(restic.FileInfo) error, ignoreNotADirectory bool) error { d, err := cn.smbShare.Open(dir) if err != nil { return err @@ -473,16 +473,8 @@ func (b *Backend) Close() error { return err } -var ( - ErrExist = fs.ErrExist // "file already exists" -) - -// PathError records an error and the operation and file path that caused it. -type PathError = fs.PathError - const ( - PathSeparator = '/' // OS-specific path separator - PathListSeparator = ';' // OS-specific path list separator + pathSeparator = '/' // Always using '/' for SMB ) // CreateTemp creates a new temporary file in the directory dir, @@ -500,7 +492,7 @@ func (b *Backend) CreateTemp(cn *conn, dir, pattern string) (*smb2.File, error) prefix, suffix, err := prefixAndSuffix(pattern) if err != nil { - return nil, &PathError{Op: "createtemp", Path: pattern, Err: err} + return nil, &fs.PathError{Op: "createtemp", Path: pattern, Err: err} } prefix = joinPath(dir, prefix) @@ -513,7 +505,7 @@ func (b *Backend) CreateTemp(cn *conn, dir, pattern string) (*smb2.File, error) if try++; try < 10000 { continue } - return nil, &PathError{Op: "createtemp", Path: prefix + "*" + suffix, Err: ErrExist} + return nil, &fs.PathError{Op: "createtemp", Path: prefix + "*" + suffix, Err: fs.ErrExist} } return f, err } @@ -555,7 +547,7 @@ func joinPath(dir, name string) string { if len(dir) > 0 && IsPathSeparator(dir[len(dir)-1]) { return dir + name } - return dir + string(PathSeparator) + name + return dir + string(pathSeparator) + name } // IsPathSeparator reports whether c is a directory separator character. From ffea194387f37266c48c27b9926581b00880f127 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Tue, 31 Jan 2023 14:56:58 -0700 Subject: [PATCH 08/25] Document env variables --- doc/040_backup.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/040_backup.rst b/doc/040_backup.rst index b9996311d..10fb73815 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -603,6 +603,10 @@ environment variables. The following lists these environment variables: GOOGLE_PROJECT_ID Project ID for Google Cloud Storage GOOGLE_APPLICATION_CREDENTIALS Application Credentials for Google Cloud Storage (e.g. $HOME/.config/gs-secret-restic-key.json) + RESTIC_SMB_USER SMB user for NTLM authentication + RESTIC_SMB_PASSWORD SMB password for NTLM authentication + RESTIC_SMB_DOMAIN DOMAIN for SMB authentication + RCLONE_BWLIMIT rclone bandwidth limit See :ref:`caching` for the rules concerning cache locations when From 59ecedacaf8d7273e374da4330d1b2f606af73e4 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Tue, 31 Jan 2023 17:35:18 -0700 Subject: [PATCH 09/25] Fix review comments for temp file creation Add rclone copyright notice for smb files. Change temp file creation code to match sftp code. Remove fastrand dependency. --- go.mod | 1 - go.sum | 2 - internal/backend/smb/conpool.go | 22 +++++- internal/backend/smb/smb.go | 129 +++++++++----------------------- 4 files changed, 58 insertions(+), 96 deletions(-) diff --git a/go.mod b/go.mod index ae7511074..76beb6846 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,6 @@ require ( github.com/restic/chunker v0.4.0 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 - github.com/valyala/fastrand v1.1.0 golang.org/x/crypto v0.5.0 golang.org/x/net v0.5.0 golang.org/x/oauth2 v0.4.0 diff --git a/go.sum b/go.sum index 1a12f025c..3561bbdf8 100644 --- a/go.sum +++ b/go.sum @@ -169,8 +169,6 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= -github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= -github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= diff --git a/internal/backend/smb/conpool.go b/internal/backend/smb/conpool.go index b6cbbf769..5ab981f7d 100644 --- a/internal/backend/smb/conpool.go +++ b/internal/backend/smb/conpool.go @@ -1,3 +1,23 @@ +// Parts of this code have been copied from Rclone (https://github.com/rclone) +// Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/ + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. package smb import ( @@ -63,7 +83,7 @@ func (b *Backend) getSessions() int32 { // dial starts a client connection to the given SMB server. It is a // convenience function that connects to the given network address, -// initiates the SMB handshake, and then sets up a Client. +// initiates the SMB handshake, and then returns a session for SMB communication. func (b *Backend) dial(ctx context.Context, network, addr string) (*conn, error) { dialer := net.Dialer{} tconn, err := dialer.Dial(network, addr) diff --git a/internal/backend/smb/smb.go b/internal/backend/smb/smb.go index b8f8ab2a1..a2a1d9f03 100644 --- a/internal/backend/smb/smb.go +++ b/internal/backend/smb/smb.go @@ -1,14 +1,34 @@ +// Parts of this code have been copied from Rclone (https://github.com/rclone) +// Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/ + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. package smb import ( "context" + "crypto/rand" + "encoding/hex" "hash" "io" - "io/fs" "os" "path" "path/filepath" - "strconv" "sync" "syscall" "time" @@ -21,8 +41,6 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" - - "github.com/valyala/fastrand" ) // Backend stores data on an SMB endpoint. @@ -160,8 +178,9 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea return backoff.Permanent(err) } - finalname := b.Filename(h) - dir := filepath.Dir(finalname) + filename := b.Filename(h) + tmpFilename := filename + "-restic-temp-" + tempSuffix() + dir := filepath.Dir(tmpFilename) defer func() { // Mark non-retriable errors as such @@ -173,9 +192,6 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea b.sem.GetToken() defer b.sem.ReleaseToken() - // Create new file with a temporary name. - tmpname := filepath.Base(finalname) + "-tmp-" - b.addSession() // Show session in use defer b.removeSession() @@ -185,7 +201,8 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea } defer b.putConnection(&cn) - f, err := b.CreateTemp(cn, dir, tmpname) + // create new file + f, err := cn.smbShare.OpenFile(tmpFilename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) if b.IsNotExist(err) { debug.Log("error %v: creating dir", err) @@ -196,7 +213,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea debug.Log("error creating dir %v: %v", dir, mkdirErr) } else { // try again - f, err = b.CreateTemp(cn, dir, tmpname) + f, err = cn.smbShare.OpenFile(tmpFilename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) } } @@ -237,14 +254,14 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea if err = f.Close(); err != nil { return errors.WithStack(err) } - if err = cn.smbShare.Rename(f.Name(), finalname); err != nil { + if err = cn.smbShare.Rename(f.Name(), filename); err != nil { return errors.WithStack(err) } // try to mark file as read-only to avoid accidential modifications // ignore if the operation fails as some filesystems don't allow the chmod call // e.g. exfat and network file systems with certain mount options - err = cn.setFileReadonly(finalname, b.Modes.File) + err = cn.setFileReadonly(filename, b.Modes.File) if err != nil && !os.IsPermission(err) { return errors.WithStack(err) } @@ -473,85 +490,13 @@ func (b *Backend) Close() error { return err } -const ( - pathSeparator = '/' // Always using '/' for SMB -) - -// CreateTemp creates a new temporary file in the directory dir, -// opens the file for reading and writing, and returns the resulting file. -// The filename is generated by taking pattern and adding a random string to the end. -// If pattern includes a "*", the random string replaces the last "*". -// If dir is the empty string, CreateTemp uses the default directory for temporary files, as returned by TempDir. -// Multiple programs or goroutines calling CreateTemp simultaneously will not choose the same file. -// The caller can use the file's Name method to find the pathname of the file. -// It is the caller's responsibility to remove the file when it is no longer needed. -func (b *Backend) CreateTemp(cn *conn, dir, pattern string) (*smb2.File, error) { - if dir == "" { - dir = os.TempDir() - } - - prefix, suffix, err := prefixAndSuffix(pattern) +// tempSuffix generates a random string suffix that should be sufficiently long +// to avoid accidental conflicts. +func tempSuffix() string { + var nonce [16]byte + _, err := rand.Read(nonce[:]) if err != nil { - return nil, &fs.PathError{Op: "createtemp", Path: pattern, Err: err} - } - prefix = joinPath(dir, prefix) - - try := 0 - for { - name := prefix + nextRandom() + suffix - f, err := cn.smbShare.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) - - if os.IsExist(err) { - if try++; try < 10000 { - continue - } - return nil, &fs.PathError{Op: "createtemp", Path: prefix + "*" + suffix, Err: fs.ErrExist} - } - return f, err + panic(err) } -} - -var errPatternHasSeparator = errors.New("pattern contains path separator") - -// prefixAndSuffix splits pattern by the last wildcard "*", if applicable, -// returning prefix as the part before "*" and suffix as the part after "*". -func prefixAndSuffix(pattern string) (prefix, suffix string, err error) { - for i := 0; i < len(pattern); i++ { - if IsPathSeparator(pattern[i]) { - return "", "", errPatternHasSeparator - } - } - if pos := lastIndex(pattern, '*'); pos != -1 { - prefix, suffix = pattern[:pos], pattern[pos+1:] - } else { - prefix = pattern - } - return prefix, suffix, nil -} - -// LastIndexByte from the strings package. -func lastIndex(s string, sep byte) int { - for i := len(s) - 1; i >= 0; i-- { - if s[i] == sep { - return i - } - } - return -1 -} - -func nextRandom() string { - return strconv.FormatUint(uint64(fastrand.Uint32()), 10) -} - -func joinPath(dir, name string) string { - if len(dir) > 0 && IsPathSeparator(dir[len(dir)-1]) { - return dir + name - } - return dir + string(pathSeparator) + name -} - -// IsPathSeparator reports whether c is a directory separator character. -func IsPathSeparator(c uint8) bool { - // NOTE: Windows accepts / as path separator. - return c == '\\' || c == '/' + return hex.EncodeToString(nonce[:]) } From aab8a5f36fe176ef45e04e11eb536511f8fbc4f6 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Tue, 31 Jan 2023 17:48:36 -0700 Subject: [PATCH 10/25] Move copyright notice below imports --- internal/backend/smb/conpool.go | 25 ++++++++++---------- internal/backend/smb/smb.go | 41 +++++++++++++++++---------------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/internal/backend/smb/conpool.go b/internal/backend/smb/conpool.go index 5ab981f7d..31979ebe2 100644 --- a/internal/backend/smb/conpool.go +++ b/internal/backend/smb/conpool.go @@ -1,3 +1,16 @@ +package smb + +import ( + "context" + "fmt" + "net" + "strconv" + "sync/atomic" + + "github.com/hirochachacha/go-smb2" + "github.com/restic/restic/internal/debug" +) + // Parts of this code have been copied from Rclone (https://github.com/rclone) // Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/ @@ -18,18 +31,6 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -package smb - -import ( - "context" - "fmt" - "net" - "strconv" - "sync/atomic" - - "github.com/hirochachacha/go-smb2" - "github.com/restic/restic/internal/debug" -) // conn encapsulates a SMB client and corresponding SMB client type conn struct { diff --git a/internal/backend/smb/smb.go b/internal/backend/smb/smb.go index a2a1d9f03..3ebe15786 100644 --- a/internal/backend/smb/smb.go +++ b/internal/backend/smb/smb.go @@ -1,23 +1,3 @@ -// Parts of this code have been copied from Rclone (https://github.com/rclone) -// Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/ - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. package smb import ( @@ -43,6 +23,27 @@ import ( "github.com/restic/restic/internal/restic" ) +// Parts of this code have been copied from Rclone (https://github.com/rclone) +// Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/ + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + // Backend stores data on an SMB endpoint. type Backend struct { sem sema.Semaphore From 46c26643a7f5702b6b32f027a6afefa20f8944e6 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Wed, 1 Feb 2023 01:34:34 -0700 Subject: [PATCH 11/25] Add SecretString support for options --- internal/options/options.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/options/options.go b/internal/options/options.go index 7490ac430..1f606b617 100644 --- a/internal/options/options.go +++ b/internal/options/options.go @@ -211,6 +211,10 @@ func (o Options) Apply(ns string, dst interface{}) error { v.Field(i).SetInt(int64(d)) + case "SecretString": + ss := NewSecretString(value) + v.Field(i).Set(reflect.ValueOf(ss)) + default: panic("type " + v.Type().Field(i).Type.Name() + " not handled") } From d062a82896226167bf5ccd4cbeca6c1677f71346 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Wed, 1 Feb 2023 01:36:05 -0700 Subject: [PATCH 12/25] Correct formatting --- internal/options/options.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/options/options.go b/internal/options/options.go index 1f606b617..36fa7e306 100644 --- a/internal/options/options.go +++ b/internal/options/options.go @@ -213,6 +213,7 @@ func (o Options) Apply(ns string, dst interface{}) error { case "SecretString": ss := NewSecretString(value) + v.Field(i).Set(reflect.ValueOf(ss)) default: From a3e9be16569f52ecd0bab35e07505afa8242d05c Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Sat, 4 Feb 2023 23:01:01 -0700 Subject: [PATCH 13/25] Increase default max SMB connections to 5 --- internal/backend/smb/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/backend/smb/config.go b/internal/backend/smb/config.go index 6762ef955..befbf43ed 100644 --- a/internal/backend/smb/config.go +++ b/internal/backend/smb/config.go @@ -33,7 +33,7 @@ type Config struct { const ( DefaultSmbPort int = 445 // DefaultSmbPort returns the default port for SMB DefaultDomain string = "WORKGROUP" // DefaultDomain returns the default domain for SMB - DefaultConnections uint = 2 // DefaultConnections returns the number of concurrent connections for SMB. + DefaultConnections uint = 5 // DefaultConnections returns the number of concurrent connections for SMB. DefaultIdleTimeout time.Duration = 60 * time.Second // DefaultIdleTimeout returns the default max time before closing idle connections for SMB. ) From c9dba2cdd70a77fdc75fcdb5180dc101ba3aa74e Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Sun, 5 Feb 2023 07:41:58 -0700 Subject: [PATCH 14/25] Use connection pointer directly in putConnection Setting *pc back to nil is too easily defeated to be useful. This is more concise and prevents pointer from getting heap-allocated. --- internal/backend/smb/conpool.go | 7 +------ internal/backend/smb/smb.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/internal/backend/smb/conpool.go b/internal/backend/smb/conpool.go index 31979ebe2..7b2578a56 100644 --- a/internal/backend/smb/conpool.go +++ b/internal/backend/smb/conpool.go @@ -188,12 +188,7 @@ func (b *Backend) getConnection(ctx context.Context, share string) (c *conn, err } // Return a SMB connection to the pool -// -// It nils the pointed to connection out so it can't be reused -func (b *Backend) putConnection(pc **conn) { - c := *pc - *pc = nil - +func (b *Backend) putConnection(c *conn) { var nopErr error if c.smbShare != nil { // stat the current directory diff --git a/internal/backend/smb/smb.go b/internal/backend/smb/smb.go index 3ebe15786..78b3ae3c4 100644 --- a/internal/backend/smb/smb.go +++ b/internal/backend/smb/smb.go @@ -93,7 +93,7 @@ func open(ctx context.Context, cfg Config) (*Backend, error) { if err != nil { return nil, err } - defer b.putConnection(&cn) + defer b.putConnection(cn) stat, err := cn.smbShare.Stat(l.Filename(restic.Handle{Type: restic.ConfigFile})) m := backend.DeriveModesFromFileInfo(stat, err) @@ -124,7 +124,7 @@ func Create(ctx context.Context, cfg Config) (*Backend, error) { if err != nil { return b, err } - defer b.putConnection(&cn) + defer b.putConnection(cn) // test if config file already exists _, err = cn.smbShare.Lstat(b.Filename(restic.Handle{Type: restic.ConfigFile})) @@ -200,7 +200,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea if err != nil { return err } - defer b.putConnection(&cn) + defer b.putConnection(cn) // create new file f, err := cn.smbShare.OpenFile(tmpFilename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) @@ -297,7 +297,7 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o if err != nil { return nil, err } - defer b.putConnection(&cn) + defer b.putConnection(cn) b.sem.GetToken() f, err := cn.smbShare.Open(b.Filename(h)) @@ -338,7 +338,7 @@ func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, e if err != nil { return restic.FileInfo{}, err } - defer b.putConnection(&cn) + defer b.putConnection(cn) fi, err := cn.smbShare.Stat(b.Filename(h)) if err != nil { @@ -360,7 +360,7 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { if err != nil { return err } - defer b.putConnection(&cn) + defer b.putConnection(cn) // reset read-only flag err = cn.smbShare.Chmod(fn, 0666) @@ -380,7 +380,7 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi if err != nil { return err } - defer b.putConnection(&cn) + defer b.putConnection(cn) basedir, subdirs := b.Basedir(t) if subdirs { @@ -480,7 +480,7 @@ func (b *Backend) Delete(ctx context.Context) error { if err != nil { return err } - defer b.putConnection(&cn) + defer b.putConnection(cn) return cn.smbShare.RemoveAll(b.Location()) } From 2882872736875ab5cd6f3da87b65636701e6e3dc Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Wed, 8 Feb 2023 15:10:51 -0700 Subject: [PATCH 15/25] Fix SMB test setup for Mac --- .github/workflows/tests.yml | 48 ++++++++++++++++++++++++++++---- internal/backend/smb/conpool.go | 2 +- internal/backend/smb/smb.go | 2 +- internal/backend/smb/smb_test.go | 6 +++- 4 files changed, 50 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d087dcbbe..dd11aacc7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,30 +21,34 @@ jobs: - job_name: Windows go: 1.19.x os: windows-latest + test_smb: true - job_name: macOS go: 1.19.x os: macOS-latest test_fuse: false - test_smb: false + test_smb: true - job_name: Linux go: 1.19.x os: ubuntu-latest test_cloud_backends: true test_fuse: true + test_smb: true check_changelog: true - job_name: Linux (race) go: 1.19.x os: ubuntu-latest test_fuse: true + test_smb: true test_opts: "-race" - job_name: Linux go: 1.18.x os: ubuntu-latest test_fuse: true + test_smb: true name: ${{ matrix.job_name }} Go ${{ matrix.go }} runs-on: ${{ matrix.os }} @@ -83,11 +87,45 @@ jobs: chmod 755 $HOME/bin/rclone rm -rf rclone* - echo "install samba" - user="smbuser" - pass="mGoWwqvgdnwtmh07" + smbuser="smbuser" + smbpass="mGoWwqvgdnwtmh07" - if [ "$RUNNER_OS" != "macOS" ]; then + if [ "$RUNNER_OS" == "macOS" ]; then + smbhome=/Users/$smbuser + echo "Get computer name" + computername=$(sudo -S scutil --get ComputerName) + echo "Create smb user" + sudo dscl . -create $smbhome + sudo dscl . -create $smbhome UserShell /bin/bash + sudo dscl . -create $smbhome RealName $smbuser + LastID=`dscl . -list /Users UniqueID | awk '{print $2}' | sort -n | tail -1` + NextID=$((LastID + 1)) + sudo dscl . -create $smbhome UniqueID $NextID + sudo dscl . -create $smbhome PrimaryGroupID 80 + sudo dscl . -create $smbhome NFSHomeDirectory $smbhome + sudo dscl . -passwd $smbhome $smbpass + sudo dscl . -append /Groups/admin GroupMembership $smbuser + echo "Make home dir" + cd /Users/ + sudo createhomedir -u $smbuser -c + sudo mkdir $smbhome/smbshare + sudo chown -R $smbuser $smbhome + sudo chmod -R 755 $smbhome + echo "Setup smb share" + sudo sharing -a $smbhome/smbshare -S smbuser -n smbuser -s 001 + echo "Enable share for os user" + sudo pwpolicy -u $smbuser -sethashtypes SMB-NT off + sudo pwpolicy -u $smbuser -enableuser + echo "Export domain" + if [[ $computername != *.local ]]; then computername=$computername".local"; fi + export RESTIC_SMB_DOMAIN=$computername + sudo pwpolicy -u $smbuser -sethashtypes SMB-NT on + sudo dscl . -passwd /Users/$smbuser $smbpass + sudo launchctl load -w /System/Library/LaunchDaemons/com.apple.smbd.plist + sudo defaults write /Library/Preferences/SystemConfiguration/com.apple.smb.server.plist EnabledServices -array disk + + else + echo "install samba" sudo apt-get update sudo apt-get install samba -y diff --git a/internal/backend/smb/conpool.go b/internal/backend/smb/conpool.go index 7b2578a56..8eddcdbe6 100644 --- a/internal/backend/smb/conpool.go +++ b/internal/backend/smb/conpool.go @@ -11,7 +11,7 @@ import ( "github.com/restic/restic/internal/debug" ) -// Parts of this code have been copied from Rclone (https://github.com/rclone) +// Parts of this code have been adapted from Rclone (https://github.com/rclone) // Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/ // Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/internal/backend/smb/smb.go b/internal/backend/smb/smb.go index 78b3ae3c4..65e22cc0c 100644 --- a/internal/backend/smb/smb.go +++ b/internal/backend/smb/smb.go @@ -23,7 +23,7 @@ import ( "github.com/restic/restic/internal/restic" ) -// Parts of this code have been copied from Rclone (https://github.com/rclone) +// Parts of this code have been adapted from Rclone (https://github.com/rclone) // Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/ // Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/internal/backend/smb/smb_test.go b/internal/backend/smb/smb_test.go index 894d13c35..c210bb51b 100644 --- a/internal/backend/smb/smb_test.go +++ b/internal/backend/smb/smb_test.go @@ -2,6 +2,7 @@ package smb_test import ( "context" + "os" "testing" "github.com/google/uuid" @@ -26,7 +27,10 @@ func newTestSuite(t testing.TB) *test.Suite { cfg.Connections = smb.DefaultConnections timeout := smb.DefaultIdleTimeout cfg.IdleTimeout = timeout - cfg.Domain = smb.DefaultDomain + domain := os.Getenv("RESTIC_SMB_DOMAIN") + if domain == "" { + cfg.Domain = smb.DefaultDomain + } t.Logf("create new backend at %v", cfg.Host+"/"+cfg.ShareName) From 418670916587a84170ed2a44a1ddd9afffc890ed Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Wed, 8 Feb 2023 15:21:49 -0700 Subject: [PATCH 16/25] Fix SMB setup for linux Corrected the variable name for smbuser --- .github/workflows/tests.yml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dd11aacc7..32de08a7e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -137,14 +137,14 @@ jobs: echo ' interfaces = 127.0.0.0/8 eth0' | sudo tee -a /etc/samba/smb.conf echo ' bind interfaces only = yes' | sudo tee -a /etc/samba/smb.conf echo '' | sudo tee -a /etc/samba/smb.conf - echo "[$user]" | sudo tee -a /etc/samba/smb.conf + echo "[$smbuser]" | sudo tee -a /etc/samba/smb.conf echo ' comment = Samba on Ubuntu' | sudo tee -a /etc/samba/smb.conf - echo " path = /samba/$user" | sudo tee -a /etc/samba/smb.conf + echo " path = /samba/$smbuser" | sudo tee -a /etc/samba/smb.conf echo ' browseable = yes' | sudo tee -a /etc/samba/smb.conf echo ' read only = no' | sudo tee -a /etc/samba/smb.conf echo ' force create mode = 0660' | sudo tee -a /etc/samba/smb.conf echo ' force directory mode = 2770' | sudo tee -a /etc/samba/smb.conf - echo " valid users = $user" | sudo tee -a /etc/samba/smb.conf + echo " valid users = $smbuser" | sudo tee -a /etc/samba/smb.conf echo "restart services" sudo systemctl restart smbd @@ -157,22 +157,22 @@ jobs: sudo chgrp sambashare /samba echo "add samba user" - sudo id -u "$user" &>/dev/null || sudo useradd -M -d "/samba/$user" -s /usr/sbin/nologin -G sambashare "$user" + sudo id -u "$smbuser" &>/dev/null || sudo useradd -M -d "/samba/$smbuser" -s /usr/sbin/nologin -G sambashare "$smbuser" echo "create samba share user directory" - sudo mkdir "/samba/$user" + sudo mkdir "/samba/$smbuser" echo "change samba share user directory ownership" - sudo chown "$user":sambashare "/samba/$user" + sudo chown "$smbuser":sambashare "/samba/$smbuser" echo "modify permissions on samba share user directory" - sudo chmod 2770 "/samba/$user" + sudo chmod 2770 "/samba/$smbuser" echo "change smb password" - (echo "$pass"; echo "$pass") | sudo smbpasswd -a "$user" + (echo "$smbpass"; echo "$smbpass") | sudo smbpasswd -a "$smbuser" echo "enable samba user" - sudo smbpasswd -e "$user" + sudo smbpasswd -e "$smbuser" echo "restart services" sudo systemctl restart smbd @@ -218,24 +218,24 @@ jobs: unzip libiconv.zip # Create new smbshare - $user="smbuser" - $pass="mGoWwqvgdnwtmh07" - $SecurePassword = $pass | ConvertTo-SecureString -AsPlainText -Force + $smbuser="smbuser" + $smbpass="mGoWwqvgdnwtmh07" + $SecurePassword = $smbpass | ConvertTo-SecureString -AsPlainText -Force echo "Create user" - New-LocalUser $user -Password $SecurePassword -FullName "SMB User" -Description "Account used for smb access." + New-LocalUser $smbuser -Password $SecurePassword -FullName "SMB User" -Description "Account used for smb access." echo "Making user admin" - Add-LocalGroupMember -Group "Administrators" -Member "$user" + Add-LocalGroupMember -Group "Administrators" -Member "$smbuser" - $path="C:\$user" + $path="C:\$smbuser" mkdir $path echo "Create share" - New-SmbShare -Name $user -Path $path -FullAccess "Administrators" -EncryptData $True + New-SmbShare -Name $smbuser -Path $path -FullAccess "Administrators" -EncryptData $True echo "Grant access to share" - Grant-SmbShareAccess -Name $user -AccountName $user -AccessRight Full -Force + Grant-SmbShareAccess -Name $smbuser -AccountName $smbuser -AccessRight Full -Force # add $USERPROFILE/tar/bin to path echo $Env:USERPROFILE\tar\bin >> $Env:GITHUB_PATH From 4b0f27a0b8de147792183fcab0fe3eaba15ee873 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Wed, 15 Feb 2023 14:20:47 -0700 Subject: [PATCH 17/25] Correct help for default smb Connections to 5 --- internal/backend/smb/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/backend/smb/config.go b/internal/backend/smb/config.go index befbf43ed..d6f368b98 100644 --- a/internal/backend/smb/config.go +++ b/internal/backend/smb/config.go @@ -23,7 +23,7 @@ type Config struct { Domain string `option:"domain" help:"specify the domain for authentication."` Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"` - Connections uint `option:"connections" help:"set a limit for the number of concurrent operations (default: 2)"` + Connections uint `option:"connections" help:"set a limit for the number of concurrent operations (default: 5)"` IdleTimeout time.Duration `option:"idle-timeout" help:"Max time in seconds before closing idle connections. If no connections have been returned to the connection pool in the time given, the connection pool will be emptied. Set to 0 to keep connections indefinitely.(default: 60)"` RequireMessageSigning bool `option:"require-message-signing" help:"Mandates message signing otherwise does not allow the connection. If this is false, messaging signing is just enabled and not enforced. (default: false)"` Dialect uint16 `option:"dialect" help:"Force a specific dialect to be used. For SMB311 use '785', for SMB302 use '770', for SMB300 use '768', for SMB210 use '528', for SMB202 use '514', for SMB2 use '767'. If unspecfied (0), following dialects are tried in order - SMB311, SMB302, SMB300, SMB210, SMB202 (default: 0)"` From db9d46c5e5ae0a719d7cf1ca004e4cadb91e3fd6 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Wed, 15 Mar 2023 20:55:35 -0600 Subject: [PATCH 18/25] Enable smb test for go 1.19 --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d8344c5d6..49982aa8a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,6 +48,7 @@ jobs: go: 1.19.x os: ubuntu-latest test_fuse: true + test_smb: true - job_name: Linux go: 1.18.x From aad1cafe970d2d69664ac346f53c2e6a68f7656c Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Wed, 7 Jun 2023 16:13:47 -0600 Subject: [PATCH 19/25] fix linter warnings. Correct method definition for unused param ctx --- internal/backend/smb/conpool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/backend/smb/conpool.go b/internal/backend/smb/conpool.go index 8eddcdbe6..14379cc0c 100644 --- a/internal/backend/smb/conpool.go +++ b/internal/backend/smb/conpool.go @@ -167,7 +167,7 @@ func (c *conn) mountShare(share string) (err error) { } // Get a SMB connection from the pool, or open a new one -func (b *Backend) getConnection(ctx context.Context, share string) (c *conn, err error) { +func (b *Backend) getConnection(_ context.Context, share string) (c *conn, err error) { b.poolMu.Lock() for len(b.pool) > 0 { c = b.pool[0] From 53a62d452c54ce875108b9d7f70ba77a7ea97f4f Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia Date: Mon, 10 Jul 2023 15:23:00 -0600 Subject: [PATCH 20/25] fix lint - unused param prefix now used for smb --- internal/backend/smb/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/backend/smb/config.go b/internal/backend/smb/config.go index c673a5524..0d133c2ba 100644 --- a/internal/backend/smb/config.go +++ b/internal/backend/smb/config.go @@ -128,13 +128,13 @@ func createConfig(user string, host string, port int, sharename, directory strin // ApplyEnvironment saves values from the environment to the config. func (cfg *Config) ApplyEnvironment(prefix string) error { if cfg.User == "" { - cfg.User = os.Getenv("RESTIC_SMB_USER") + cfg.User = os.Getenv(prefix + "RESTIC_SMB_USER") } if cfg.Password.String() == "" { - cfg.Password = options.NewSecretString(os.Getenv("RESTIC_SMB_PASSWORD")) + cfg.Password = options.NewSecretString(os.Getenv(prefix + "RESTIC_SMB_PASSWORD")) } if cfg.Domain == "" { - cfg.Domain = os.Getenv("RESTIC_SMB_DOMAIN") + cfg.Domain = os.Getenv(prefix + "RESTIC_SMB_DOMAIN") } if cfg.Domain == "" { cfg.Domain = DefaultDomain From 2ff957cc05db184527e14f1f37bbdfd30b53d604 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia <99904+aneesh-n@users.noreply.github.com> Date: Tue, 9 Jan 2024 16:15:50 -0700 Subject: [PATCH 21/25] Fix lint issue in go.mod --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e6be8bdae..5c75bee3d 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect github.com/google/s2a-go v0.1.7 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect From b0199616d8890d76455e08e8a0962ed79f64c692 Mon Sep 17 00:00:00 2001 From: Aneesh N <99904+aneesh-n@users.noreply.github.com> Date: Wed, 28 Feb 2024 17:11:18 -0700 Subject: [PATCH 22/25] Update go.mod --- go.mod | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d4237b750..d0edf8a53 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.5.0 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/hirochachacha/go-smb2 v1.1.0 + github.com/hirochachacha/go-smb2 v1.1.0 github.com/klauspost/compress v1.17.6 github.com/minio/minio-go/v7 v7.0.66 github.com/minio/sha256-simd v1.0.1 @@ -48,10 +48,11 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/fgprof v0.9.3 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/geoffgarside/ber v1.1.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/golang-jwt/jwt/v5 v5.0.0 // indirect + github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect From d556942afb25b1196bbdd9b4b59388439058eb44 Mon Sep 17 00:00:00 2001 From: Aneesh N <99904+aneesh-n@users.noreply.github.com> Date: Wed, 28 Feb 2024 17:13:00 -0700 Subject: [PATCH 23/25] Update go.mod --- go.mod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.mod b/go.mod index d0edf8a53..fc9fbd99b 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,8 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= + github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/felixge/fgprof v0.9.3 // indirect github.com/geoffgarside/ber v1.1.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect From ff75c9de4a5184c86f7afea1d4c1e858e3944331 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia <99904+aneesh-n@users.noreply.github.com> Date: Thu, 29 Feb 2024 14:41:19 -0700 Subject: [PATCH 24/25] Fix go.mod --- go.mod | 8 +++----- go.sum | 2 ++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index fc9fbd99b..719716ca6 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.5.0 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/hirochachacha/go-smb2 v1.1.0 + github.com/hirochachacha/go-smb2 v1.1.0 github.com/klauspost/compress v1.17.6 github.com/minio/minio-go/v7 v7.0.66 github.com/minio/sha256-simd v1.0.1 @@ -47,14 +47,12 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= - github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/felixge/fgprof v0.9.3 // indirect - github.com/geoffgarside/ber v1.1.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/geoffgarside/ber v1.1.0 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/golang-jwt/jwt/v5 v5.0.0 // indirect + github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect diff --git a/go.sum b/go.sum index 8059c3369..fb6240df4 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,8 @@ github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= +github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= From ada8d09441d5b834541d6df75f7b73c374519da8 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia <99904+aneesh-n@users.noreply.github.com> Date: Tue, 12 Mar 2024 14:48:50 -0600 Subject: [PATCH 25/25] Fix formatting in go mod --- go.mod | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 7be0f7463..5bcfeb98c 100644 --- a/go.mod +++ b/go.mod @@ -13,9 +13,9 @@ require ( github.com/go-ole/go-ole v1.3.0 github.com/google/go-cmp v0.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/hirochachacha/go-smb2 v1.1.0 + github.com/hirochachacha/go-smb2 v1.1.0 github.com/klauspost/compress v1.17.7 - github.com/minio/minio-go/v7 v7.0.66 + github.com/minio/minio-go/v7 v7.0.66 github.com/minio/sha256-simd v1.0.1 github.com/ncw/swift/v2 v2.0.2 github.com/pkg/errors v0.9.1 @@ -48,16 +48,16 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/fgprof v0.9.3 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/geoffgarside/ber v1.1.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect - github.com/go-logr/stdr v1.2.2 // indirect + github.com/geoffgarside/ber v1.1.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect github.com/google/s2a-go v0.1.7 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect