package restic import ( "encoding/json" "fmt" "io" "os" "path/filepath" "strings" "sync" "github.com/restic/restic/backend" "github.com/restic/restic/debug" ) type Cache struct { base string } func NewCache(be backend.IDer) (c *Cache, err error) { // try to get explicit cache dir from environment dir := os.Getenv("RESTIC_CACHE") // otherwise try OS specific default if dir == "" { dir, err = GetCacheDir() if err != nil { return nil, err } } basedir := filepath.Join(dir, be.ID().String()) debug.Log("Cache.New", "opened cache at %v", basedir) return &Cache{base: basedir}, nil } func (c *Cache) Has(t backend.Type, subtype string, id backend.ID) (bool, error) { // try to open file filename, err := c.filename(t, subtype, id) if err != nil { return false, err } fd, err := os.Open(filename) defer fd.Close() if err != nil { if os.IsNotExist(err) { debug.Log("Cache.Has", "test for file %v: not cached", filename) return false, nil } debug.Log("Cache.Has", "test for file %v: error %v", filename, err) return false, err } debug.Log("Cache.Has", "test for file %v: is cached", filename) return true, nil } func (c *Cache) Store(t backend.Type, subtype string, id backend.ID) (io.WriteCloser, error) { filename, err := c.filename(t, subtype, id) if err != nil { return nil, err } dirname := filepath.Dir(filename) err = os.MkdirAll(dirname, 0700) if err != nil { return nil, err } file, err := os.Create(filename) if err != nil { debug.Log("Cache.Store", "error creating file %v: %v", filename, err) return nil, err } debug.Log("Cache.Store", "created file %v", filename) return file, nil } func (c *Cache) Load(t backend.Type, subtype string, id backend.ID) (io.ReadCloser, error) { // try to open file filename, err := c.filename(t, subtype, id) if err != nil { return nil, err } return os.Open(filename) } func (c *Cache) Purge(t backend.Type, subtype string, id backend.ID) error { filename, err := c.filename(t, subtype, id) if err != nil { return err } err = os.Remove(filename) debug.Log("Cache.Purge", "Remove file %v: %v", filename, err) if err != nil && os.IsNotExist(err) { return nil } return err } func (c *Cache) Clear(s backend.Backend) error { list, err := c.List(backend.Snapshot) if err != nil { return err } for _, entry := range list { debug.Log("Cache.Clear", "found entry %v", entry) if ok, err := s.Test(backend.Snapshot, entry.ID); !ok || err != nil { debug.Log("Cache.Clear", "snapshot %v doesn't exist any more, removing %v", entry.ID, entry) err = c.Purge(backend.Snapshot, entry.Subtype, entry.ID) if err != nil { return err } } } return nil } type CacheEntry struct { ID backend.ID Subtype string } func (c CacheEntry) String() string { if c.Subtype != "" { return c.ID.Str() + "." + c.Subtype } return c.ID.Str() } func (c *Cache) List(t backend.Type) ([]CacheEntry, error) { var dir string switch t { case backend.Snapshot: dir = filepath.Join(c.base, "snapshots") case backend.Tree: dir = filepath.Join(c.base, "trees") default: return nil, fmt.Errorf("cache not supported for type %v", t) } fd, err := os.Open(dir) if err != nil { if os.IsNotExist(err) { return []CacheEntry{}, nil } return nil, err } defer fd.Close() fis, err := fd.Readdir(-1) if err != nil { return nil, err } entries := make([]CacheEntry, 0, len(fis)) for _, fi := range fis { parts := strings.SplitN(fi.Name(), ".", 2) id, err := backend.ParseID(parts[0]) // ignore invalid cache entries for now if err != nil { continue } e := CacheEntry{ID: id} if len(parts) == 2 { e.Subtype = parts[1] } entries = append(entries, e) } return entries, nil } // Construct file name for given Type. func (c *Cache) filename(t backend.Type, subtype string, id backend.ID) (string, error) { filename := id.String() if subtype != "" { filename += "." + subtype } switch t { case backend.Snapshot: return filepath.Join(c.base, "snapshots", filename), nil case backend.Tree: return filepath.Join(c.base, "trees", filename), nil } return "", fmt.Errorf("cache not supported for type %v", t) } // high-level functions // RefreshSnapshots loads the maps for all snapshots and saves them to the // local cache. Old cache entries are purged. func (c *Cache) RefreshSnapshots(s Server, p *Progress) error { defer p.Done() // list cache entries entries, err := c.List(backend.Snapshot) if err != nil { return err } // list snapshots first snapshots, err := s.List(backend.Snapshot) if err != nil { return err } // check that snapshot blobs are cached for _, id := range snapshots { // remove snapshot from list of entries for i, e := range entries { if e.ID.Equal(id) { entries = append(entries[:i], entries[i+1:]...) break } } has, err := c.Has(backend.Snapshot, "blobs", id) if err != nil { return err } if has { continue } // else start progress reporting p.Start() // build new cache _, err = cacheSnapshotBlobs(p, s, c, id) if err != nil { debug.Log("Cache.RefreshSnapshots", "unable to cache snapshot blobs for %v: %v", id.Str(), err) return err } } // remove other entries for _, e := range entries { debug.Log("Cache.RefreshSnapshots", "remove entry %v", e) err = c.Purge(backend.Snapshot, e.Subtype, e.ID) if err != nil { return err } } return nil } // cacheSnapshotBlobs creates a cache of all the blobs used within the // snapshot. It collects all blobs from all trees and saves the resulting map // to the cache and returns the map. func cacheSnapshotBlobs(p *Progress, s Server, c *Cache, id backend.ID) (*Map, error) { debug.Log("CacheSnapshotBlobs", "create cache for snapshot %v", id.Str()) sn, err := LoadSnapshot(s, id) if err != nil { debug.Log("CacheSnapshotBlobs", "unable to load snapshot %v: %v", id.Str(), err) return nil, err } m := NewMap() // add top-level node m.Insert(sn.Tree) p.Report(Stat{Trees: 1}) // start walker var wg sync.WaitGroup ch := make(chan WalkTreeJob) wg.Add(1) go func() { WalkTree(s, sn.Tree.Storage, nil, ch) wg.Done() }() for i := 0; i < maxConcurrencyPreload; i++ { wg.Add(1) go func() { for job := range ch { if job.Tree == nil { continue } p.Report(Stat{Trees: 1}) debug.Log("CacheSnapshotBlobs", "got job %v", job) m.Merge(job.Tree.Map) } wg.Done() }() } wg.Wait() // save blob list for snapshot return m, c.StoreMap(id, m) } func (c *Cache) StoreMap(snid backend.ID, m *Map) error { wr, err := c.Store(backend.Snapshot, "blobs", snid) if err != nil { return nil } defer wr.Close() enc := json.NewEncoder(wr) err = enc.Encode(m) if err != nil { return err } return nil } func (c *Cache) LoadMap(s Server, snid backend.ID) (*Map, error) { rd, err := c.Load(backend.Snapshot, "blobs", snid) if err != nil { return nil, err } m := &Map{} err = json.NewDecoder(rd).Decode(m) return m, err }