diff --git a/cmd/khepri/cmd_backup.go b/cmd/khepri/cmd_backup.go index 1b4d80988..5a3c2f277 100644 --- a/cmd/khepri/cmd_backup.go +++ b/cmd/khepri/cmd_backup.go @@ -1,117 +1,13 @@ package main import ( - "crypto/sha256" "errors" "fmt" - "io" "log" - "os" - "path/filepath" "github.com/fd0/khepri" ) -func hash(filename string) (khepri.ID, error) { - h := sha256.New() - f, err := os.Open(filename) - if err != nil { - return nil, err - } - - io.Copy(h, f) - return h.Sum([]byte{}), nil -} - -func store_file(repo *khepri.Repository, path string) (khepri.ID, error) { - obj, idch, err := repo.Create(khepri.TYPE_BLOB) - if err != nil { - return nil, err - } - - file, err := os.Open(path) - defer func() { - file.Close() - }() - - _, err = io.Copy(obj, file) - if err != nil { - return nil, err - } - - err = obj.Close() - if err != nil { - return nil, err - } - - return <-idch, nil -} - -func archive_dir(repo *khepri.Repository, path string) (khepri.ID, error) { - log.Printf("archiving dir %q", path) - - dir, err := os.Open(path) - if err != nil { - log.Printf("open(%q): %v\n", path, err) - return nil, err - } - - entries, err := dir.Readdir(-1) - if err != nil { - log.Printf("readdir(%q): %v\n", path, err) - return nil, err - } - - // use nil ID for empty directories - if len(entries) == 0 { - return nil, nil - } - - t := khepri.NewTree() - for _, e := range entries { - node := khepri.NodeFromFileInfo(e) - - var id khepri.ID - var err error - - if e.IsDir() { - id, err = archive_dir(repo, filepath.Join(path, e.Name())) - } else { - id, err = store_file(repo, filepath.Join(path, e.Name())) - } - - node.Content = id - - t.Nodes = append(t.Nodes, node) - - if err != nil { - log.Printf(" error storing %q: %v\n", e.Name(), err) - continue - } - } - - log.Printf(" dir %q: %v entries", path, len(t.Nodes)) - - obj, idch, err := repo.Create(khepri.TYPE_BLOB) - - if err != nil { - log.Printf("error creating object for tree: %v", err) - return nil, err - } - - err = t.Save(obj) - if err != nil { - log.Printf("error saving tree to repo: %v", err) - } - - obj.Close() - - id := <-idch - log.Printf("tree for %q saved at %s", path, id) - - return id, nil -} - func commandBackup(repo *khepri.Repository, args []string) error { if len(args) != 1 { return errors.New("usage: backup dir") @@ -119,13 +15,18 @@ func commandBackup(repo *khepri.Repository, args []string) error { target := args[0] - id, err := archive_dir(repo, target) + tree, err := khepri.NewTreeFromPath(repo, target) + if err != nil { + return err + } + + id, err := tree.Save(repo) if err != nil { return err } sn := khepri.NewSnapshot(target) - sn.TreeID = id + sn.Content = id snid, err := sn.Save(repo) if err != nil { diff --git a/cmd/khepri/cmd_dump.go b/cmd/khepri/cmd_dump.go new file mode 100644 index 000000000..d0fb09371 --- /dev/null +++ b/cmd/khepri/cmd_dump.go @@ -0,0 +1,90 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "log" + "os" + + "github.com/fd0/khepri" +) + +func dump_tree(repo *khepri.Repository, id khepri.ID) error { + tree, err := khepri.NewTreeFromRepo(repo, id) + if err != nil { + return err + } + + buf, err := json.MarshalIndent(tree, "", " ") + if err != nil { + return err + } + + fmt.Printf("tree %s\n%s\n", id, buf) + + for _, node := range tree.Nodes { + if node.Type == "dir" { + err = dump_tree(repo, node.Subtree) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + } + } + } + + return nil +} + +func dump_snapshot(repo *khepri.Repository, id khepri.ID) error { + sn, err := khepri.LoadSnapshot(repo, id) + if err != nil { + log.Fatalf("error loading snapshot %s", id) + } + + buf, err := json.MarshalIndent(sn, "", " ") + if err != nil { + return err + } + + fmt.Printf("%s\n%s\n", sn, buf) + + return dump_tree(repo, sn.Content) +} + +func dump_file(repo *khepri.Repository, id khepri.ID) error { + rd, err := repo.Get(khepri.TYPE_BLOB, id) + if err != nil { + return err + } + + io.Copy(os.Stdout, rd) + + return nil +} + +func commandDump(repo *khepri.Repository, args []string) error { + if len(args) != 2 { + return errors.New("usage: dump [snapshot|tree|file] ID") + } + + tpe := args[0] + + id, err := khepri.ParseID(args[1]) + if err != nil { + errx(1, "invalid id %q: %v", args[0], err) + } + + switch tpe { + case "snapshot": + return dump_snapshot(repo, id) + case "tree": + return dump_tree(repo, id) + case "file": + return dump_file(repo, id) + default: + return fmt.Errorf("invalid type %q", tpe) + } + + return nil +} diff --git a/cmd/khepri/cmd_fsck.go b/cmd/khepri/cmd_fsck.go index f85a380c5..8b5d71326 100644 --- a/cmd/khepri/cmd_fsck.go +++ b/cmd/khepri/cmd_fsck.go @@ -41,7 +41,7 @@ func fsck_snapshot(repo *khepri.Repository, id khepri.ID) (bool, error) { return false, err } - return fsck_tree(repo, sn.TreeID) + return fsck_tree(repo, sn.Content) } func commandFsck(repo *khepri.Repository, args []string) error { diff --git a/cmd/khepri/cmd_restore.go b/cmd/khepri/cmd_restore.go index 0f90ab470..db7b6088b 100644 --- a/cmd/khepri/cmd_restore.go +++ b/cmd/khepri/cmd_restore.go @@ -2,44 +2,95 @@ package main import ( "errors" + "fmt" "io" "log" "os" - "path" + "path/filepath" + "syscall" "github.com/fd0/khepri" ) -func restore_file(repo *khepri.Repository, node khepri.Node, target string) error { - log.Printf(" restore file %q\n", target) +func restore_file(repo *khepri.Repository, node *khepri.Node, path string) (err error) { + switch node.Type { + case "file": + // TODO: handle hard links + rd, err := repo.Get(khepri.TYPE_BLOB, node.Content) + if err != nil { + return err + } - rd, err := repo.Get(khepri.TYPE_BLOB, node.Content) + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600) + defer f.Close() + if err != nil { + return err + } + + _, err = io.Copy(f, rd) + if err != nil { + return err + } + + case "symlink": + err = os.Symlink(node.LinkTarget, path) + if err != nil { + return err + } + + err = os.Lchown(path, int(node.UID), int(node.GID)) + if err != nil { + return err + } + + f, err := os.OpenFile(path, khepri.O_PATH|syscall.O_NOFOLLOW, 0600) + defer f.Close() + if err != nil { + return err + } + + var utimes = []syscall.Timeval{ + syscall.NsecToTimeval(node.AccessTime.UnixNano()), + syscall.NsecToTimeval(node.ModTime.UnixNano()), + } + err = syscall.Futimes(int(f.Fd()), utimes) + if err != nil { + return err + } + + return nil + case "dev": + err = syscall.Mknod(path, syscall.S_IFBLK|0600, int(node.Device)) + if err != nil { + return err + } + case "chardev": + err = syscall.Mknod(path, syscall.S_IFCHR|0600, int(node.Device)) + if err != nil { + return err + } + case "fifo": + err = syscall.Mkfifo(path, 0600) + if err != nil { + return err + } + case "socket": + // nothing to do, we do not restore sockets + default: + return fmt.Errorf("filetype %q not implemented!\n", node.Type) + } + + err = os.Chmod(path, node.Mode) if err != nil { return err } - f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY, 0600) - defer f.Close() + err = os.Chown(path, int(node.UID), int(node.GID)) if err != nil { return err } - _, err = io.Copy(f, rd) - if err != nil { - return err - } - - err = f.Chmod(node.Mode) - if err != nil { - return err - } - - err = f.Chown(int(node.User), int(node.Group)) - if err != nil { - return err - } - - err = os.Chtimes(target, node.AccessTime, node.ModTime) + err = os.Chtimes(path, node.AccessTime, node.ModTime) if err != nil { return err } @@ -47,61 +98,48 @@ func restore_file(repo *khepri.Repository, node khepri.Node, target string) erro return nil } -func restore_dir(repo *khepri.Repository, id khepri.ID, target string) error { - log.Printf(" restore dir %q\n", target) - rd, err := repo.Get(khepri.TYPE_BLOB, id) - if err != nil { - return err - } +func restore_subtree(repo *khepri.Repository, tree *khepri.Tree, path string) { + fmt.Printf("restore_subtree(%s)\n", path) - t := khepri.NewTree() - err = t.Restore(rd) - if err != nil { - return err - } + for _, node := range tree.Nodes { + nodepath := filepath.Join(path, node.Name) + // fmt.Printf("%s:%s\n", node.Type, nodepath) - for _, node := range t.Nodes { - name := path.Base(node.Name) - if name == "." || name == ".." { - return errors.New("invalid path") - } - - nodepath := path.Join(target, name) - if node.Mode.IsDir() { - err = os.Mkdir(nodepath, 0700) + if node.Type == "dir" { + err := os.Mkdir(nodepath, 0700) if err != nil { - return err + fmt.Fprintf(os.Stderr, "%s\n", err) + continue } err = os.Chmod(nodepath, node.Mode) if err != nil { - return err + fmt.Fprintf(os.Stderr, "%s\n", err) + continue } - err = os.Chown(nodepath, int(node.User), int(node.Group)) + err = os.Chown(nodepath, int(node.UID), int(node.GID)) if err != nil { - return err + fmt.Fprintf(os.Stderr, "%s\n", err) + continue } - err = restore_dir(repo, node.Content, nodepath) - if err != nil { - return err - } + restore_subtree(repo, node.Tree, filepath.Join(path, node.Name)) err = os.Chtimes(nodepath, node.AccessTime, node.ModTime) if err != nil { - return err + fmt.Fprintf(os.Stderr, "%s\n", err) + continue } } else { - err = restore_file(repo, node, nodepath) + err := restore_file(repo, node, nodepath) if err != nil { - return err + fmt.Fprintf(os.Stderr, "%s\n", err) + continue } } } - - return nil } func commandRestore(repo *khepri.Repository, args []string) error { @@ -126,11 +164,13 @@ func commandRestore(repo *khepri.Repository, args []string) error { log.Fatalf("error loading snapshot %s", id) } - err = restore_dir(repo, sn.TreeID, target) + tree, err := khepri.NewTreeFromRepo(repo, sn.Content) if err != nil { - return err + log.Fatalf("error loading tree %s", sn.Content) } + restore_subtree(repo, tree, target) + log.Printf("%q restored to %q\n", id, target) return nil diff --git a/cmd/khepri/main.go b/cmd/khepri/main.go index 50479cca4..5219456a8 100644 --- a/cmd/khepri/main.go +++ b/cmd/khepri/main.go @@ -34,6 +34,7 @@ func init() { commands["list"] = commandList commands["snapshots"] = commandSnapshots commands["fsck"] = commandFsck + commands["dump"] = commandDump } func main() { diff --git a/cmd/stat/stat.go b/cmd/stat/stat.go new file mode 100644 index 000000000..5f3671e14 --- /dev/null +++ b/cmd/stat/stat.go @@ -0,0 +1,37 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/fd0/khepri" +) + +func main() { + if len(os.Args) == 1 { + fmt.Printf("usage: %s [file] [file] [...]\n", os.Args[0]) + os.Exit(1) + } + + for _, path := range os.Args[1:] { + fmt.Printf("lstat %s\n", path) + + fi, err := os.Lstat(path) + if err != nil { + fmt.Fprintf(os.Stderr, "%v", err) + continue + } + + node, err := khepri.NodeFromFileInfo(path, fi) + if err != nil { + fmt.Printf("err: %v\n", err) + } + + buf, err := json.MarshalIndent(node, "", " ") + if err != nil { + panic(err) + } + fmt.Printf("%s\n", string(buf)) + } +} diff --git a/cmd/tree_serialise/main.go b/cmd/tree_serialise/main.go new file mode 100644 index 000000000..220f62f38 --- /dev/null +++ b/cmd/tree_serialise/main.go @@ -0,0 +1,222 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +func check(err error) { + if err != nil { + panic(err) + } +} + +// References content within a repository. +type ID []byte + +func (id ID) String() string { + return hex.EncodeToString(id) +} + +func (id ID) MarshalJSON() ([]byte, error) { + return json.Marshal(id.String()) +} + +func (id *ID) UnmarshalJSON(b []byte) error { + var s string + err := json.Unmarshal(b, &s) + if err != nil { + return err + } + + *id = make([]byte, len(s)/2) + _, err = hex.Decode(*id, []byte(s)) + if err != nil { + return err + } + + return nil +} + +// ParseID converts the given string to an ID. +func ParseID(s string) ID { + b, err := hex.DecodeString(s) + + if err != nil { + panic(err) + } + + return ID(b) +} + +type Repository interface { + Store([]byte) ID + Get(ID) []byte +} + +type Repo map[string][]byte + +func (r Repo) Store(buf []byte) ID { + hash := sha256.New() + _, err := hash.Write(buf) + check(err) + + id := ID(hash.Sum([]byte{})) + r[id.String()] = buf + + return id +} + +func (r Repo) Get(id ID) []byte { + buf, ok := r[id.String()] + if !ok { + panic("no such id") + } + + return buf +} + +func (r Repo) Dump(wr io.Writer) { + for k, v := range r { + _, err := wr.Write([]byte(k)) + check(err) + _, err = wr.Write([]byte(":")) + check(err) + _, err = wr.Write(v) + check(err) + _, err = wr.Write([]byte("\n")) + check(err) + } +} + +type Tree struct { + Nodes []*Node `json:"nodes,omitempty"` +} + +type Node struct { + Name string `json:"name"` + Tree *Tree `json:"tree,omitempty"` + Subtree ID `json:"subtree,omitempty"` + Content ID `json:"content,omitempty"` +} + +func (tree Tree) Save(repo Repository) ID { + // fmt.Printf("nodes: %#v\n", tree.Nodes) + for _, node := range tree.Nodes { + if node.Tree != nil { + node.Subtree = node.Tree.Save(repo) + node.Tree = nil + } + } + + buf, err := json.Marshal(tree) + check(err) + + return repo.Store(buf) +} + +func (tree Tree) PP(wr io.Writer) { + tree.pp(0, wr) +} + +func (tree Tree) pp(indent int, wr io.Writer) { + for _, node := range tree.Nodes { + if node.Tree != nil { + fmt.Printf("%s%s/\n", strings.Repeat(" ", indent), node.Name) + node.Tree.pp(indent+1, wr) + } else { + fmt.Printf("%s%s [%s]\n", strings.Repeat(" ", indent), node.Name, node.Content) + } + + } +} + +func create_tree(path string) *Tree { + dir, err := os.Open(path) + check(err) + + entries, err := dir.Readdir(-1) + check(err) + + tree := &Tree{ + Nodes: make([]*Node, 0, len(entries)), + } + + for _, entry := range entries { + node := &Node{} + node.Name = entry.Name() + + if !entry.Mode().IsDir() && entry.Mode()&os.ModeType != 0 { + fmt.Fprintf(os.Stderr, "skipping %q\n", filepath.Join(path, entry.Name())) + continue + } + + tree.Nodes = append(tree.Nodes, node) + + if entry.IsDir() { + node.Tree = create_tree(filepath.Join(path, entry.Name())) + continue + } + + file, err := os.Open(filepath.Join(path, entry.Name())) + defer file.Close() + check(err) + + hash := sha256.New() + io.Copy(hash, file) + + node.Content = hash.Sum([]byte{}) + } + + return tree +} + +func load_tree(repo Repository, id ID) *Tree { + tree := &Tree{} + + buf := repo.Get(id) + json.Unmarshal(buf, tree) + + for _, node := range tree.Nodes { + if node.Subtree != nil { + node.Tree = load_tree(repo, node.Subtree) + node.Subtree = nil + } + } + + return tree +} + +func main() { + repo := make(Repo) + + tree := create_tree(os.Args[1]) + // encoder := json.NewEncoder(os.Stdout) + // fmt.Println("---------------------------") + // encoder.Encode(tree) + // fmt.Println("---------------------------") + + id := tree.Save(repo) + + // for k, v := range repo { + // fmt.Printf("%s: %s\n", k, v) + // } + + // fmt.Println("---------------------------") + + tree2 := load_tree(repo, id) + tree2.PP(os.Stdout) + // encoder.Encode(tree2) + + // dumpfile, err := os.Create("dump") + // defer dumpfile.Close() + // check(err) + + // repo.Dump(dumpfile) +} diff --git a/cmd/tree_test/khepri-repo/blobs/44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a b/cmd/tree_test/khepri-repo/blobs/44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/cmd/tree_test/khepri-repo/blobs/44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/cmd/tree_test/khepri-repo/blobs/655ee6b9ff7fb7324f1e8566eca49c67d251459787765f9c5c6af513e3db0c72 b/cmd/tree_test/khepri-repo/blobs/655ee6b9ff7fb7324f1e8566eca49c67d251459787765f9c5c6af513e3db0c72 new file mode 100644 index 000000000..7a7b553a5 --- /dev/null +++ b/cmd/tree_test/khepri-repo/blobs/655ee6b9ff7fb7324f1e8566eca49c67d251459787765f9c5c6af513e3db0c72 @@ -0,0 +1 @@ +{"nodes":[{"name":"blobs","type":"dir","mode":2147484096,"mtime":"2014-08-11T19:51:11.894770622+02:00","atime":"2014-08-11T19:51:11.894770622+02:00","ctime":"2014-08-11T19:51:11.894770622+02:00","uid":1000,"gid":100,"user":"fd0","inode":428349318,"subtree":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"},{"name":"refs","type":"dir","mode":2147484096,"mtime":"2014-08-11T19:51:11.894770622+02:00","atime":"2014-08-11T19:51:11.894770622+02:00","ctime":"2014-08-11T19:51:11.894770622+02:00","uid":1000,"gid":100,"user":"fd0","inode":4883656,"subtree":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"},{"name":"tmp","type":"dir","mode":2147484096,"mtime":"2014-08-11T19:51:11.894770622+02:00","atime":"2014-08-11T19:51:11.894770622+02:00","ctime":"2014-08-11T19:51:11.894770622+02:00","uid":1000,"gid":100,"user":"fd0","inode":169971890,"subtree":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"}]} \ No newline at end of file diff --git a/cmd/tree_test/khepri-repo/blobs/9e3528071fbfda7702010f2b711174c17df3156b77c14015ce633c9722c0e146 b/cmd/tree_test/khepri-repo/blobs/9e3528071fbfda7702010f2b711174c17df3156b77c14015ce633c9722c0e146 new file mode 100644 index 000000000..6c2868f21 --- /dev/null +++ b/cmd/tree_test/khepri-repo/blobs/9e3528071fbfda7702010f2b711174c17df3156b77c14015ce633c9722c0e146 @@ -0,0 +1 @@ +{"nodes":[{"name":"main.go","type":"file","mode":420,"mtime":"2014-08-10T23:18:15.815999516+02:00","atime":"2014-08-10T23:18:15.815999516+02:00","ctime":"2014-08-10T23:18:15.819332884+02:00","uid":1000,"gid":100,"user":"fd0","inode":193352413,"size":1181,"links":1,"content":"c45d3975908245296eb5730cf4bbbe9d8c39b0736658b5df9beada0739d40569"},{"name":"khepri-repo","type":"dir","mode":2147484096,"mtime":"2014-08-11T19:51:11.894770622+02:00","atime":"2014-08-11T19:51:11.894770622+02:00","ctime":"2014-08-11T19:51:11.894770622+02:00","uid":1000,"gid":100,"user":"fd0","inode":275443328,"subtree":"655ee6b9ff7fb7324f1e8566eca49c67d251459787765f9c5c6af513e3db0c72"}]} \ No newline at end of file diff --git a/cmd/tree_test/main.go b/cmd/tree_test/main.go new file mode 100644 index 000000000..fdf36408b --- /dev/null +++ b/cmd/tree_test/main.go @@ -0,0 +1,69 @@ +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/fd0/khepri" +) + +func check(err error) { + if err == nil { + return + } + + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) +} + +func save(repo *khepri.Repository, path string) { + tree, err := khepri.NewTreeFromPath(repo, path) + + check(err) + + id, err := tree.Save(repo) + + fmt.Printf("saved tree as %s\n", id) +} + +func restore(repo *khepri.Repository, idstr string) { + id, err := khepri.ParseID(idstr) + check(err) + + tree, err := khepri.NewTreeFromRepo(repo, id) + check(err) + + walk(0, tree) +} + +func walk(indent int, tree *khepri.Tree) { + for _, node := range tree.Nodes { + if node.Type == "dir" { + fmt.Printf("%s%s:%s/\n", strings.Repeat(" ", indent), node.Type, node.Name) + walk(indent+1, node.Tree) + } else { + fmt.Printf("%s%s:%s\n", strings.Repeat(" ", indent), node.Type, node.Name) + } + } +} + +func main() { + if len(os.Args) != 3 { + fmt.Fprintf(os.Stderr, "usage: %s [save|restore] DIR\n", os.Args[0]) + os.Exit(1) + } + + command := os.Args[1] + arg := os.Args[2] + + repo, err := khepri.NewRepository("khepri-repo") + check(err) + + switch command { + case "save": + save(repo, arg) + case "restore": + restore(repo, arg) + } +} diff --git a/repository.go b/repository.go index 68e1065cb..4309b7e07 100644 --- a/repository.go +++ b/repository.go @@ -31,9 +31,11 @@ func (n Name) Encode() string { return url.QueryEscape(string(n)) } +type HashFunc func() hash.Hash + type Repository struct { path string - hash func() hash.Hash + hash HashFunc } type Type int @@ -99,6 +101,11 @@ func (r *Repository) create() error { return nil } +// SetHash changes the hash function used for deriving IDs. Default is SHA256. +func (r *Repository) SetHash(h HashFunc) { + r.hash = h +} + // Path returns the directory used for this repository. func (r *Repository) Path() string { return r.path diff --git a/snapshot.go b/snapshot.go index c35bdcda7..d6da7b099 100644 --- a/snapshot.go +++ b/snapshot.go @@ -2,6 +2,7 @@ package khepri import ( "encoding/json" + "fmt" "os" "os/user" "time" @@ -9,12 +10,14 @@ import ( type Snapshot struct { Time time.Time `json:"time"` - TreeID ID `json:"tree"` + Content ID `json:"content"` + Tree *Tree `json:"-"` Dir string `json:"dir"` Hostname string `json:"hostname,omitempty"` Username string `json:"username,omitempty"` UID string `json:"uid,omitempty"` GID string `json:"gid,omitempty"` + id ID `json:omit` } func NewSnapshot(dir string) *Snapshot { @@ -39,7 +42,7 @@ func NewSnapshot(dir string) *Snapshot { } func (sn *Snapshot) Save(repo *Repository) (ID, error) { - if sn.TreeID == nil { + if sn.Content == nil { panic("Snapshot.Save() called with nil tree id") } @@ -59,7 +62,9 @@ func (sn *Snapshot) Save(repo *Repository) (ID, error) { return nil, err } - return <-id_ch, nil + sn.id = <-id_ch + + return sn.id, nil } func LoadSnapshot(repo *Repository, id ID) (*Snapshot, error) { @@ -78,5 +83,15 @@ func LoadSnapshot(repo *Repository, id ID) (*Snapshot, error) { return nil, err } + sn.id = id + return sn, nil } + +func (sn *Snapshot) ID() ID { + return sn.id +} + +func (sn *Snapshot) String() string { + return fmt.Sprintf("", sn.Dir, sn.Time.Format(time.RFC822Z)) +} diff --git a/snapshot_test.go b/snapshot_test.go index c294312d7..622407377 100644 --- a/snapshot_test.go +++ b/snapshot_test.go @@ -17,7 +17,7 @@ func TestSnapshot(t *testing.T) { }() sn := khepri.NewSnapshot("/home/foobar") - sn.TreeID, err = khepri.ParseID("c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2") + sn.Content, err = khepri.ParseID("c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2") ok(t, err) sn.Time, err = time.Parse(time.RFC3339Nano, "2014-08-03T17:49:05.378595539+02:00") ok(t, err) diff --git a/test/fake-data.tar.gz b/test/fake-data.tar.gz index 526b65a64..58502ab4f 100644 Binary files a/test/fake-data.tar.gz and b/test/fake-data.tar.gz differ diff --git a/tree.go b/tree.go index f2f3c72a4..4e90e32c1 100644 --- a/tree.go +++ b/tree.go @@ -2,54 +2,238 @@ package khepri import ( "encoding/json" + "fmt" "io" "os" + "os/user" + "path/filepath" + "strconv" "syscall" "time" ) type Tree struct { - Nodes []Node `json:"nodes"` + Nodes []*Node `json:"nodes,omitempty"` } type Node struct { Name string `json:"name"` - Mode os.FileMode `json:"mode"` - ModTime time.Time `json:"mtime"` - AccessTime time.Time `json:"atime"` - User uint32 `json:"user"` - Group uint32 `json:"group"` + Type string `json:"type"` + Mode os.FileMode `json:"mode,omitempty"` + ModTime time.Time `json:"mtime,omitempty"` + AccessTime time.Time `json:"atime,omitempty"` + ChangeTime time.Time `json:"ctime,omitempty"` + UID uint32 `json:"uid"` + GID uint32 `json:"gid"` + User string `json:"user,omitempty"` + Group string `json:"group,omitempty"` + Inode uint64 `json:"inode,omitempty"` + Size uint64 `json:"size,omitempty"` + Links uint64 `json:"links,omitempty"` + LinkTarget string `json:"linktarget,omitempty"` + Device uint64 `json:"device,omitempty"` Content ID `json:"content,omitempty"` + Subtree ID `json:"subtree,omitempty"` + Tree *Tree `json:"-"` } func NewTree() *Tree { return &Tree{ - Nodes: []Node{}, + Nodes: []*Node{}, } } -func (t *Tree) Restore(r io.Reader) error { - dec := json.NewDecoder(r) - return dec.Decode(t) +func NewTreeFromPath(repo *Repository, dir string) (*Tree, error) { + fd, err := os.Open(dir) + defer fd.Close() + if err != nil { + return nil, err + } + + entries, err := fd.Readdir(-1) + if err != nil { + return nil, err + } + + tree := &Tree{ + Nodes: make([]*Node, 0, len(entries)), + } + + for _, entry := range entries { + path := filepath.Join(dir, entry.Name()) + node, err := NodeFromFileInfo(path, entry) + if err != nil { + return nil, err + } + + tree.Nodes = append(tree.Nodes, node) + + if entry.IsDir() { + node.Tree, err = NewTreeFromPath(repo, path) + if err != nil { + return nil, err + } + continue + } + + if node.Type == "file" { + file, err := os.Open(path) + defer file.Close() + if err != nil { + return nil, err + } + + wr, idch, err := repo.Create(TYPE_BLOB) + if err != nil { + return nil, err + } + + io.Copy(wr, file) + err = wr.Close() + if err != nil { + return nil, err + } + + node.Content = <-idch + } + } + + return tree, nil } -func (t *Tree) Save(w io.Writer) error { - enc := json.NewEncoder(w) - return enc.Encode(t) +func (tree *Tree) Save(repo *Repository) (ID, error) { + for _, node := range tree.Nodes { + if node.Tree != nil { + var err error + node.Subtree, err = node.Tree.Save(repo) + if err != nil { + return nil, err + } + } + } + + buf, err := json.Marshal(tree) + if err != nil { + return nil, err + } + + wr, idch, err := repo.Create(TYPE_BLOB) + if err != nil { + return nil, err + } + + _, err = wr.Write(buf) + if err != nil { + return nil, err + } + + err = wr.Close() + if err != nil { + return nil, err + } + + return <-idch, nil } -func NodeFromFileInfo(fi os.FileInfo) Node { - node := Node{ +func NewTreeFromRepo(repo *Repository, id ID) (*Tree, error) { + tree := NewTree() + + rd, err := repo.Get(TYPE_BLOB, id) + defer rd.Close() + if err != nil { + return nil, err + } + + decoder := json.NewDecoder(rd) + + err = decoder.Decode(tree) + if err != nil { + return nil, err + } + + for _, node := range tree.Nodes { + if node.Subtree != nil { + node.Tree, err = NewTreeFromRepo(repo, node.Subtree) + if err != nil { + return nil, err + } + } + } + + return tree, nil +} + +// TODO: make sure that node.Type is valid + +func (node *Node) fill_extra(path string, fi os.FileInfo) (err error) { + stat, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return + } + + node.ChangeTime = time.Unix(stat.Ctim.Unix()) + node.AccessTime = time.Unix(stat.Atim.Unix()) + node.UID = stat.Uid + node.GID = stat.Gid + + if u, nil := user.LookupId(strconv.Itoa(int(stat.Uid))); err == nil { + node.User = u.Username + } + + // TODO: implement getgrnam() + // if g, nil := user.LookupId(strconv.Itoa(int(stat.Uid))); err == nil { + // node.User = u.Username + // } + + node.Inode = stat.Ino + + switch node.Type { + case "file": + node.Size = uint64(stat.Size) + node.Links = stat.Nlink + case "dir": + // nothing to do + case "symlink": + node.LinkTarget, err = os.Readlink(path) + case "dev": + node.Device = stat.Rdev + case "chardev": + node.Device = stat.Rdev + case "fifo": + // nothing to do + case "socket": + // nothing to do + default: + panic(fmt.Sprintf("invalid node type %q", node.Type)) + } + + return err +} + +func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) { + node := &Node{ Name: fi.Name(), - Mode: fi.Mode(), + Mode: fi.Mode() & os.ModePerm, ModTime: fi.ModTime(), } - if stat, ok := fi.Sys().(*syscall.Stat_t); ok { - node.User = stat.Uid - node.Group = stat.Gid - node.AccessTime = time.Unix(stat.Atim.Unix()) + switch fi.Mode() & (os.ModeType | os.ModeCharDevice) { + case 0: + node.Type = "file" + case os.ModeDir: + node.Type = "dir" + case os.ModeSymlink: + node.Type = "symlink" + case os.ModeDevice | os.ModeCharDevice: + node.Type = "chardev" + case os.ModeDevice: + node.Type = "dev" + case os.ModeNamedPipe: + node.Type = "fifo" + case os.ModeSocket: + node.Type = "socket" } - return node + err := node.fill_extra(path, fi) + return node, err } diff --git a/tree_test.go b/tree_test.go deleted file mode 100644 index cd8403e79..000000000 --- a/tree_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package khepri_test - -import ( - "bytes" - "strings" - "testing" - "time" - - "github.com/fd0/khepri" -) - -func parseTime(str string) time.Time { - t, err := time.Parse(time.RFC3339Nano, str) - if err != nil { - panic(err) - } - - return t -} - -func TestTree(t *testing.T) { - var tree = &khepri.Tree{ - Nodes: []khepri.Node{ - khepri.Node{ - Name: "foobar", - Mode: 0755, - ModTime: parseTime("2014-04-20T22:16:54.161401+02:00"), - AccessTime: parseTime("2014-04-21T22:16:54.161401+02:00"), - User: 1000, - Group: 1001, - Content: []byte{0x41, 0x42, 0x43}, - }, - khepri.Node{ - Name: "baz", - Mode: 0755, - User: 1000, - ModTime: parseTime("2014-04-20T22:16:54.161401+02:00"), - AccessTime: parseTime("2014-04-21T22:16:54.161401+02:00"), - Group: 1001, - Content: []byte("\xde\xad\xbe\xef\xba\xdc\x0d\xe0"), - }, - }, - } - - const raw = `{"nodes":[{"name":"foobar","mode":493,"mtime":"2014-04-20T22:16:54.161401+02:00","atime":"2014-04-21T22:16:54.161401+02:00","user":1000,"group":1001,"content":"414243"},{"name":"baz","mode":493,"mtime":"2014-04-20T22:16:54.161401+02:00","atime":"2014-04-21T22:16:54.161401+02:00","user":1000,"group":1001,"content":"deadbeefbadc0de0"}]}` - - // test save - buf := &bytes.Buffer{} - - tree.Save(buf) - equals(t, raw, strings.TrimRight(buf.String(), "\n")) - - tree2 := new(khepri.Tree) - err := tree2.Restore(buf) - ok(t, err) - equals(t, tree, tree2) - - // test nodes for equality - for i, n := range tree.Nodes { - equals(t, n.Content, tree2.Nodes[i].Content) - } - - // test restore - buf = bytes.NewBufferString(raw) - - tree2 = new(khepri.Tree) - err = tree2.Restore(buf) - ok(t, err) - - // test if tree has correctly been restored - equals(t, tree, tree2) -} diff --git a/zerrors_linux.go b/zerrors_linux.go new file mode 100644 index 000000000..8ba94670a --- /dev/null +++ b/zerrors_linux.go @@ -0,0 +1,5 @@ +package khepri + +// Add constant O_PATH missing from Go1.3, will be added to Go1.4 according to +// https://code.google.com/p/go/issues/detail?id=7830 +const O_PATH = 010000000