diff options
author | Alexander Neumann <alexander@bumpern.de> | 2016-02-14 15:13:44 +0100 |
---|---|---|
committer | Alexander Neumann <alexander@bumpern.de> | 2016-02-20 17:31:20 +0100 |
commit | b63399d606d6b95f1dfd25ad70d7371f3c49ba6b (patch) | |
tree | 194475c3cd70fc94235b58d3ba480c9cd65ffe9b /cmd | |
parent | 273d028a82ac50870653edfe66c6b7f4f27736e9 (diff) |
Move things around for gb
This moves all restic source files to src/, and all vendored
dependencies to vendor/src.
Diffstat (limited to 'cmd')
32 files changed, 0 insertions, 4042 deletions
diff --git a/cmd/restic/.gitignore b/cmd/restic/.gitignore deleted file mode 100644 index aee2e4ce1..000000000 --- a/cmd/restic/.gitignore +++ /dev/null @@ -1 +0,0 @@ -config.mk diff --git a/cmd/restic/cleanup.go b/cmd/restic/cleanup.go deleted file mode 100644 index b4d4771d5..000000000 --- a/cmd/restic/cleanup.go +++ /dev/null @@ -1,65 +0,0 @@ -package main - -import ( - "fmt" - "os" - "os/signal" - "sync" - "syscall" - - "github.com/restic/restic/debug" -) - -var cleanupHandlers struct { - sync.Mutex - list []func() error - done bool -} - -var stderr = os.Stderr - -func init() { - c := make(chan os.Signal) - signal.Notify(c, syscall.SIGINT) - - go CleanupHandler(c) -} - -// AddCleanupHandler adds the function f to the list of cleanup handlers so -// that it is executed when all the cleanup handlers are run, e.g. when SIGINT -// is received. -func AddCleanupHandler(f func() error) { - cleanupHandlers.Lock() - defer cleanupHandlers.Unlock() - - cleanupHandlers.list = append(cleanupHandlers.list, f) -} - -// RunCleanupHandlers runs all registered cleanup handlers -func RunCleanupHandlers() { - cleanupHandlers.Lock() - defer cleanupHandlers.Unlock() - - if cleanupHandlers.done { - return - } - cleanupHandlers.done = true - - for _, f := range cleanupHandlers.list { - err := f() - if err != nil { - fmt.Fprintf(stderr, "error in cleanup handler: %v\n", err) - } - } -} - -// CleanupHandler handles the SIGINT signal. -func CleanupHandler(c <-chan os.Signal) { - for s := range c { - debug.Log("CleanupHandler", "signal %v received, cleaning up", s) - fmt.Println("\x1b[2KInterrupt received, cleaning up") - RunCleanupHandlers() - fmt.Println("exiting") - os.Exit(0) - } -} diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go deleted file mode 100644 index 4524001c2..000000000 --- a/cmd/restic/cmd_backup.go +++ /dev/null @@ -1,338 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "strings" - "time" - - "github.com/restic/restic" - "github.com/restic/restic/backend" - "github.com/restic/restic/debug" - "github.com/restic/restic/filter" - "github.com/restic/restic/repository" - "golang.org/x/crypto/ssh/terminal" -) - -type CmdBackup struct { - Parent string `short:"p" long:"parent" description:"use this parent snapshot (default: last snapshot in repo that has the same target)"` - Force bool `short:"f" long:"force" description:"Force re-reading the target. Overrides the \"parent\" flag"` - Excludes []string `short:"e" long:"exclude" description:"Exclude a pattern (can be specified multiple times)"` - - global *GlobalOptions -} - -func init() { - _, err := parser.AddCommand("backup", - "save file/directory", - "The backup command creates a snapshot of a file or directory", - &CmdBackup{global: &globalOpts}) - if err != nil { - panic(err) - } -} - -func formatBytes(c uint64) string { - b := float64(c) - - switch { - case c > 1<<40: - return fmt.Sprintf("%.3f TiB", b/(1<<40)) - case c > 1<<30: - return fmt.Sprintf("%.3f GiB", b/(1<<30)) - case c > 1<<20: - return fmt.Sprintf("%.3f MiB", b/(1<<20)) - case c > 1<<10: - return fmt.Sprintf("%.3f KiB", b/(1<<10)) - default: - return fmt.Sprintf("%dB", c) - } -} - -func formatSeconds(sec uint64) string { - hours := sec / 3600 - sec -= hours * 3600 - min := sec / 60 - sec -= min * 60 - if hours > 0 { - return fmt.Sprintf("%d:%02d:%02d", hours, min, sec) - } - - return fmt.Sprintf("%d:%02d", min, sec) -} - -func formatPercent(numerator uint64, denominator uint64) string { - if denominator == 0 { - return "" - } - - percent := 100.0 * float64(numerator) / float64(denominator) - - if percent > 100 { - percent = 100 - } - - return fmt.Sprintf("%3.2f%%", percent) -} - -func formatRate(bytes uint64, duration time.Duration) string { - sec := float64(duration) / float64(time.Second) - rate := float64(bytes) / sec / (1 << 20) - return fmt.Sprintf("%.2fMiB/s", rate) -} - -func formatDuration(d time.Duration) string { - sec := uint64(d / time.Second) - return formatSeconds(sec) -} - -func printTree2(indent int, t *restic.Tree) { - for _, node := range t.Nodes { - if node.Tree() != nil { - fmt.Printf("%s%s/\n", strings.Repeat(" ", indent), node.Name) - printTree2(indent+1, node.Tree()) - } else { - fmt.Printf("%s%s\n", strings.Repeat(" ", indent), node.Name) - } - } -} - -func (cmd CmdBackup) Usage() string { - return "DIR/FILE [DIR/FILE] [...]" -} - -func (cmd CmdBackup) newScanProgress() *restic.Progress { - if !cmd.global.ShowProgress() { - return nil - } - - p := restic.NewProgress(time.Second) - p.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) { - fmt.Printf("\x1b[2K[%s] %d directories, %d files, %s\r", formatDuration(d), s.Dirs, s.Files, formatBytes(s.Bytes)) - } - p.OnDone = func(s restic.Stat, d time.Duration, ticker bool) { - fmt.Printf("\x1b[2Kscanned %d directories, %d files in %s\n", s.Dirs, s.Files, formatDuration(d)) - } - - return p -} - -func (cmd CmdBackup) newArchiveProgress(todo restic.Stat) *restic.Progress { - if !cmd.global.ShowProgress() { - return nil - } - - archiveProgress := restic.NewProgress(time.Second) - - var bps, eta uint64 - itemsTodo := todo.Files + todo.Dirs - - archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) { - sec := uint64(d / time.Second) - if todo.Bytes > 0 && sec > 0 && ticker { - bps = s.Bytes / sec - if s.Bytes >= todo.Bytes { - eta = 0 - } else if bps > 0 { - eta = (todo.Bytes - s.Bytes) / bps - } - } - - itemsDone := s.Files + s.Dirs - - status1 := fmt.Sprintf("[%s] %s %s/s %s / %s %d / %d items %d errors ", - formatDuration(d), - formatPercent(s.Bytes, todo.Bytes), - formatBytes(bps), - formatBytes(s.Bytes), formatBytes(todo.Bytes), - itemsDone, itemsTodo, - s.Errors) - status2 := fmt.Sprintf("ETA %s ", formatSeconds(eta)) - - w, _, err := terminal.GetSize(int(os.Stdout.Fd())) - if err == nil { - maxlen := w - len(status2) - - if maxlen < 4 { - status1 = "" - } else if len(status1) > maxlen { - status1 = status1[:maxlen-4] - status1 += "... " - } - } - - fmt.Printf("\x1b[2K%s%s\r", status1, status2) - } - - archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) { - fmt.Printf("\nduration: %s, %s\n", formatDuration(d), formatRate(todo.Bytes, d)) - } - - return archiveProgress -} - -func samePaths(expected, actual []string) bool { - if expected == nil || actual == nil { - return true - } - - if len(expected) != len(actual) { - return false - } - for i := range expected { - if expected[i] != actual[i] { - return false - } - } - - return true -} - -var errNoSnapshotFound = errors.New("no snapshot found") - -func findLatestSnapshot(repo *repository.Repository, targets []string) (backend.ID, error) { - var ( - latest time.Time - latestID backend.ID - found bool - ) - - for snapshotID := range repo.List(backend.Snapshot, make(chan struct{})) { - snapshot, err := restic.LoadSnapshot(repo, snapshotID) - if err != nil { - return backend.ID{}, fmt.Errorf("Error listing snapshot: %v", err) - } - if snapshot.Time.After(latest) && samePaths(snapshot.Paths, targets) { - latest = snapshot.Time - latestID = snapshotID - found = true - } - } - - if !found { - return backend.ID{}, errNoSnapshotFound - } - - return latestID, nil -} - -// filterExisting returns a slice of all existing items, or an error if no -// items exist at all. -func filterExisting(items []string) (result []string, err error) { - for _, item := range items { - _, err := os.Lstat(item) - if err != nil && os.IsNotExist(err) { - continue - } - - result = append(result, item) - } - - if len(result) == 0 { - return nil, errors.New("all target directories/files do not exist") - } - - return -} - -func (cmd CmdBackup) Execute(args []string) error { - if len(args) == 0 { - return fmt.Errorf("wrong number of parameters, Usage: %s", cmd.Usage()) - } - - target := make([]string, 0, len(args)) - for _, d := range args { - if a, err := filepath.Abs(d); err == nil { - d = a - } - target = append(target, d) - } - - target, err := filterExisting(target) - if err != nil { - return err - } - - repo, err := cmd.global.OpenRepository() - if err != nil { - return err - } - - lock, err := lockRepo(repo) - defer unlockRepo(lock) - if err != nil { - return err - } - - err = repo.LoadIndex() - if err != nil { - return err - } - - var parentSnapshotID *backend.ID - - // Force using a parent - if !cmd.Force && cmd.Parent != "" { - id, err := restic.FindSnapshot(repo, cmd.Parent) - if err != nil { - return fmt.Errorf("invalid id %q: %v", cmd.Parent, err) - } - - parentSnapshotID = &id - } - - // Find last snapshot to set it as parent, if not already set - if !cmd.Force && parentSnapshotID == nil { - id, err := findLatestSnapshot(repo, target) - if err == nil { - parentSnapshotID = &id - } else if err != errNoSnapshotFound { - return err - } - } - - if parentSnapshotID != nil { - cmd.global.Verbosef("using parent snapshot %v\n", parentSnapshotID.Str()) - } - - cmd.global.Verbosef("scan %v\n", target) - - selectFilter := func(item string, fi os.FileInfo) bool { - matched, err := filter.List(cmd.Excludes, item) - if err != nil { - cmd.global.Warnf("error for exclude pattern: %v", err) - } - - if matched { - debug.Log("backup.Execute", "path %q excluded by a filter", item) - } - - return !matched - } - - stat, err := restic.Scan(target, selectFilter, cmd.newScanProgress()) - if err != nil { - return err - } - - arch := restic.NewArchiver(repo) - arch.Excludes = cmd.Excludes - arch.SelectFilter = selectFilter - - arch.Error = func(dir string, fi os.FileInfo, err error) error { - // TODO: make ignoring errors configurable - cmd.global.Warnf("\x1b[2K\rerror for %s: %v\n", dir, err) - return nil - } - - _, id, err := arch.Snapshot(cmd.newArchiveProgress(stat), target, parentSnapshotID) - if err != nil { - return err - } - - cmd.global.Verbosef("snapshot %s saved\n", id.Str()) - - return nil -} diff --git a/cmd/restic/cmd_cache.go b/cmd/restic/cmd_cache.go deleted file mode 100644 index 39733b997..000000000 --- a/cmd/restic/cmd_cache.go +++ /dev/null @@ -1,56 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/restic/restic" -) - -type CmdCache struct { - global *GlobalOptions -} - -func init() { - _, err := parser.AddCommand("cache", - "manage cache", - "The cache command creates and manages the local cache", - &CmdCache{global: &globalOpts}) - if err != nil { - panic(err) - } -} - -func (cmd CmdCache) Usage() string { - return "[update|clear]" -} - -func (cmd CmdCache) Execute(args []string) error { - // if len(args) == 0 || len(args) > 2 { - // return fmt.Errorf("wrong number of parameters, Usage: %s", cmd.Usage()) - // } - - repo, err := cmd.global.OpenRepository() - if err != nil { - return err - } - - lock, err := lockRepo(repo) - defer unlockRepo(lock) - if err != nil { - return err - } - - cache, err := restic.NewCache(repo, cmd.global.CacheDir) - if err != nil { - return err - } - - fmt.Printf("clear cache for old snapshots\n") - err = cache.Clear(repo) - if err != nil { - return err - } - fmt.Printf("done\n") - - return nil -} diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go deleted file mode 100644 index 1a7423d18..000000000 --- a/cmd/restic/cmd_cat.go +++ /dev/null @@ -1,199 +0,0 @@ -package main - -import ( - "encoding/json" - "errors" - "fmt" - "os" - - "github.com/restic/restic" - "github.com/restic/restic/backend" - "github.com/restic/restic/debug" - "github.com/restic/restic/pack" - "github.com/restic/restic/repository" -) - -type CmdCat struct { - global *GlobalOptions -} - -func init() { - _, err := parser.AddCommand("cat", - "dump something", - "The cat command dumps data structures or data from a repository", - &CmdCat{global: &globalOpts}) - if err != nil { - panic(err) - } -} - -func (cmd CmdCat) Usage() string { - return "[pack|blob|tree|snapshot|key|masterkey|config|lock] ID" -} - -func (cmd CmdCat) Execute(args []string) error { - if len(args) < 1 || (args[0] != "masterkey" && args[0] != "config" && len(args) != 2) { - return fmt.Errorf("type or ID not specified, Usage: %s", cmd.Usage()) - } - - repo, err := cmd.global.OpenRepository() - if err != nil { - return err - } - - lock, err := lockRepo(repo) - defer unlockRepo(lock) - if err != nil { - return err - } - - tpe := args[0] - - var id backend.ID - if tpe != "masterkey" && tpe != "config" { - id, err = backend.ParseID(args[1]) - if err != nil { - if tpe != "snapshot" { - return err - } - - // find snapshot id with prefix - id, err = restic.FindSnapshot(repo, args[1]) - if err != nil { - return err - } - } - } - - // handle all types that don't need an index - switch tpe { - case "config": - buf, err := json.MarshalIndent(repo.Config, "", " ") - if err != nil { - return err - } - - fmt.Println(string(buf)) - return nil - case "index": - buf, err := repo.LoadAndDecrypt(backend.Index, id) - if err != nil { - return err - } - - _, err = os.Stdout.Write(append(buf, '\n')) - return err - - case "snapshot": - sn := &restic.Snapshot{} - err = repo.LoadJSONUnpacked(backend.Snapshot, id, sn) - if err != nil { - return err - } - - buf, err := json.MarshalIndent(&sn, "", " ") - if err != nil { - return err - } - - fmt.Println(string(buf)) - - return nil - case "key": - h := backend.Handle{Type: backend.Key, Name: id.String()} - buf, err := backend.LoadAll(repo.Backend(), h, nil) - if err != nil { - return err - } - - key := &repository.Key{} - err = json.Unmarshal(buf, key) - if err != nil { - return err - } - - buf, err = json.MarshalIndent(&key, "", " ") - if err != nil { - return err - } - - fmt.Println(string(buf)) - return nil - case "masterkey": - buf, err := json.MarshalIndent(repo.Key(), "", " ") - if err != nil { - return err - } - - fmt.Println(string(buf)) - return nil - case "lock": - lock, err := restic.LoadLock(repo, id) - if err != nil { - return err - } - - buf, err := json.MarshalIndent(&lock, "", " ") - if err != nil { - return err - } - - fmt.Println(string(buf)) - - return nil - } - - // load index, handle all the other types - err = repo.LoadIndex() - if err != nil { - return err - } - - switch tpe { - case "pack": - h := backend.Handle{Type: backend.Data, Name: id.String()} - buf, err := backend.LoadAll(repo.Backend(), h, nil) - if err != nil { - return err - } - - _, err = os.Stdout.Write(buf) - return err - - case "blob": - blob, err := repo.Index().Lookup(id) - if err != nil { - return err - } - - buf := make([]byte, blob.Length) - data, err := repo.LoadBlob(blob.Type, id, buf) - if err != nil { - return err - } - - _, err = os.Stdout.Write(data) - return err - - case "tree": - debug.Log("cat", "cat tree %v", id.Str()) - tree := restic.NewTree() - err = repo.LoadJSONPack(pack.Tree, id, tree) - if err != nil { - debug.Log("cat", "unable to load tree %v: %v", id.Str(), err) - return err - } - - buf, err := json.MarshalIndent(&tree, "", " ") - if err != nil { - debug.Log("cat", "error json.MarshalIndent(): %v", err) - return err - } - - _, err = os.Stdout.Write(append(buf, '\n')) - return nil - - default: - return errors.New("invalid type") - } -} diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go deleted file mode 100644 index 8c59c2ffe..000000000 --- a/cmd/restic/cmd_check.go +++ /dev/null @@ -1,165 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "os" - "time" - - "golang.org/x/crypto/ssh/terminal" - - "github.com/restic/restic" - "github.com/restic/restic/checker" -) - -type CmdCheck struct { - ReadData bool `long:"read-data" default:"false" description:"Read data blobs"` - CheckUnused bool `long:"check-unused" default:"false" description:"Check for unused blobs"` - - global *GlobalOptions -} - -func init() { - _, err := parser.AddCommand("check", - "check the repository", - "The check command check the integrity and consistency of the repository", - &CmdCheck{global: &globalOpts}) - if err != nil { - panic(err) - } -} - -func (cmd CmdCheck) Usage() string { - return "[check-options]" -} - -func (cmd CmdCheck) newReadProgress(todo restic.Stat) *restic.Progress { - if !cmd.global.ShowProgress() { - return nil - } - - readProgress := restic.NewProgress(time.Second) - - readProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) { - status := fmt.Sprintf("[%s] %s %d / %d items", - formatDuration(d), - formatPercent(s.Blobs, todo.Blobs), - s.Blobs, todo.Blobs) - - w, _, err := terminal.GetSize(int(os.Stdout.Fd())) - if err == nil { - if len(status) > w { - max := w - len(status) - 4 - status = status[:max] + "... " - } - } - - fmt.Printf("\x1b[2K%s\r", status) - } - - readProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) { - fmt.Printf("\nduration: %s\n", formatDuration(d)) - } - - return readProgress -} - -func (cmd CmdCheck) Execute(args []string) error { - if len(args) != 0 { - return errors.New("check has no arguments") - } - - repo, err := cmd.global.OpenRepository() - if err != nil { - return err - } - - if !cmd.global.NoLock { - cmd.global.Verbosef("Create exclusive lock for repository\n") - lock, err := lockRepoExclusive(repo) - defer unlockRepo(lock) - if err != nil { - return err - } - } - - chkr := checker.New(repo) - - cmd.global.Verbosef("Load indexes\n") - hints, errs := chkr.LoadIndex() - - dupFound := false - for _, hint := range hints { - cmd.global.Printf("%v\n", hint) - if _, ok := hint.(checker.ErrDuplicatePacks); ok { - dupFound = true - } - } - - if dupFound { - cmd.global.Printf("\nrun `restic rebuild-index' to correct this\n") - } - - if len(errs) > 0 { - for _, err := range errs { - cmd.global.Warnf("error: %v\n", err) - } - return fmt.Errorf("LoadIndex returned errors") - } - - done := make(chan struct{}) - defer close(done) - - errorsFound := false - errChan := make(chan error) - - cmd.global.Verbosef("Check all packs\n") - go chkr.Packs(errChan, done) - - for err := range errChan { - errorsFound = true - fmt.Fprintf(os.Stderr, "%v\n", err) - } - - cmd.global.Verbosef("Check snapshots, trees and blobs\n") - errChan = make(chan error) - go chkr.Structure(errChan, done) - - for err := range errChan { - errorsFound = true - if e, ok := err.(checker.TreeError); ok { - fmt.Fprintf(os.Stderr, "error for tree %v:\n", e.ID.Str()) - for _, treeErr := range e.Errors { - fmt.Fprintf(os.Stderr, " %v\n", treeErr) - } - } else { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - } - } - - if cmd.CheckUnused { - for _, id := range chkr.UnusedBlobs() { - cmd.global.Verbosef("unused blob %v\n", id.Str()) - errorsFound = true - } - } - - if cmd.ReadData { - cmd.global.Verbosef("Read all data\n") - - p := cmd.newReadProgress(restic.Stat{Blobs: chkr.CountPacks()}) - errChan := make(chan error) - - go chkr.ReadData(p, errChan, done) - - for err := range errChan { - errorsFound = true - fmt.Fprintf(os.Stderr, "%v\n", err) - } - } - - if errorsFound { - return errors.New("repository contains errors") - } - return nil -} diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go deleted file mode 100644 index b7d456e4e..000000000 --- a/cmd/restic/cmd_dump.go +++ /dev/null @@ -1,177 +0,0 @@ -// +build debug - -package main - -import ( - "encoding/json" - "fmt" - "io" - "os" - - "github.com/juju/errors" - "github.com/restic/restic" - "github.com/restic/restic/backend" - "github.com/restic/restic/pack" - "github.com/restic/restic/repository" -) - -type CmdDump struct { - global *GlobalOptions - - repo *repository.Repository -} - -func init() { - _, err := parser.AddCommand("dump", - "dump data structures", - "The dump command dumps data structures from a repository as JSON documents", - &CmdDump{global: &globalOpts}) - if err != nil { - panic(err) - } -} - -func (cmd CmdDump) Usage() string { - return "[indexes|snapshots|trees|all]" -} - -func prettyPrintJSON(wr io.Writer, item interface{}) error { - buf, err := json.MarshalIndent(item, "", " ") - if err != nil { - return err - } - - _, err = wr.Write(append(buf, '\n')) - return err -} - -func printSnapshots(repo *repository.Repository, wr io.Writer) error { - done := make(chan struct{}) - defer close(done) - - for id := range repo.List(backend.Snapshot, done) { - snapshot, err := restic.LoadSnapshot(repo, id) - if err != nil { - fmt.Fprintf(os.Stderr, "LoadSnapshot(%v): %v", id.Str(), err) - continue - } - - fmt.Fprintf(wr, "snapshot_id: %v\n", id) - - err = prettyPrintJSON(wr, snapshot) - if err != nil { - return err - } - } - - return nil -} - -func printTrees(repo *repository.Repository, wr io.Writer) error { - done := make(chan struct{}) - defer close(done) - - trees := []backend.ID{} - - for _, idx := range repo.Index().All() { - for blob := range idx.Each(nil) { - if blob.Type != pack.Tree { - continue - } - - trees = append(trees, blob.ID) - } - } - - for _, id := range trees { - tree, err := restic.LoadTree(repo, id) - if err != nil { - fmt.Fprintf(os.Stderr, "LoadTree(%v): %v", id.Str(), err) - continue - } - - fmt.Fprintf(wr, "tree_id: %v\n", id) - - prettyPrintJSON(wr, tree) - } - - return nil -} - -func (cmd CmdDump) DumpIndexes() error { - done := make(chan struct{}) - defer close(done) - - for id := range cmd.repo.List(backend.Index, done) { - fmt.Printf("index_id: %v\n", id) - - idx, err := repository.LoadIndex(cmd.repo, id.String()) - if err != nil { - return err - } - - err = idx.Dump(os.Stdout) - if err != nil { - return err - } - } - - return nil -} - -func (cmd CmdDump) Execute(args []string) error { - if len(args) != 1 { - return fmt.Errorf("type not specified, Usage: %s", cmd.Usage()) - } - - repo, err := cmd.global.OpenRepository() - if err != nil { - return err - } - cmd.repo = repo - - lock, err := lockRepo(repo) - defer unlockRepo(lock) - if err != nil { - return err - } - - err = repo.LoadIndex() - if err != nil { - return err - } - - tpe := args[0] - - switch tpe { - case "indexes": - return cmd.DumpIndexes() - case "snapshots": - return printSnapshots(repo, os.Stdout) - case "trees": - return printTrees(repo, os.Stdout) - case "all": - fmt.Printf("snapshots:\n") - err := printSnapshots(repo, os.Stdout) - if err != nil { - return err - } - - fmt.Printf("\ntrees:\n") - - err = printTrees(repo, os.Stdout) - if err != nil { - return err - } - - fmt.Printf("\nindexes:\n") - err = cmd.DumpIndexes() - if err != nil { - return err - } - - return nil - default: - return errors.Errorf("no such type %q", tpe) - } -} diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go deleted file mode 100644 index fa2d7859e..000000000 --- a/cmd/restic/cmd_find.go +++ /dev/null @@ -1,197 +0,0 @@ -package main - -import ( - "fmt" - "path/filepath" - "time" - - "github.com/restic/restic" - "github.com/restic/restic/backend" - "github.com/restic/restic/debug" - "github.com/restic/restic/repository" -) - -type findResult struct { - node *restic.Node - path string -} - -type CmdFind struct { - Oldest string `short:"o" long:"oldest" description:"Oldest modification date/time"` - Newest string `short:"n" long:"newest" description:"Newest modification date/time"` - Snapshot string `short:"s" long:"snapshot" description:"Snapshot ID to search in"` - - oldest, newest time.Time - pattern string - global *GlobalOptions -} - -var timeFormats = []string{ - "2006-01-02", - "2006-01-02 15:04", - "2006-01-02 15:04:05", - "2006-01-02 15:04:05 -0700", - "2006-01-02 15:04:05 MST", - "02.01.2006", - "02.01.2006 15:04", - "02.01.2006 15:04:05", - "02.01.2006 15:04:05 -0700", - "02.01.2006 15:04:05 MST", - "Mon Jan 2 15:04:05 -0700 MST 2006", -} - -func init() { - _, err := parser.AddCommand("find", - "find a file/directory", - "The find command searches for files or directories in snapshots", - &CmdFind{global: &globalOpts}) - if err != nil { - panic(err) - } -} - -func parseTime(str string) (time.Time, error) { - for _, fmt := range timeFormats { - if t, err := time.ParseInLocation(fmt, str, time.Local); err == nil { - return t, nil - } - } - - return time.Time{}, fmt.Errorf("unable to parse time: %q", str) -} - -func (c CmdFind) findInTree(repo *repository.Repository, id backend.ID, path string) ([]findResult, error) { - debug.Log("restic.find", "checking tree %v\n", id) - tree, err := restic.LoadTree(repo, id) - if err != nil { - return nil, err - } - - results := []findResult{} - for _, node := range tree.Nodes { - debug.Log("restic.find", " testing entry %q\n", node.Name) - - m, err := filepath.Match(c.pattern, node.Name) - if err != nil { - return nil, err - } - - if m { - debug.Log("restic.find", " pattern matches\n") - if !c.oldest.IsZero() && node.ModTime.Before(c.oldest) { - debug.Log("restic.find", " ModTime is older than %s\n", c.oldest) - continue - } - - if !c.newest.IsZero() && node.ModTime.After(c.newest) { - debug.Log("restic.find", " ModTime is newer than %s\n", c.newest) - continue - } - - results = append(results, findResult{node: node, path: path}) - } else { - debug.Log("restic.find", " pattern does not match\n") - } - - if node.Type == "dir" { - subdirResults, err := c.findInTree(repo, *node.Subtree, filepath.Join(path, node.Name)) - if err != nil { - return nil, err - } - - results = append(results, subdirResults...) - } - } - - return results, nil -} - -func (c CmdFind) findInSnapshot(repo *repository.Repository, id backend.ID) error { - debug.Log("restic.find", "searching in snapshot %s\n for entries within [%s %s]", id.Str(), c.oldest, c.newest) - - sn, err := restic.LoadSnapshot(repo, id) - if err != nil { - return err - } - - results, err := c.findInTree(repo, *sn.Tree, "") - if err != nil { - return err - } - - if len(results) == 0 { - return nil - } - c.global.Verbosef("found %d matching entries in snapshot %s\n", len(results), id) - for _, res := range results { - res.node.Name = filepath.Join(res.path, res.node.Name) - c.global.Printf(" %s\n", res.node) - } - - return nil -} - -func (CmdFind) Usage() string { - return "[find-OPTIONS] PATTERN" -} - -func (c CmdFind) Execute(args []string) error { - if len(args) != 1 { - return fmt.Errorf("wrong number of arguments, Usage: %s", c.Usage()) - } - - var err error - - if c.Oldest != "" { - c.oldest, err = parseTime(c.Oldest) - if err != nil { - return err - } - } - - if c.Newest != "" { - c.newest, err = parseTime(c.Newest) - if err != nil { - return err - } - } - - repo, err := c.global.OpenRepository() - if err != nil { - return err - } - - lock, err := lockRepo(repo) - defer unlockRepo(lock) - if err != nil { - return err - } - - err = repo.LoadIndex() - if err != nil { - return err - } - - c.pattern = args[0] - - if c.Snapshot != "" { - snapshotID, err := restic.FindSnapshot(repo, c.Snapshot) - if err != nil { - return fmt.Errorf("invalid id %q: %v", args[1], err) - } - - return c.findInSnapshot(repo, snapshotID) - } - - done := make(chan struct{}) - defer close(done) - for snapshotID := range repo.List(backend.Snapshot, done) { - err := c.findInSnapshot(repo, snapshotID) - - if err != nil { - return err - } - } - - return nil -} diff --git a/cmd/restic/cmd_init.go b/cmd/restic/cmd_init.go deleted file mode 100644 index 827684b3e..000000000 --- a/cmd/restic/cmd_init.go +++ /dev/null @@ -1,52 +0,0 @@ -package main - -import ( - "errors" - - "github.com/restic/restic/repository" -) - -type CmdInit struct { - global *GlobalOptions -} - -func (cmd CmdInit) Execute(args []string) error { - if cmd.global.Repo == "" { - return errors.New("Please specify repository location (-r)") - } - - be, err := create(cmd.global.Repo) - if err != nil { - cmd.global.Exitf(1, "creating backend at %s failed: %v\n", cmd.global.Repo, err) - } - - if cmd.global.password == "" { - cmd.global.password = cmd.global.ReadPasswordTwice( - "enter password for new backend: ", - "enter password again: ") - } - - s := repository.New(be) - err = s.Init(cmd.global.password) - if err != nil { - cmd.global.Exitf(1, "creating key in backend at %s failed: %v\n", cmd.global.Repo, err) - } - - cmd.global.Verbosef("created restic backend %v at %s\n", s.Config.ID[:10], cmd.global.Repo) - cmd.global.Verbosef("\n") - cmd.global.Verbosef("Please note that knowledge of your password is required to access\n") - cmd.global.Verbosef("the repository. Losing your password means that your data is\n") - cmd.global.Verbosef("irrecoverably lost.\n") - - return nil -} - -func init() { - _, err := parser.AddCommand("init", - "create repository", - "The init command creates a new repository", - &CmdInit{global: &globalOpts}) - if err != nil { - panic(err) - } -} diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go deleted file mode 100644 index c585ae4cd..000000000 --- a/cmd/restic/cmd_key.go +++ /dev/null @@ -1,165 +0,0 @@ -package main - -import ( - "errors" - "fmt" - - "github.com/restic/restic/backend" - "github.com/restic/restic/repository" -) - -type CmdKey struct { - global *GlobalOptions - newPassword string -} - -func init() { - _, err := parser.AddCommand("key", - "manage keys", - "The key command manages keys (passwords) of a repository", - &CmdKey{global: &globalOpts}) - if err != nil { - panic(err) - } -} - -func (cmd CmdKey) listKeys(s *repository.Repository) error { - tab := NewTable() - tab.Header = fmt.Sprintf(" %-10s %-10s %-10s %s", "ID", "User", "Host", "Created") - tab.RowFormat = "%s%-10s %-10s %-10s %s" - - plen, err := s.PrefixLength(backend.Key) - if err != nil { - return err - } - - done := make(chan struct{}) - defer close(done) - - for id := range s.List(backend.Key, done) { - k, err := repository.LoadKey(s, id.String()) - if err != nil { - cmd.global.Warnf("LoadKey() failed: %v\n", err) - continue - } - - var current string - if id.String() == s.KeyName() { - current = "*" - } else { - current = " " - } - tab.Rows = append(tab.Rows, []interface{}{current, id.String()[:plen], - k.Username, k.Hostname, k.Created.Format(TimeFormat)}) - } - - return tab.Write(cmd.global.stdout) -} - -func (cmd CmdKey) getNewPassword() string { - if cmd.newPassword != "" { - return cmd.newPassword - } - - return cmd.global.ReadPasswordTwice( - "enter password for new key: ", - "enter password again: ") -} - -func (cmd CmdKey) addKey(repo *repository.Repository) error { - id, err := repository.AddKey(repo, cmd.getNewPassword(), repo.Key()) - if err != nil { - return fmt.Errorf("creating new key failed: %v\n", err) - } - - cmd.global.Verbosef("saved new key as %s\n", id) - - return nil -} - -func (cmd CmdKey) deleteKey(repo *repository.Repository, name string) error { - if name == repo.KeyName() { - return errors.New("refusing to remove key currently used to access repository") - } - - err := repo.Backend().Remove(backend.Key, name) - if err != nil { - return err - } - - cmd.global.Verbosef("removed key %v\n", name) - return nil -} - -func (cmd CmdKey) changePassword(repo *repository.Repository) error { - id, err := repository.AddKey(repo, cmd.getNewPassword(), repo.Key()) - if err != nil { - return fmt.Errorf("creating new key failed: %v\n", err) - } - - err = repo.Backend().Remove(backend.Key, repo.KeyName()) - if err != nil { - return err - } - - cmd.global.Verbosef("saved new key as %s\n", id) - - return nil -} - -func (cmd CmdKey) Usage() string { - return "[list|add|rm|passwd] [ID]" -} - -func (cmd CmdKey) Execute(args []string) error { - if len(args) < 1 || (args[0] == "rm" && len(args) != 2) { - return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage()) - } - - repo, err := cmd.global.OpenRepository() - if err != nil { - return err - } - - switch args[0] { - case "list": - lock, err := lockRepo(repo) - defer unlockRepo(lock) - if err != nil { - return err - } - - return cmd.listKeys(repo) - case "add": - lock, err := lockRepo(repo) - defer unlockRepo(lock) - if err != nil { - return err - } - - return cmd.addKey(repo) - case "rm": - lock, err := lockRepoExclusive(repo) - defer unlockRepo(lock) - if err != nil { - return err - } - - id, err := backend.Find(repo.Backend(), backend.Key, args[1]) - if err != nil { - return err - } - - return cmd.deleteKey(repo, id) - case "passwd": - lock, err := lockRepoExclusive(repo) - defer unlockRepo(lock) - if err != nil { - return err - } - - return cmd.changePassword(repo) - } - - return nil -} diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go deleted file mode 100644 index fc13ff5a1..000000000 --- a/cmd/restic/cmd_list.go +++ /dev/null @@ -1,80 +0,0 @@ -package main - -import ( - "errors" - "fmt" - - "github.com/restic/restic/backend" -) - -type CmdList struct { - global *GlobalOptions -} - -func init() { - _, err := parser.AddCommand("list", - "lists data", - "The list command lists structures or data of a repository", - &CmdList{global: &globalOpts}) - if err != nil { - panic(err) - } -} - -func (cmd CmdList) Usage() string { - return "[blobs|packs|index|snapshots|keys|locks]" -} - -func (cmd CmdList) Execute(args []string) error { - if len(args) != 1 { - return fmt.Errorf("type not specified, Usage: %s", cmd.Usage()) - } - - repo, err := cmd.global.OpenRepository() - if err != nil { - return err - } - - if !cmd.global.NoLock { - lock, err := lockRepo(repo) - defer unlockRepo(lock) - if err != nil { - return err - } - } - - var t backend.Type - switch args[0] { - case "blobs": - err = repo.LoadIndex() - if err != nil { - return err - } - - for _, idx := range repo.Index().All() { - for blob := range idx.Each(nil) { - cmd.global.Printf("%s\n", blob.ID) - } - } - - return nil - case "packs": - t = backend.Data - case "index": - t = backend.Index - case "snapshots": - t = backend.Snapshot - case "keys": - t = backend.Key - case "locks": - t = backend.Lock - default: - return errors.New("invalid type") - } - - for id := range repo.List(t, nil) { - cmd.global.Printf("%s\n", id) - } - - return nil -} diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go deleted file mode 100644 index e3bf0242b..000000000 --- a/cmd/restic/cmd_ls.go +++ /dev/null @@ -1,101 +0,0 @@ -package main - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/restic/restic" - "github.com/restic/restic/backend" - "github.com/restic/restic/repository" -) - -type CmdLs struct { - Long bool `short:"l" long:"long" description:"Use a long listing format showing size and mode"` - - global *GlobalOptions -} - -func init() { - _, err := parser.AddCommand("ls", - "list files", - "The ls command lists all files and directories in a snapshot", - &CmdLs{global: &globalOpts}) - if err != nil { - panic(err) - } -} - -func (cmd CmdLs) printNode(prefix string, n *restic.Node) string { - if !cmd.Long { - return filepath.Join(prefix, n.Name) - } - - switch n.Type { - case "file": - return fmt.Sprintf("%s %5d %5d %6d %s %s", - n.Mode, n.UID, n.GID, n.Size, n.ModTime, filepath.Join(prefix, n.Name)) - case "dir": - return fmt.Sprintf("%s %5d %5d %6d %s %s", - n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime, filepath.Join(prefix, n.Name)) - case "symlink": - return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s", - n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime, filepath.Join(prefix, n.Name), n.LinkTarget) - default: - return fmt.Sprintf("<Node(%s) %s>", n.Type, n.Name) - } -} - -func (cmd CmdLs) printTree(prefix string, repo *repository.Repository, id backend.ID) error { - tree, err := restic.LoadTree(repo, id) - if err != nil { - return err - } - - for _, entry := range tree.Nodes { - cmd.global.Printf(cmd.printNode(prefix, entry) + "\n") - - if entry.Type == "dir" && entry.Subtree != nil { - err = cmd.printTree(filepath.Join(prefix, entry.Name), repo, *entry.Subtree) - if err != nil { - return err - } - } - } - - return nil -} - -func (cmd CmdLs) Usage() string { - return "snapshot-ID [DIR]" -} - -func (cmd CmdLs) Execute(args []string) error { - if len(args) < 1 || len(args) > 2 { - return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage()) - } - - repo, err := cmd.global.OpenRepository() - if err != nil { - return err - } - - err = repo.LoadIndex() - if err != nil { - return err - } - - id, err := restic.FindSnapshot(repo, args[0]) - if err != nil { - return err - } - - sn, err := restic.LoadSnapshot(repo, id) - if err != nil { - return err - } - - cmd.global.Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time) - - return cmd.printTree("", repo, *sn.Tree) -} diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go deleted file mode 100644 index b8d8cb277..000000000 --- a/cmd/restic/cmd_mount.go +++ /dev/null @@ -1,107 +0,0 @@ -// +build !openbsd -// +build !windows - -package main - -import ( - "fmt" - "os" - - "github.com/restic/restic/fuse" - - systemFuse "bazil.org/fuse" - "bazil.org/fuse/fs" -) - -type CmdMount struct { - Root bool `long:"owner-root" description:"use 'root' as the owner of files and dirs" default:"false"` - - global *GlobalOptions - ready chan struct{} - done chan struct{} -} - -func init() { - _, err := parser.AddCommand("mount", - "mount a repository", - "The mount command mounts a repository read-only to a given directory", - &CmdMount{ - global: &globalOpts, - ready: make(chan struct{}, 1), - done: make(chan struct{}), - }) - if err != nil { - panic(err) - } -} - -func (cmd CmdMount) Usage() string { - return "MOUNTPOINT" -} - -func (cmd CmdMount) Execute(args []string) error { - if len(args) == 0 { - return fmt.Errorf("wrong number of parameters, Usage: %s", cmd.Usage()) - } - - repo, err := cmd.global.OpenRepository() - if err != nil { - return err - } - - err = repo.LoadIndex() - if err != nil { - return err - } - - mountpoint := args[0] - if _, err := os.Stat(mountpoint); os.IsNotExist(err) { - cmd.global.Verbosef("Mountpoint %s doesn't exist, creating it\n", mountpoint) - err = os.Mkdir(mountpoint, os.ModeDir|0700) - if err != nil { - return err - } - } - c, err := systemFuse.Mount( - mountpoint, - systemFuse.ReadOnly(), - systemFuse.FSName("restic"), - ) - if err != nil { - return err - } - - root := fs.Tree{} - root.Add("snapshots", fuse.NewSnapshotsDir(repo, cmd.Root)) - - cmd.global.Printf("Now serving %s at %s\n", repo.Backend().Location(), mountpoint) - cmd.global.Printf("Don't forget to umount after quitting!\n") - - AddCleanupHandler(func() error { - return systemFuse.Unmount(mountpoint) - }) - - cmd.ready <- struct{}{} - - errServe := make(chan error) - go func() { - err = fs.Serve(c, &root) - if err != nil { - errServe <- err - } - - <-c.Ready - errServe <- c.MountError - }() - - select { - case err := <-errServe: - return err - case <-cmd.done: - err := c.Close() - if err != nil { - cmd.global.Printf("Error closing fuse connection: %s\n", err) - } - return systemFuse.Unmount(mountpoint) - } -} diff --git a/cmd/restic/cmd_optimize.go b/cmd/restic/cmd_optimize.go deleted file mode 100644 index 1e29ce1d7..000000000 --- a/cmd/restic/cmd_optimize.go +++ /dev/null @@ -1,84 +0,0 @@ -package main - -import ( - "errors" - "fmt" - - "github.com/restic/restic/backend" - "github.com/restic/restic/checker" -) - -type CmdOptimize struct { - global *GlobalOptions -} - -func init() { - _, err := parser.AddCommand("optimize", - "optimize the repository", - "The optimize command reorganizes the repository and removes uneeded data", - &CmdOptimize{global: &globalOpts}) - if err != nil { - panic(err) - } -} - -func (cmd CmdOptimize) Usage() string { - return "[optimize-options]" -} - -func (cmd CmdOptimize) Execute(args []string) error { - if len(args) != 0 { - return errors.New("optimize has no arguments") - } - - repo, err := cmd.global.OpenRepository() - if err != nil { - return err - } - - cmd.global.Verbosef("Create exclusive lock for repository\n") - lock, err := lockRepoExclusive(repo) - defer unlockRepo(lock) - if err != nil { - return err - } - - chkr := checker.New(repo) - - cmd.global.Verbosef("Load indexes\n") - _, errs := chkr.LoadIndex() - - if len(errs) > 0 { - for _, err := range errs { - cmd.global.Warnf("error: %v\n", err) - } - return fmt.Errorf("LoadIndex returned errors") - } - - done := make(chan struct{}) - errChan := make(chan error) - go chkr.Structure(errChan, done) - - for err := range errChan { - if e, ok := err.(checker.TreeError); ok { - cmd.global.Warnf("error for tree %v:\n", e.ID.Str()) - for _, treeErr := range e.Errors { - cmd.global.Warnf(" %v\n", treeErr) - } - } else { - cmd.global.Warnf("error: %v\n", err) - } - } - - unusedBlobs := backend.NewIDSet(chkr.UnusedBlobs()...) - cmd.global.Verbosef("%d unused blobs found, repacking...\n", len(unusedBlobs)) - - repacker := checker.NewRepacker(repo, unusedBlobs) - err = repacker.Repack() - if err != nil { - return err - } - - cmd.global.Verbosef("repacking done\n") - return nil -} diff --git a/cmd/restic/cmd_rebuild_index.go b/cmd/restic/cmd_rebuild_index.go deleted file mode 100644 index 27dae2392..000000000 --- a/cmd/restic/cmd_rebuild_index.go +++ /dev/null @@ -1,204 +0,0 @@ -package main - -import ( - "bytes" - "fmt" - - "github.com/restic/restic/backend" - "github.com/restic/restic/debug" - "github.com/restic/restic/pack" - "github.com/restic/restic/repository" -) - -type CmdRebuildIndex struct { - global *GlobalOptions - - repo *repository.Repository -} - -func init() { - _, err := parser.AddCommand("rebuild-index", - "rebuild the index", - "The rebuild-index command builds a new index", - &CmdRebuildIndex{global: &globalOpts}) - if err != nil { - panic(err) - } -} - -func (cmd CmdRebuildIndex) storeIndex(index *repository.Index) (*repository.Index, error) { - debug.Log("RebuildIndex.RebuildIndex", "saving index") - - cmd.global.Printf(" saving new index\n") - id, err := repository.SaveIndex(cmd.repo, index) - if err != nil { - debug.Log("RebuildIndex.RebuildIndex", "error saving index: %v", err) - return nil, err - } - - debug.Log("RebuildIndex.RebuildIndex", "index saved as %v", id.Str()) - index = repository.NewIndex() - - return index, nil -} - -func (cmd CmdRebuildIndex) RebuildIndex() error { - debug.Log("RebuildIndex.RebuildIndex", "start") - - done := make(chan struct{}) - defer close(done) - - indexIDs := backend.NewIDSet() - for id := range cmd.repo.List(backend.Index, done) { - indexIDs.Insert(id) - } - - cmd.global.Printf("rebuilding index from %d indexes\n", len(indexIDs)) - - debug.Log("RebuildIndex.RebuildIndex", "found %v indexes", len(indexIDs)) - - combinedIndex := repository.NewIndex() - packsDone := backend.NewIDSet() - - type Blob struct { - id backend.ID - tpe pack.BlobType - } - blobsDone := make(map[Blob]struct{}) - - i := 0 - for indexID := range indexIDs { - cmd.global.Printf(" loading index %v\n", i) - - debug.Log("RebuildIndex.RebuildIndex", "load index %v", indexID.Str()) - idx, err := repository.LoadIndex(cmd.repo, indexID.String()) - if err != nil { - return err - } - - debug.Log("RebuildIndex.RebuildIndex", "adding blobs from index %v", indexID.Str()) - - for packedBlob := range idx.Each(done) { - packsDone.Insert(packedBlob.PackID) - b := Blob{ - id: packedBlob.ID, - tpe: packedBlob.Type, - } - if _, ok := blobsDone[b]; ok { - continue - } - - blobsDone[b] = struct{}{} - combinedIndex.Store(packedBlob) - } - - combinedIndex.AddToSupersedes(indexID) - - if repository.IndexFull(combinedIndex) { - combinedIndex, err = cmd.storeIndex(combinedIndex) - if err != nil { - return err - } - } - - i++ - } - - var err error - if combinedIndex.Length() > 0 { - combinedIndex, err = cmd.storeIndex(combinedIndex) - if err != nil { - return err - } - } - - cmd.global.Printf("removing %d old indexes\n", len(indexIDs)) - for id := range indexIDs { - debug.Log("RebuildIndex.RebuildIndex", "remove index %v", id.Str()) - - err := cmd.repo.Backend().Remove(backend.Index, id.String()) - if err != nil { - debug.Log("RebuildIndex.RebuildIndex", "error removing index %v: %v", id.Str(), err) - return err - } - } - - cmd.global.Printf("checking for additional packs\n") - newPacks := 0 - var buf []byte - for packID := range cmd.repo.List(backend.Data, done) { - if packsDone.Has(packID) { - continue - } - - debug.Log("RebuildIndex.RebuildIndex", "pack %v not indexed", packID.Str()) - newPacks++ - - var err error - - h := backend.Handle{Type: backend.Data, Name: packID.String()} - buf, err = backend.LoadAll(cmd.repo.Backend(), h, buf) - if err != nil { - debug.Log("RebuildIndex.RebuildIndex", "error while loading pack %v", packID.Str()) - return fmt.Errorf("error while loading pack %v: %v", packID.Str(), err) - } - - hash := backend.Hash(buf) - if !hash.Equal(packID) { - debug.Log("RebuildIndex.RebuildIndex", "Pack ID does not match, want %v, got %v", packID.Str(), hash.Str()) - return fmt.Errorf("Pack ID does not match, want %v, got %v", packID.Str(), hash.Str()) - } - - up, err := pack.NewUnpacker(cmd.repo.Key(), bytes.NewReader(buf)) - if err != nil { - debug.Log("RebuildIndex.RebuildIndex", "error while unpacking pack %v", packID.Str()) - return err - } - - for _, blob := range up.Entries { - debug.Log("RebuildIndex.RebuildIndex", "pack %v: blob %v", packID.Str(), blob) - combinedIndex.Store(repository.PackedBlob{ - Type: blob.Type, - ID: blob.ID, - PackID: packID, - Offset: blob.Offset, - Length: blob.Length, - }) - } - - if repository.IndexFull(combinedIndex) { - combinedIndex, err = cmd.storeIndex(combinedIndex) - if err != nil { - return err - } - } - } - - if combinedIndex.Length() > 0 { - combinedIndex, err = cmd.storeIndex(combinedIndex) - if err != nil { - return err - } - } - - cmd.global.Printf("added %d packs to the index\n", newPacks) - - debug.Log("RebuildIndex.RebuildIndex", "done") - return nil -} - -func (cmd CmdRebuildIndex) Execute(args []string) error { - repo, err := cmd.global.OpenRepository() - if err != nil { - return err - } - cmd.repo = repo - - lock, err := lockRepoExclusive(repo) - defer unlockRepo(lock) - if err != nil { - return err - } - - return cmd.RebuildIndex() -} diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go deleted file mode 100644 index 0daf74966..000000000 --- a/cmd/restic/cmd_restore.go +++ /dev/null @@ -1,116 +0,0 @@ -package main - -import ( - "errors" - "fmt" - - "github.com/restic/restic" - "github.com/restic/restic/debug" - "github.com/restic/restic/filter" -) - -type CmdRestore struct { - Exclude []string `short:"e" long:"exclude" description:"Exclude a pattern (can be specified multiple times)"` - Include []string `short:"i" long:"include" description:"Include a pattern, exclude everything else (can be specified multiple times)"` - Target string `short:"t" long:"target" description:"Directory to restore to"` - - global *GlobalOptions -} - -func init() { - _, err := parser.AddCommand("restore", - "restore a snapshot", - "The restore command restores a snapshot to a directory", - &CmdRestore{global: &globalOpts}) - if err != nil { - panic(err) - } -} - -func (cmd CmdRestore) Usage() string { - return "snapshot-ID" -} - -func (cmd CmdRestore) Execute(args []string) error { - if len(args) != 1 { - return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage()) - } - - if cmd.Target == "" { - return errors.New("please specify a directory to restore to (--target)") - } - - if len(cmd.Exclude) > 0 && len(cmd.Include) > 0 { - return errors.New("exclude and include patterns are mutually exclusive") - } - - snapshotIDString := args[0] - - debug.Log("restore", "restore %v to %v", snapshotIDString, cmd.Target) - - repo, err := cmd.global.OpenRepository() - if err != nil { - return err - } - - if !cmd.global.NoLock { - lock, err := lockRepo(repo) - defer unlockRepo(lock) - if err != nil { - return err - } - } - - err = repo.LoadIndex() - if err != nil { - return err - } - - id, err := restic.FindSnapshot(repo, snapshotIDString) - if err != nil { - cmd.global.Exitf(1, "invalid id %q: %v", snapshotIDString, err) - } - - res, err := restic.NewRestorer(repo, id) - if err != nil { - cmd.global.Exitf(2, "creating restorer failed: %v\n", err) - } - - res.Error = func(dir string, node *restic.Node, err error) error { - cmd.global.Warnf("error for %s: %+v\n", dir, err) - return nil - } - - selectExcludeFilter := func(item string, dstpath string, node *restic.Node) bool { - matched, err := filter.List(cmd.Exclude, item) - if err != nil { - cmd.global.Warnf("error for exclude pattern: %v", err) - } - - return !matched - } - - selectIncludeFilter := func(item string, dstpath string, node *restic.Node) bool { - matched, err := filter.List(cmd.Include, item) - if err != nil { - cmd.global.Warnf("error for include pattern: %v", err) - } - - return matched - } - - if len(cmd.Exclude) > 0 { - res.SelectFilter = selectExcludeFilter - } else if len(cmd.Include) > 0 { - res.SelectFilter = selectIncludeFilter - } - - cmd.global.Verbosef("restoring %s to %s\n", res.Snapshot(), cmd.Target) - - err = res.RestoreTo(cmd.Target) - if err != nil { - return err - } - - return nil -} diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go deleted file mode 100644 index f1dcf72bd..000000000 --- a/cmd/restic/cmd_snapshots.go +++ /dev/null @@ -1,134 +0,0 @@ -package main - -import ( - "encoding/hex" - "fmt" - "io" - "os" - "sort" - "strings" - - "github.com/restic/restic" - "github.com/restic/restic/backend" -) - -type Table struct { - Header string - Rows [][]interface{} - - RowFormat string -} - -func NewTable() Table { - return Table{ - Rows: [][]interface{}{}, - } -} - -func (t Table) Write(w io.Writer) error { - _, err := fmt.Fprintln(w, t.Header) - if err != nil { - return err - } - _, err = fmt.Fprintln(w, strings.Repeat("-", 70)) - if err != nil { - return err - } - - for _, row := range t.Rows { - _, err = fmt.Fprintf(w, t.RowFormat+"\n", row...) - if err != nil { - return err - } - } - - return nil -} - -const TimeFormat = "2006-01-02 15:04:05" - -type CmdSnapshots struct { - global *GlobalOptions -} - -func init() { - _, err := parser.AddCommand("snapshots", - "show snapshots", - "The snapshots command lists all snapshots stored in a repository", - &CmdSnapshots{global: &globalOpts}) - if err != nil { - panic(err) - } -} - -func (cmd CmdSnapshots) Usage() string { - return "" -} - -func (cmd CmdSnapshots) Execute(args []string) error { - if len(args) != 0 { - return fmt.Errorf("wrong number of arguments, usage: %s", cmd.Usage()) - } - - repo, err := cmd.global.OpenRepository() - if err != nil { - return err - } - - lock, err := lockRepo(repo) - defer unlockRepo(lock) - if err != nil { - return err - } - - tab := NewTable() - tab.Header = fmt.Sprintf("%-8s %-19s %-10s %s", "ID", "Date", "Source", "Directory") - tab.RowFormat = "%-8s %-19s %-10s %s" - - done := make(chan struct{}) - defer close(done) - - list := []*restic.Snapshot{} - for id := range repo.List(backend.Snapshot, done) { - sn, err := restic.LoadSnapshot(repo, id) - if err != nil { - fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err) - continue - } - - pos := sort.Search(len(list), func(i int) bool { - return list[i].Time.After(sn.Time) - }) - - if pos < len(list) { - list = append(list, nil) - copy(list[pos+1:], list[pos:]) - list[pos] = sn - } else { - list = append(list, sn) - } - } - - plen, err := repo.PrefixLength(backend.Snapshot) - if err != nil { - return err - } - - for _, sn := range list { - if len(sn.Paths) == 0 { - continue - } - id := sn.ID() - tab.Rows = append(tab.Rows, []interface{}{hex.EncodeToString(id[:plen/2]), sn.Time.Format(TimeFormat), sn.Hostname, sn.Paths[0]}) - - if len(sn.Paths) > 1 { - for _, path := range sn.Paths[1:] { - tab.Rows = append(tab.Rows, []interface{}{"", "", "", path}) - } - } - } - - tab.Write(os.Stdout) - - return nil -} diff --git a/cmd/restic/cmd_unlock.go b/cmd/restic/cmd_unlock.go deleted file mode 100644 index 47345350c..000000000 --- a/cmd/restic/cmd_unlock.go +++ /dev/null @@ -1,43 +0,0 @@ -package main - -import "github.com/restic/restic" - -type CmdUnlock struct { - RemoveAll bool `long:"remove-all" description:"Remove all locks, even stale ones"` - - global *GlobalOptions -} - -func init() { - _, err := parser.AddCommand("unlock", - "remove locks", - "The unlock command checks for stale locks and removes them", - &CmdUnlock{global: &globalOpts}) - if err != nil { - panic(err) - } -} - -func (cmd CmdUnlock) Usage() string { - return "[unlock-options]" -} - -func (cmd CmdUnlock) Execute(args []string) error { - repo, err := cmd.global.OpenRepository() - if err != nil { - return err - } - - fn := restic.RemoveStaleLocks - if cmd.RemoveAll { - fn = restic.RemoveAllLocks - } - - err = fn(repo) - if err != nil { - return err - } - - cmd.global.Verbosef("successfully removed locks\n") - return nil -} diff --git a/cmd/restic/cmd_version.go b/cmd/restic/cmd_version.go deleted file mode 100644 index 5e64790a1..000000000 --- a/cmd/restic/cmd_version.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "fmt" - "runtime" -) - -type CmdVersion struct{} - -func init() { - _, err := parser.AddCommand("version", - "display version", - "The version command displays detailed information about the version", - &CmdVersion{}) - if err != nil { - panic(err) - } -} - -func (cmd CmdVersion) Execute(args []string) error { - fmt.Printf("restic %s\ncompiled at %s with %v\n", - version, compiledAt, runtime.Version()) - - return nil -} diff --git a/cmd/restic/doc.go b/cmd/restic/doc.go deleted file mode 100644 index 19b609b8d..000000000 --- a/cmd/restic/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// This package contains the code for the restic executable. -package main diff --git a/cmd/restic/global.go b/cmd/restic/global.go deleted file mode 100644 index 09ce319ab..000000000 --- a/cmd/restic/global.go +++ /dev/null @@ -1,287 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "io" - "os" - "strings" - "syscall" - - "github.com/jessevdk/go-flags" - "github.com/restic/restic/backend" - "github.com/restic/restic/backend/local" - "github.com/restic/restic/backend/s3" - "github.com/restic/restic/backend/sftp" - "github.com/restic/restic/debug" - "github.com/restic/restic/location" - "github.com/restic/restic/repository" - "golang.org/x/crypto/ssh/terminal" -) - -var version = "compiled manually" -var compiledAt = "unknown time" - -// GlobalOptions holds all those options that can be set for every command. -type GlobalOptions struct { - Repo string `short:"r" long:"repo" description:"Repository directory to backup to/restore from"` - CacheDir string ` long:"cache-dir" description:"Directory to use as a local cache"` - Quiet bool `short:"q" long:"quiet" default:"false" description:"Do not output comprehensive progress report"` - NoLock bool ` long:"no-lock" default:"false" description:"Do not lock the repo, this allows some operations on read-only repos."` - Options []string `short:"o" long:"option" description:"Specify options in the form 'foo.key=value'"` - - password string - stdout io.Writer - stderr io.Writer -} - -func init() { - restoreTerminal() -} - -// checkErrno returns nil when err is set to syscall.Errno(0), since this is no -// error condition. -func checkErrno(err error) error { - e, ok := err.(syscall.Errno) - if !ok { - return err - } - - if e == 0 { - return nil - } - - return err -} - -// restoreTerminal installs a cleanup handler that restores the previous -// terminal state on exit. -func restoreTerminal() { - fd := int(os.Stdout.Fd()) - if !terminal.IsTerminal(fd) { - return - } - - state, err := terminal.GetState(fd) - if err != nil { - fmt.Fprintf(os.Stderr, "unable to get terminal state: %v\n", err) - return - } - - AddCleanupHandler(func() error { - err := checkErrno(terminal.Restore(fd, state)) - if err != nil { - fmt.Fprintf(os.Stderr, "unable to get restore terminal state: %#+v\n", err) - } - return err - }) -} - -var globalOpts = GlobalOptions{stdout: os.Stdout, stderr: os.Stderr} -var parser = flags.NewParser(&globalOpts, flags.HelpFlag|flags.PassDoubleDash) - -// Printf writes the message to the configured stdout stream. -func (o GlobalOptions) Printf(format string, args ...interface{}) { - _, err := fmt.Fprintf(o.stdout, format, args...) - if err != nil { - fmt.Fprintf(os.Stderr, "unable to write to stdout: %v\n", err) - os.Exit(100) - } -} - -// Verbosef calls Printf to write the message when the verbose flag is set. -func (o GlobalOptions) Verbosef(format string, args ...interface{}) { - if o.Quiet { - return - } - - o.Printf(format, args...) -} - -// ShowProgress returns true iff the progress status should be written, i.e. -// the quiet flag is not set and the output is a terminal. -func (o GlobalOptions) ShowProgress() bool { - if o.Quiet { - return false - } - - if !terminal.IsTerminal(int(os.Stdout.Fd())) { - return false - } - - return true -} - -// Warnf writes the message to the configured stderr stream. -func (o GlobalOptions) Warnf(format string, args ...interface{}) { - _, err := fmt.Fprintf(o.stderr, format, args...) - if err != nil { - fmt.Fprintf(os.Stderr, "unable to write to stderr: %v\n", err) - os.Exit(100) - } -} - -// Exitf uses Warnf to write the message and then calls os.Exit(exitcode). -func (o GlobalOptions) Exitf(exitcode int, format string, args ...interface{}) { - if format[len(format)-1] != '\n' { - format += "\n" - } - - o.Warnf(format, args...) - os.Exit(exitcode) -} - -// readPassword reads the password from the given reader directly. -func readPassword(in io.Reader) (password string, err error) { - buf := make([]byte, 1000) - n, err := io.ReadFull(in, buf) - buf = buf[:n] - - if err != nil && err != io.ErrUnexpectedEOF { - return "", err - } - - return strings.TrimRight(string(buf), "\r\n"), nil -} - -// readPasswordTerminal reads the password from the given reader which must be a -// tty. Prompt is printed on the writer out before attempting to read the -// password. -func readPasswordTerminal(in *os.File, out io.Writer, prompt string) (password string, err error) { - fmt.Fprint(out, prompt) - buf, err := terminal.ReadPassword(int(in.Fd())) - fmt.Fprintln(out) - if err != nil { - return "", err - } - - password = string(buf) - return password, nil -} - -// ReadPassword reads the password from stdin. -func (o GlobalOptions) ReadPassword(prompt string) string { - var ( - password string - err error - ) - - if terminal.IsTerminal(int(os.Stdin.Fd())) { - password, err = readPasswordTerminal(os.Stdin, os.Stderr, prompt) - } else { - password, err = readPassword(os.Stdin) - } - - if err != nil { - o.Exitf(2, "unable to read password: %v", err) - } - - if len(password) == 0 { - o.Exitf(1, "an empty password is not a password") - } - - return password -} - -// ReadPasswordTwice calls ReadPassword two times and returns an error when the -// passwords don't match. -func (o GlobalOptions) ReadPasswordTwice(prompt1, prompt2 string) string { - pw1 := o.ReadPassword(prompt1) - pw2 := o.ReadPassword(prompt2) - if pw1 != pw2 { - o.Exitf(1, "passwords do not match") - } - - return pw1 -} - -// OpenRepository reads the password and opens the repository. -func (o GlobalOptions) OpenRepository() (*repository.Repository, error) { - if o.Repo == "" { - return nil, errors.New("Please specify repository location (-r)") - } - - be, err := open(o.Repo) - if err != nil { - return nil, err - } - - s := repository.New(be) - - if o.password == "" { - o.password = o.ReadPassword("enter password for repository: ") - } - - err = s.SearchKey(o.password) - if err != nil { - return nil, fmt.Errorf("unable to open repo: %v", err) - } - - return s, nil -} - -// Open the backend specified by a location config. -func open(s string) (backend.Backend, error) { - debug.Log("open", "parsing location %v", s) - loc, err := location.Parse(s) - if err != nil { - return nil, err - } - - switch loc.Scheme { - case "local": - debug.Log("open", "opening local repository at %#v", loc.Config) - return local.Open(loc.Config.(string)) - case "sftp": - debug.Log("open", "opening sftp repository at %#v", loc.Config) - return sftp.OpenWithConfig(loc.Config.(sftp.Config)) - case "s3": - cfg := loc.Config.(s3.Config) - if cfg.KeyID == "" { - cfg.KeyID = os.Getenv("AWS_ACCESS_KEY_ID") - - } - if cfg.Secret == "" { - cfg.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY") - } - - debug.Log("open", "opening s3 repository at %#v", cfg) - return s3.Open(cfg) - } - - debug.Log("open", "invalid repository location: %v", s) - return nil, fmt.Errorf("invalid scheme %q", loc.Scheme) -} - -// Create the backend specified by URI. -func create(s string) (backend.Backend, error) { - debug.Log("open", "parsing location %v", s) - loc, err := location.Parse(s) - if err != nil { - return nil, err - } - - switch loc.Scheme { - case "local": - debug.Log("open", "create local repository at %#v", loc.Config) - return local.Create(loc.Config.(string)) - case "sftp": - debug.Log("open", "create sftp repository at %#v", loc.Config) - return sftp.CreateWithConfig(loc.Config.(sftp.Config)) - case "s3": - cfg := loc.Config.(s3.Config) - if cfg.KeyID == "" { - cfg.KeyID = os.Getenv("AWS_ACCESS_KEY_ID") - - } - if cfg.Secret == "" { - cfg.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY") - } - - debug.Log("open", "create s3 repository at %#v", loc.Config) - return s3.Open(cfg) - } - - debug.Log("open", "invalid repository scheme: %v", s) - return nil, fmt.Errorf("invalid scheme %q", loc.Scheme) -} diff --git a/cmd/restic/integration_fuse_test.go b/cmd/restic/integration_fuse_test.go deleted file mode 100644 index 1e696706b..000000000 --- a/cmd/restic/integration_fuse_test.go +++ /dev/null @@ -1,162 +0,0 @@ -// +build !openbsd -// +build !windows - -package main - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "testing" - "time" - - "github.com/restic/restic" - "github.com/restic/restic/backend" - "github.com/restic/restic/repository" - . "github.com/restic/restic/test" -) - -const ( - mountWait = 20 - mountSleep = 100 * time.Millisecond - mountTestSubdir = "snapshots" -) - -// waitForMount blocks (max mountWait * mountSleep) until the subdir -// "snapshots" appears in the dir. -func waitForMount(dir string) error { - for i := 0; i < mountWait; i++ { - f, err := os.Open(dir) - if err != nil { - return err - } - - names, err := f.Readdirnames(-1) - if err != nil { - return err - } - - if err = f.Close(); err != nil { - return err - } - - for _, name := range names { - if name == mountTestSubdir { - return nil - } - } - - time.Sleep(mountSleep) - } - - return fmt.Errorf("subdir %q of dir %s never appeared", mountTestSubdir, dir) -} - -func cmdMount(t testing.TB, global GlobalOptions, dir string, ready, done chan struct{}) { - defer func() { - ready <- struct{}{} - }() - - cmd := &CmdMount{global: &global, ready: ready, done: done} - OK(t, cmd.Execute([]string{dir})) - if TestCleanupTempDirs { - RemoveAll(t, dir) - } -} - -func TestMount(t *testing.T) { - if !RunFuseTest { - t.Skip("Skipping fuse tests") - } - - checkSnapshots := func(repo *repository.Repository, mountpoint string, snapshotIDs []backend.ID) { - snapshotsDir, err := os.Open(filepath.Join(mountpoint, "snapshots")) - OK(t, err) - namesInSnapshots, err := snapshotsDir.Readdirnames(-1) - OK(t, err) - Assert(t, - len(namesInSnapshots) == len(snapshotIDs), - "Invalid number of snapshots: expected %d, got %d", len(snapshotIDs), len(namesInSnapshots)) - - namesMap := make(map[string]bool) - for _, name := range namesInSnapshots { - namesMap[name] = false - } - - for _, id := range snapshotIDs { - snapshot, err := restic.LoadSnapshot(repo, id) - OK(t, err) - _, ok := namesMap[snapshot.Time.Format(time.RFC3339)] - Assert(t, ok, "Snapshot %s isn't present in fuse dir", snapshot.Time.Format(time.RFC3339)) - namesMap[snapshot.Time.Format(time.RFC3339)] = true - } - for name, present := range namesMap { - Assert(t, present, "Directory %s is present in fuse dir but is not a snapshot", name) - } - OK(t, snapshotsDir.Close()) - } - - withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) { - cmdInit(t, global) - repo, err := global.OpenRepository() - OK(t, err) - - mountpoint, err := ioutil.TempDir(TestTempDir, "restic-test-mount-") - OK(t, err) - - // We remove the mountpoint now to check that cmdMount creates it - RemoveAll(t, mountpoint) - - ready := make(chan struct{}, 2) - done := make(chan struct{}) - go cmdMount(t, global, mountpoint, ready, done) - <-ready - defer close(done) - OK(t, waitForMount(mountpoint)) - - mountpointDir, err := os.Open(mountpoint) - OK(t, err) - names, err := mountpointDir.Readdirnames(-1) - OK(t, err) - Assert(t, len(names) == 1 && names[0] == "snapshots", `The fuse virtual directory "snapshots" doesn't exist`) - OK(t, mountpointDir.Close()) - - checkSnapshots(repo, mountpoint, []backend.ID{}) - - datafile := filepath.Join("testdata", "backup-data.tar.gz") - fd, err := os.Open(datafile) - if os.IsNotExist(err) { - t.Skipf("unable to find data file %q, skipping", datafile) - return - } - OK(t, err) - OK(t, fd.Close()) - - SetupTarTestFixture(t, env.testdata, datafile) - - // first backup - cmdBackup(t, global, []string{env.testdata}, nil) - snapshotIDs := cmdList(t, global, "snapshots") - Assert(t, len(snapshotIDs) == 1, - "expected one snapshot, got %v", snapshotIDs) - - checkSnapshots(repo, mountpoint, snapshotIDs) - - // second backup, implicit incremental - cmdBackup(t, global, []string{env.testdata}, nil) - snapshotIDs = cmdList(t, global, "snapshots") - Assert(t, len(snapshotIDs) == 2, - "expected two snapshots, got %v", snapshotIDs) - - checkSnapshots(repo, mountpoint, snapshotIDs) - - // third backup, explicit incremental - cmdBackup(t, global, []string{env.testdata}, &snapshotIDs[0]) - snapshotIDs = cmdList(t, global, "snapshots") - Assert(t, len(snapshotIDs) == 3, - "expected three snapshots, got %v", snapshotIDs) - - checkSnapshots(repo, mountpoint, snapshotIDs) - }) -} diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go deleted file mode 100644 index b3bada889..000000000 --- a/cmd/restic/integration_helpers_test.go +++ /dev/null @@ -1,228 +0,0 @@ -package main - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "runtime" - "testing" - - . "github.com/restic/restic/test" -) - -type dirEntry struct { - path string - fi os.FileInfo -} - -func walkDir(dir string) <-chan *dirEntry { - ch := make(chan *dirEntry, 100) - - go func() { - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - return nil - } - - name, err := filepath.Rel(dir, path) - if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - return nil - } - - ch <- &dirEntry{ - path: name, - fi: info, - } - - return nil - }) - - if err != nil { - fmt.Fprintf(os.Stderr, "Walk() error: %v\n", err) - } - - close(ch) - }() - - // first element is root - _ = <-ch - - return ch -} - -func isSymlink(fi os.FileInfo) bool { - mode := fi.Mode() & (os.ModeType | os.ModeCharDevice) - return mode == os.ModeSymlink -} - -func sameModTime(fi1, fi2 os.FileInfo) bool { - switch runtime.GOOS { - case "darwin", "freebsd", "openbsd": - if isSymlink(fi1) && isSymlink(fi2) { - return true - } - } - - return fi1.ModTime() == fi2.ModTime() -} - -// directoriesEqualContents checks if both directories contain exactly the same -// contents. -func directoriesEqualContents(dir1, dir2 string) bool { - ch1 := walkDir(dir1) - ch2 := walkDir(dir2) - - changes := false - - var a, b *dirEntry - for { - var ok bool - - if ch1 != nil && a == nil { - a, ok = <-ch1 - if !ok { - ch1 = nil - } - } - - if ch2 != nil && b == nil { - b, ok = <-ch2 - if !ok { - ch2 = nil - } - } - - if ch1 == nil && ch2 == nil { - break - } - - if ch1 == nil { - fmt.Printf("+%v\n", b.path) - changes = true - } else if ch2 == nil { - fmt.Printf("-%v\n", a.path) - changes = true - } else if !a.equals(b) { - if a.path < b.path { - fmt.Printf("-%v\n", a.path) - changes = true - a = nil - continue - } else if a.path > b.path { - fmt.Printf("+%v\n", b.path) - changes = true - b = nil - continue - } else { - fmt.Printf("%%%v\n", a.path) - changes = true - } - } - - a, b = nil, nil - } - - if changes { - return false - } - - return true -} - -type dirStat struct { - files, dirs, other uint - size uint64 -} - -func isFile(fi os.FileInfo) bool { - return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0 -} - -// dirStats walks dir and collects stats. -func dirStats(dir string) (stat dirStat) { - for entry := range walkDir(dir) { - if isFile(entry.fi) { - stat.files++ - stat.size += uint64(entry.fi.Size()) - continue - } - - if entry.fi.IsDir() { - stat.dirs++ - continue - } - - stat.other++ - } - - return stat -} - -type testEnvironment struct { - base, cache, repo, testdata string -} - -func configureRestic(t testing.TB, cache, repo string) GlobalOptions { - return GlobalOptions{ - CacheDir: cache, - Repo: repo, - Quiet: true, - - password: TestPassword, - stdout: os.Stdout, - stderr: os.Stderr, - } -} - -func cleanupTempdir(t testing.TB, tempdir string) { - if !TestCleanupTempDirs { - t.Logf("leaving temporary directory %v used for test", tempdir) - return - } - - RemoveAll(t, tempdir) -} - -// withTestEnvironment creates a test environment and calls f with it. After f has -// returned, the temporary directory is removed. -func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions)) { - if !RunIntegrationTest { - t.Skip("integration tests disabled") - } - - tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-") - OK(t, err) - - env := testEnvironment{ - base: tempdir, - cache: filepath.Join(tempdir, "cache"), - repo: filepath.Join(tempdir, "repo"), - testdata: filepath.Join(tempdir, "testdata"), - } - - OK(t, os.MkdirAll(env.testdata, 0700)) - OK(t, os.MkdirAll(env.cache, 0700)) - OK(t, os.MkdirAll(env.repo, 0700)) - - f(&env, configureRestic(t, env.cache, env.repo)) - - if !TestCleanupTempDirs { - t.Logf("leaving temporary directory %v used for test", tempdir) - return - } - - RemoveAll(t, tempdir) -} - -// removeFile resets the read-only flag and then deletes the file. -func removeFile(fn string) error { - err := os.Chmod(fn, 0666) - if err != nil { - return err - } - - return os.Remove(fn) -} diff --git a/cmd/restic/integration_helpers_unix_test.go b/cmd/restic/integration_helpers_unix_test.go deleted file mode 100644 index a182898e8..000000000 --- a/cmd/restic/integration_helpers_unix_test.go +++ /dev/null @@ -1,41 +0,0 @@ -//+build !windows - -package main - -import ( - "fmt" - "os" - "syscall" -) - -func (e *dirEntry) equals(other *dirEntry) bool { - if e.path != other.path { - fmt.Fprintf(os.Stderr, "%v: path does not match (%v != %v)\n", e.path, e.path, other.path) - return false - } - - if e.fi.Mode() != other.fi.Mode() { - fmt.Fprintf(os.Stderr, "%v: mode does not match (%v != %v)\n", e.path, e.fi.Mode(), other.fi.Mode()) - return false - } - - if !sameModTime(e.fi, other.fi) { - fmt.Fprintf(os.Stderr, "%v: ModTime does not match (%v != %v)\n", e.path, e.fi.ModTime(), other.fi.ModTime()) - return false - } - - stat, _ := e.fi.Sys().(*syscall.Stat_t) - stat2, _ := other.fi.Sys().(*syscall.Stat_t) - - if stat.Uid != stat2.Uid { - fmt.Fprintf(os.Stderr, "%v: UID does not match (%v != %v)\n", e.path, stat.Uid, stat2.Uid) - return false - } - - if stat.Gid != stat2.Gid { - fmt.Fprintf(os.Stderr, "%v: GID does not match (%v != %v)\n", e.path, stat.Gid, stat2.Gid) - return false - } - - return true -} diff --git a/cmd/restic/integration_helpers_windows_test.go b/cmd/restic/integration_helpers_windows_test.go deleted file mode 100644 index d67e9ca11..000000000 --- a/cmd/restic/integration_helpers_windows_test.go +++ /dev/null @@ -1,27 +0,0 @@ -//+build windows - -package main - -import ( - "fmt" - "os" -) - -func (e *dirEntry) equals(other *dirEntry) bool { - if e.path != other.path { - fmt.Fprintf(os.Stderr, "%v: path does not match (%v != %v)\n", e.path, e.path, other.path) - return false - } - - if e.fi.Mode() != other.fi.Mode() { - fmt.Fprintf(os.Stderr, "%v: mode does not match (%v != %v)\n", e.path, e.fi.Mode(), other.fi.Mode()) - return false - } - - if !sameModTime(e.fi, other.fi) { - fmt.Fprintf(os.Stderr, "%v: ModTime does not match (%v != %v)\n", e.path, e.fi.ModTime(), other.fi.ModTime()) - return false - } - - return true -} diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go deleted file mode 100644 index bc734bf0a..000000000 --- a/cmd/restic/integration_test.go +++ /dev/null @@ -1,816 +0,0 @@ -package main - -import ( - "bufio" - "bytes" - "crypto/rand" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "regexp" - "strings" - "syscall" - "testing" - "time" - - "github.com/restic/restic/backend" - "github.com/restic/restic/debug" - "github.com/restic/restic/filter" - "github.com/restic/restic/repository" - . "github.com/restic/restic/test" -) - -func parseIDsFromReader(t testing.TB, rd io.Reader) backend.IDs { - IDs := backend.IDs{} - sc := bufio.NewScanner(rd) - - for sc.Scan() { - id, err := backend.ParseID(sc.Text()) - if err != nil { - t.Logf("parse id %v: %v", sc.Text(), err) - continue - } - - IDs = append(IDs, id) - } - - return IDs -} - -func cmdInit(t testing.TB, global GlobalOptions) { - cmd := &CmdInit{global: &global} - OK(t, cmd.Execute(nil)) - - t.Logf("repository initialized at %v", global.Repo) -} - -func cmdBackup(t testing.TB, global GlobalOptions, target []string, parentID *backend.ID) { - cmdBackupExcludes(t, global, target, parentID, nil) -} - -func cmdBackupExcludes(t testing.TB, global GlobalOptions, target []string, parentID *backend.ID, excludes []string) { - cmd := &CmdBackup{global: &global, Excludes: excludes} - if parentID != nil { - cmd.Parent = parentID.String() - } - - t.Logf("backing up %v", target) - - OK(t, cmd.Execute(target)) -} - -func cmdList(t testing.TB, global GlobalOptions, tpe string) backend.IDs { - cmd := &CmdList{global: &global} - return executeAndParseIDs(t, cmd, tpe) -} - -func executeAndParseIDs(t testing.TB, cmd *CmdList, args ...string) backend.IDs { - buf := bytes.NewBuffer(nil) - cmd.global.stdout = buf - OK(t, cmd.Execute(args)) - return parseIDsFromReader(t, buf) -} - -func cmdRestore(t testing.TB, global GlobalOptions, dir string, snapshotID backend.ID) { - cmdRestoreExcludes(t, global, dir, snapshotID, nil) -} - -func cmdRestoreExcludes(t testing.TB, global GlobalOptions, dir string, snapshotID backend.ID, excludes []string) { - cmd := &CmdRestore{global: &global, Target: dir, Exclude: excludes} - OK(t, cmd.Execute([]string{snapshotID.String()})) -} - -func cmdRestoreIncludes(t testing.TB, global GlobalOptions, dir string, snapshotID backend.ID, includes []string) { - cmd := &CmdRestore{global: &global, Target: dir, Include: includes} - OK(t, cmd.Execute([]string{snapshotID.String()})) -} - -func cmdCheck(t testing.TB, global GlobalOptions) { - cmd := &CmdCheck{ - global: &global, - ReadData: true, - CheckUnused: true, - } - OK(t, cmd.Execute(nil)) -} - -func cmdCheckOutput(t testing.TB, global GlobalOptions) string { - buf := bytes.NewBuffer(nil) - global.stdout = buf - cmd := &CmdCheck{global: &global, ReadData: true} - OK(t, cmd.Execute(nil)) - return string(buf.Bytes()) -} - -func cmdRebuildIndex(t testing.TB, global GlobalOptions) { - global.stdout = ioutil.Discard - cmd := &CmdRebuildIndex{global: &global} - OK(t, cmd.Execute(nil)) -} - -func cmdOptimize(t testing.TB, global GlobalOptions) { - cmd := &CmdOptimize{global: &global} - OK(t, cmd.Execute(nil)) -} - -func cmdLs(t testing.TB, global GlobalOptions, snapshotID string) []string { - var buf bytes.Buffer - global.stdout = &buf - - cmd := &CmdLs{global: &global} - OK(t, cmd.Execute([]string{snapshotID})) - - return strings.Split(string(buf.Bytes()), "\n") -} - -func cmdFind(t testing.TB, global GlobalOptions, pattern string) []string { - var buf bytes.Buffer - global.stdout = &buf - - cmd := &CmdFind{global: &global} - OK(t, cmd.Execute([]string{pattern})) - - return strings.Split(string(buf.Bytes()), "\n") -} - -func TestBackup(t *testing.T) { - withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) { - datafile := filepath.Join("testdata", "backup-data.tar.gz") - fd, err := os.Open(datafile) - if os.IsNotExist(err) { - t.Skipf("unable to find data file %q, skipping", datafile) - return - } - OK(t, err) - OK(t, fd.Close()) - - cmdInit(t, global) - - SetupTarTestFixture(t, env.testdata, datafile) - - // first backup - cmdBackup(t, global, []string{env.testdata}, nil) - snapshotIDs := cmdList(t, global, "snapshots") - Assert(t, len(snapshotIDs) == 1, - "expected one snapshot, got %v", snapshotIDs) - - cmdCheck(t, global) - stat1 := dirStats(env.repo) - - // second backup, implicit incremental - cmdBackup(t, global, []string{env.testdata}, nil) - snapshotIDs = cmdList(t, global, "snapshots") - Assert(t, len(snapshotIDs) == 2, - "expected two snapshots, got %v", snapshotIDs) - - stat2 := dirStats(env.repo) - if stat2.size > stat1.size+stat1.size/10 { - t.Error("repository size has grown by more than 10 percent") - } - t.Logf("repository grown by %d bytes", stat2.size-stat1.size) - - cmdCheck(t, global) - // third backup, explicit incremental - cmdBackup(t, global, []string{env.testdata}, &snapshotIDs[0]) - snapshotIDs = cmdList(t, global, "snapshots") - Assert(t, len(snapshotIDs) == 3, - "expected three snapshots, got %v", snapshotIDs) - - stat3 := dirStats(env.repo) - if stat3.size > stat1.size+stat1.size/10 { - t.Error("repository size has grown by more than 10 percent") - } - t.Logf("repository grown by %d bytes", stat3.size-stat2.size) - - // restore all backups and compare - for i, snapshotID := range snapshotIDs { - restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i)) - t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir) - cmdRestore(t, global, restoredir, snapshotIDs[0]) - Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, "testdata")), - "directories are not equal") - } - - cmdCheck(t, global) - }) -} - -func TestBackupNonExistingFile(t *testing.T) { - withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) { - datafile := filepath.Join("testdata", "backup-data.tar.gz") - fd, err := os.Open(datafile) - if os.IsNotExist(err) { - t.Skipf("unable to find data file %q, skipping", datafile) - return - } - OK(t, err) - OK(t, fd.Close()) - - SetupTarTestFixture(t, env.testdata, datafile) - - cmdInit(t, global) - - global.stderr = ioutil.Discard - - p := filepath.Join(env.testdata, "0", "0") - dirs := []string{ - filepath.Join(p, "0"), - filepath.Join(p, "1"), - filepath.Join(p, "nonexisting"), - filepath.Join(p, "5"), - } - cmdBackup(t, global, dirs, nil) - }) -} - -func TestBackupMissingFile1(t *testing.T) { - withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) { - datafile := filepath.Join("testdata", "backup-data.tar.gz") - fd, err := os.Open(datafile) - if os.IsNotExist(err) { - t.Skipf("unable to find data file %q, skipping", datafile) - return - } - OK(t, err) - OK(t, fd.Close()) - - SetupTarTestFixture(t, env.testdata, datafile) - - cmdInit(t, global) - - global.stderr = ioutil.Discard - ranHook := false - debug.Hook("pipe.walk1", func(context interface{}) { - pathname := context.(string) - - if pathname != filepath.Join("testdata", "0", "0", "9") { - return - } - - t.Logf("in hook, removing test file testdata/0/0/9/37") - ranHook = true - - OK(t, os.Remove(filepath.Join(env.testdata, "0", "0", "9", "37"))) - }) - - cmdBackup(t, global, []string{env.testdata}, nil) - cmdCheck(t, global) - - Assert(t, ranHook, "hook did not run") - debug.RemoveHook("pipe.walk1") - }) -} - -func TestBackupMissingFile2(t *testing.T) { - withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) { - datafile := filepath.Join("testdata", "backup-data.tar.gz") - fd, err := os.Open(datafile) - if os.IsNotExist(err) { - t.Skipf("unable to find data file %q, skipping", datafile) - return - } - OK(t, err) - OK(t, fd.Close()) - - SetupTarTestFixture(t, env.testdata, datafile) - - cmdInit(t, global) - - global.stderr = ioutil.Discard - ranHook := false - debug.Hook("pipe.walk2", func(context interface{}) { - pathname := context.(string) - - if pathname != filepath.Join("testdata", "0", "0", "9", "37") { - return - } - - t.Logf("in hook, removing test file testdata/0/0/9/37") - ranHook = true - - OK(t, os.Remove(filepath.Join(env.testdata, "0", "0", "9", "37"))) - }) - - cmdBackup(t, global, []string{env.testdata}, nil) - cmdCheck(t, global) - - Assert(t, ranHook, "hook did not run") - debug.RemoveHook("pipe.walk2") - }) -} - -func TestBackupDirectoryError(t *testing.T) { - withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) { - datafile := filepath.Join("testdata", "backup-data.tar.gz") - fd, err := os.Open(datafile) - if os.IsNotExist(err) { - t.Skipf("unable to find data file %q, skipping", datafile) - return - } - OK(t, err) - OK(t, fd.Close()) - - SetupTarTestFixture(t, env.testdata, datafile) - - cmdInit(t, global) - - global.stderr = ioutil.Discard - ranHook := false - - testdir := filepath.Join(env.testdata, "0", "0", "9") - - // install hook that removes the dir right before readdirnames() - debug.Hook("pipe.readdirnames", func(context interface{}) { - path := context.(string) - - if path != testdir { - return - } - - t.Logf("in hook, removing test file %v", testdir) - ranHook = true - - OK(t, os.RemoveAll(testdir)) - }) - - cmdBackup(t, global, []string{filepath.Join(env.testdata, "0", "0")}, nil) - cmdCheck(t, global) - - Assert(t, ranHook, "hook did not run") - debug.RemoveHook("pipe.walk2") - - snapshots := cmdList(t, global, "snapshots") - Assert(t, len(snapshots) > 0, - "no snapshots found in repo (%v)", datafile) - - files := cmdLs(t, global, snapshots[0].String()) - - Assert(t, len(files) > 1, "snapshot is empty") - }) -} - -func includes(haystack []string, needle string) bool { - for _, s := range haystack { - if s == needle { - return true - } - } - - return false -} - -func loadSnapshotMap(t testing.TB, global GlobalOptions) map[string]struct{} { - snapshotIDs := cmdList(t, global, "snapshots") - - m := make(map[string]struct{}) - for _, id := range snapshotIDs { - m[id.String()] = struct{}{} - } - - return m -} - -func lastSnapshot(old, new map[string]struct{}) (map[string]struct{}, string) { - for k := range new { - if _, ok := old[k]; !ok { - old[k] = struct{}{} - return old, k - } - } - - return old, "" -} - -var backupExcludeFilenames = []string{ - "testfile1", - "foo.tar.gz", - "private/secret/passwords.txt", - "work/source/test.c", -} - -func TestBackupExclude(t *testing.T) { - withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) { - cmdInit(t, global) - - datadir := filepath.Join(env.base, "testdata") - - for _, filename := range backupExcludeFilenames { - fp := filepath.Join(datadir, filename) - OK(t, os.MkdirAll(filepath.Dir(fp), 0755)) - - f, err := os.Create(fp) - OK(t, err) - - fmt.Fprintf(f, filename) - OK(t, f.Close()) - } - - snapshots := make(map[string]struct{}) - - cmdBackup(t, global, []string{datadir}, nil) - snapshots, snapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, global)) - files := cmdLs(t, global, snapshotID) - Assert(t, includes(files, filepath.Join("testdata", "foo.tar.gz")), - "expected file %q in first snapshot, but it's not included", "foo.tar.gz") - - cmdBackupExcludes(t, global, []string{datadir}, nil, []string{"*.tar.gz"}) - snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, global)) - files = cmdLs(t, global, snapshotID) - Assert(t, !includes(files, filepath.Join("testdata", "foo.tar.gz")), - "expected file %q not in first snapshot, but it's included", "foo.tar.gz") - - cmdBackupExcludes(t, global, []string{datadir}, nil, []string{"*.tar.gz", "private/secret"}) - snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, global)) - files = cmdLs(t, global, snapshotID) - Assert(t, !includes(files, filepath.Join("testdata", "foo.tar.gz")), - "expected file %q not in first snapshot, but it's included", "foo.tar.gz") - Assert(t, !includes(files, filepath.Join("testdata", "private", "secret", "passwords.txt")), - "expected file %q not in first snapshot, but it's included", "passwords.txt") - }) -} - -const ( - incrementalFirstWrite = 20 * 1042 * 1024 - incrementalSecondWrite = 12 * 1042 * 1024 - incrementalThirdWrite = 4 * 1042 * 1024 -) - -func appendRandomData(filename string, bytes uint) error { - f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0666) - if err != nil { - fmt.Fprint(os.Stderr, err) - return err - } - - _, err = f.Seek(0, 2) - if err != nil { - fmt.Fprint(os.Stderr, err) - return err - } - - _, err = io.Copy(f, io.LimitReader(rand.Reader, int64(bytes))) - if err != nil { - fmt.Fprint(os.Stderr, err) - return err - } - - return f.Close() -} - -func TestIncrementalBackup(t *testing.T) { - withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) { - cmdInit(t, global) - - datadir := filepath.Join(env.base, "testdata") - testfile := filepath.Join(datadir, "testfile") - - OK(t, appendRandomData(testfile, incrementalFirstWrite)) - - cmdBackup(t, global, []string{datadir}, nil) - cmdCheck(t, global) - stat1 := dirStats(env.repo) - - OK(t, appendRandomData(testfile, incrementalSecondWrite)) - - cmdBackup(t, global, []string{datadir}, nil) - cmdCheck(t, global) - stat2 := dirStats(env.repo) - if stat2.size-stat1.size > incrementalFirstWrite { - t.Errorf("repository size has grown by more than %d bytes", incrementalFirstWrite) - } - t.Logf("repository grown by %d bytes", stat2.size-stat1.size) - - OK(t, appendRandomData(testfile, incrementalThirdWrite)) - - cmdBackup(t, global, []string{datadir}, nil) - cmdCheck(t, global) - stat3 := dirStats(env.repo) - if stat3.size-stat2.size > incrementalFirstWrite { - t.Errorf("repository size has grown by more than %d bytes", incrementalFirstWrite) - } - t.Logf("repository grown by %d bytes", stat3.size-stat2.size) - }) -} - -func cmdKey(t testing.TB, global GlobalOptions, args ...string) string { - var buf bytes.Buffer - - global.stdout = &buf - cmd := &CmdKey{global: &global} - OK(t, cmd.Execute(args)) - - return buf.String() -} - -func cmdKeyListOtherIDs(t testing.TB, global GlobalOptions) []string { - var buf bytes.Buffer - - global.stdout = &buf - cmd := &CmdKey{global: &global} - OK(t, cmd.Execute([]string{"list"})) - - scanner := bufio.NewScanner(&buf) - exp := regexp.MustCompile(`^ ([a-f0-9]+) `) - - IDs := []string{} - for scanner.Scan() { - if id := exp.FindStringSubmatch(scanner.Text()); id != nil { - IDs = append(IDs, id[1]) - } - } - - return IDs -} - -func cmdKeyAddNewKey(t testing.TB, global GlobalOptions, newPassword string) { - cmd := &CmdKey{global: &global, newPassword: newPassword} - OK(t, cmd.Execute([]string{"add"})) -} - -func cmdKeyPasswd(t testing.TB, global GlobalOptions, newPassword string) { - cmd := &CmdKey{global: &global, newPassword: newPassword} - OK(t, cmd.Execute([]string{"passwd"})) -} - -func cmdKeyRemove(t testing.TB, global GlobalOptions, IDs []string) { - cmd := &CmdKey{global: &global} - t.Logf("remove %d keys: %q\n", len(IDs), IDs) - for _, id := range IDs { - OK(t, cmd.Execute([]string{"rm", id})) - } -} - -func TestKeyAddRemove(t *testing.T) { - passwordList := []string{ - "OnnyiasyatvodsEvVodyawit", - "raicneirvOjEfEigonOmLasOd", - } - - withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) { - cmdInit(t, global) - - cmdKeyPasswd(t, global, "geheim2") - global.password = "geheim2" - t.Logf("changed password to %q", global.password) - - for _, newPassword := range passwordList { - cmdKeyAddNewKey(t, global, newPassword) - t.Logf("added new password %q", newPassword) - global.password = newPassword - cmdKeyRemove(t, global, cmdKeyListOtherIDs(t, global)) - } - - global.password = passwordList[len(passwordList)-1] - t.Logf("testing access with last password %q\n", global.password) - cmdKey(t, global, "list") - - cmdCheck(t, global) - }) -} - -func testFileSize(filename string, size int64) error { - fi, err := os.Stat(filename) - if err != nil { - return err - } - - if fi.Size() != size { - return fmt.Errorf("wrong file size for %v: expected %v, got %v", filename, size, fi.Size()) - } - - return nil -} - -func TestRestoreFilter(t *testing.T) { - testfiles := []struct { - name string - size uint - }{ - {"testfile1.c", 100}, - {"testfile2.exe", 101}, - {"subdir1/subdir2/testfile3.docx", 102}, - {"subdir1/subdir2/testfile4.c", 102}, - } - - withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) { - cmdInit(t, global) - - for _, test := range testfiles { - p := filepath.Join(env.testdata, test.name) - OK(t, os.MkdirAll(filepath.Dir(p), 0755)) - OK(t, appendRandomData(p, test.size)) - } - - cmdBackup(t, global, []string{env.testdata}, nil) - cmdCheck(t, global) - - snapshotID := cmdList(t, global, "snapshots")[0] - - // no restore filter should restore all files - cmdRestore(t, global, filepath.Join(env.base, "restore0"), snapshotID) - for _, test := range testfiles { - OK(t, testFileSize(filepath.Join(env.base, "restore0", "testdata", test.name), int64(test.size))) - } - - for i, pat := range []string{"*.c", "*.exe", "*", "*file3*"} { - base := filepath.Join(env.base, fmt.Sprintf("restore%d", i+1)) - cmdRestoreExcludes(t, global, base, snapshotID, []string{pat}) - for _, test := range testfiles { - err := testFileSize(filepath.Join(base, "testdata", test.name), int64(test.size)) - if ok, _ := filter.Match(pat, filepath.Base(test.name)); !ok { - OK(t, err) - } else { - Assert(t, os.IsNotExist(err), - "expected %v to not exist in restore step %v, but it exists, err %v", test.name, i+1, err) - } - } - } - - }) -} - -func TestRestoreWithPermissionFailure(t *testing.T) { - withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) { - datafile := filepath.Join("testdata", "repo-restore-permissions-test.tar.gz") - SetupTarTestFixture(t, env.base, datafile) - - snapshots := cmdList(t, global, "snapshots") - Assert(t, len(snapshots) > 0, - "no snapshots found in repo (%v)", datafile) - - global.stderr = ioutil.Discard - cmdRestore(t, global, filepath.Join(env.base, "restore"), snapshots[0]) - - // make sure that all files have been restored, regardeless of any - // permission errors - files := cmdLs(t, global, snapshots[0].String()) - for _, filename := range files { - fi, err := os.Lstat(filepath.Join(env.base, "restore", filename)) - OK(t, err) - - Assert(t, !isFile(fi) || fi.Size() > 0, - "file %v restored, but filesize is 0", filename) - } - }) -} - -func setZeroModTime(filename string) error { - var utimes = []syscall.Timespec{ - syscall.NsecToTimespec(0), - syscall.NsecToTimespec(0), - } - - return syscall.UtimesNano(filename, utimes) -} - -func TestRestoreNoMetadataOnIgnoredIntermediateDirs(t *testing.T) { - withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) { - cmdInit(t, global) - - p := filepath.Join(env.testdata, "subdir1", "subdir2", "subdir3", "file.ext") - OK(t, os.MkdirAll(filepath.Dir(p), 0755)) - OK(t, appendRandomData(p, 200)) - OK(t, setZeroModTime(filepath.Join(env.testdata, "subdir1", "subdir2"))) - - cmdBackup(t, global, []string{env.testdata}, nil) - cmdCheck(t, global) - - snapshotID := cmdList(t, global, "snapshots")[0] - - // restore with filter "*.ext", this should restore "file.ext", but - // since the directories are ignored and only created because of - // "file.ext", no meta data should be restored for them. - cmdRestoreIncludes(t, global, filepath.Join(env.base, "restore0"), snapshotID, []string{"*.ext"}) - - f1 := filepath.Join(env.base, "restore0", "testdata", "subdir1", "subdir2") - fi, err := os.Stat(f1) - OK(t, err) - - Assert(t, fi.ModTime() != time.Unix(0, 0), - "meta data of intermediate directory has been restore although it was ignored") - - // restore with filter "*", this should restore meta data on everything. - cmdRestoreIncludes(t, global, filepath.Join(env.base, "restore1"), snapshotID, []string{"*"}) - - f2 := filepath.Join(env.base, "restore1", "testdata", "subdir1", "subdir2") - fi, err = os.Stat(f2) - OK(t, err) - - Assert(t, fi.ModTime() == time.Unix(0, 0), - "meta data of intermediate directory hasn't been restore") - }) -} - -func TestFind(t *testing.T) { - withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) { - datafile := filepath.Join("testdata", "backup-data.tar.gz") - cmdInit(t, global) - SetupTarTestFixture(t, env.testdata, datafile) - cmdBackup(t, global, []string{env.testdata}, nil) - cmdCheck(t, global) - - results := cmdFind(t, global, "unexistingfile") - Assert(t, len(results) != 0, "unexisting file found in repo (%v)", datafile) - - results = cmdFind(t, global, "testfile") - Assert(t, len(results) != 1, "file not found in repo (%v)", datafile) - - results = cmdFind(t, global, "test") - Assert(t, len(results) < 2, "less than two file found in repo (%v)", datafile) - }) -} - -func TestRebuildIndex(t *testing.T) { - withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) { - datafile := filepath.Join("..", "..", "checker", "testdata", "duplicate-packs-in-index-test-repo.tar.gz") - SetupTarTestFixture(t, env.base, datafile) - - out := cmdCheckOutput(t, global) - if !strings.Contains(out, "contained in several indexes") { - t.Fatalf("did not find checker hint for packs in several indexes") - } - - if !strings.Contains(out, "restic rebuild-index") { - t.Fatalf("did not find hint for rebuild-index comman") - } - - cmdRebuildIndex(t, global) - - out = cmdCheckOutput(t, global) - if len(out) != 0 { - t.Fatalf("expected no output from the checker, got: %v", out) - } - }) -} - -func TestRebuildIndexAlwaysFull(t *testing.T) { - repository.IndexFull = func(*repository.Index) bool { return true } - TestRebuildIndex(t) -} - -var optimizeTests = []struct { - testFilename string - snapshots backend.IDSet -}{ - { - filepath.Join("..", "..", "checker", "testdata", "checker-test-repo.tar.gz"), - backend.NewIDSet(ParseID("a13c11e582b77a693dd75ab4e3a3ba96538a056594a4b9076e4cacebe6e06d43")), - }, - { - filepath.Join("testdata", "old-index-repo.tar.gz"), - nil, - }, - { - filepath.Join("testdata", "old-index-repo.tar.gz"), - backend.NewIDSet( - ParseID("f7d83db709977178c9d1a09e4009355e534cde1a135b8186b8b118a3fc4fcd41"), - ParseID("51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02"), - ), - }, -} - -func TestOptimizeRemoveUnusedBlobs(t *testing.T) { - for i, test := range optimizeTests { - withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) { - SetupTarTestFixture(t, env.base, test.testFilename) - - for id := range test.snapshots { - OK(t, removeFile(filepath.Join(env.repo, "snapshots", id.String()))) - } - - cmdOptimize(t, global) - output := cmdCheckOutput(t, global) - - if len(output) > 0 { - t.Errorf("expected no output for check in test %d, got:\n%v", i, output) - } - }) - } -} - -func TestCheckRestoreNoLock(t *testing.T) { - withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) { - datafile := filepath.Join("testdata", "small-repo.tar.gz") - SetupTarTestFixture(t, env.base, datafile) - - err := filepath.Walk(env.repo, func(p string, fi os.FileInfo, e error) error { - if e != nil { - return e - } - return os.Chmod(p, fi.Mode() & ^(os.FileMode(0222))) - }) - OK(t, err) - - global.NoLock = true - cmdCheck(t, global) - - snapshotIDs := cmdList(t, global, "snapshots") - if len(snapshotIDs) == 0 { - t.Fatalf("found no snapshots") - } - - cmdRestore(t, global, filepath.Join(env.base, "restore"), snapshotIDs[0]) - }) -} diff --git a/cmd/restic/lock.go b/cmd/restic/lock.go deleted file mode 100644 index ec4368003..000000000 --- a/cmd/restic/lock.go +++ /dev/null @@ -1,125 +0,0 @@ -package main - -import ( - "fmt" - "os" - "sync" - "time" - - "github.com/restic/restic" - "github.com/restic/restic/debug" - "github.com/restic/restic/repository" -) - -var globalLocks struct { - locks []*restic.Lock - cancelRefresh chan struct{} - refreshWG sync.WaitGroup - sync.Mutex -} - -func lockRepo(repo *repository.Repository) (*restic.Lock, error) { - return lockRepository(repo, false) -} - -func lockRepoExclusive(repo *repository.Repository) (*restic.Lock, error) { - return lockRepository(repo, true) -} - -func lockRepository(repo *repository.Repository, exclusive bool) (*restic.Lock, error) { - lockFn := restic.NewLock - if exclusive { - lockFn = restic.NewExclusiveLock - } - - lock, err := lockFn(repo) - if err != nil { - return nil, err - } - - globalLocks.Lock() - if globalLocks.cancelRefresh == nil { - debug.Log("main.lockRepository", "start goroutine for lock refresh") - globalLocks.cancelRefresh = make(chan struct{}) - globalLocks.refreshWG = sync.WaitGroup{} - globalLocks.refreshWG.Add(1) - go refreshLocks(&globalLocks.refreshWG, globalLocks.cancelRefresh) - } - - globalLocks.locks = append(globalLocks.locks, lock) - globalLocks.Unlock() - - return lock, err -} - -var refreshInterval = 5 * time.Minute - -func refreshLocks(wg *sync.WaitGroup, done <-chan struct{}) { - debug.Log("main.refreshLocks", "start") - defer func() { - wg.Done() - globalLocks.Lock() - globalLocks.cancelRefresh = nil - globalLocks.Unlock() - }() - - ticker := time.NewTicker(refreshInterval) - - for { - select { - case <-done: - debug.Log("main.refreshLocks", "terminate") - return - case <-ticker.C: - debug.Log("main.refreshLocks", "refreshing locks") - globalLocks.Lock() - for _, lock := range globalLocks.locks { - err := lock.Refresh() - if err != nil { - fmt.Fprintf(os.Stderr, "unable to refresh lock: %v\n", err) - } - } - globalLocks.Unlock() - } - } -} - -func unlockRepo(lock *restic.Lock) error { - globalLocks.Lock() - defer globalLocks.Unlock() - - debug.Log("unlockRepo", "unlocking repository") - if err := lock.Unlock(); err != nil { - debug.Log("unlockRepo", "error while unlocking: %v", err) - return err - } - - for i := 0; i < len(globalLocks.locks); i++ { - if lock == globalLocks.locks[i] { - globalLocks.locks = append(globalLocks.locks[:i], globalLocks.locks[i+1:]...) - return nil - } - } - - return nil -} - -func unlockAll() error { - globalLocks.Lock() - defer globalLocks.Unlock() - - debug.Log("unlockAll", "unlocking %d locks", len(globalLocks.locks)) - for _, lock := range globalLocks.locks { - if err := lock.Unlock(); err != nil { - debug.Log("unlockAll", "error while unlocking: %v", err) - return err - } - debug.Log("unlockAll", "successfully removed lock") - } - - return nil -} - -func init() { - AddCleanupHandler(unlockAll) -} diff --git a/cmd/restic/main.go b/cmd/restic/main.go deleted file mode 100644 index 32598fe23..000000000 --- a/cmd/restic/main.go +++ /dev/null @@ -1,45 +0,0 @@ -package main - -import ( - "fmt" - "os" - "runtime" - - "github.com/jessevdk/go-flags" - "github.com/restic/restic" - "github.com/restic/restic/debug" -) - -func init() { - // set GOMAXPROCS to number of CPUs - runtime.GOMAXPROCS(runtime.NumCPU()) -} - -func main() { - // defer profile.Start(profile.MemProfileRate(100000), profile.ProfilePath(".")).Stop() - // defer profile.Start(profile.CPUProfile, profile.ProfilePath(".")).Stop() - globalOpts.Repo = os.Getenv("RESTIC_REPOSITORY") - globalOpts.password = os.Getenv("RESTIC_PASSWORD") - - debug.Log("restic", "main %#v", os.Args) - - _, err := parser.Parse() - if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp { - parser.WriteHelp(os.Stdout) - os.Exit(0) - } - - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - } - - if restic.IsAlreadyLocked(err) { - fmt.Fprintf(os.Stderr, "\nthe `unlock` command can be used to remove stale locks\n") - } - - RunCleanupHandlers() - - if err != nil { - os.Exit(1) - } -} diff --git a/cmd/restic/testdata/backup-data.tar.gz b/cmd/restic/testdata/backup-data.tar.gz Binary files differdeleted file mode 100644 index 337c18fd9..000000000 --- a/cmd/restic/testdata/backup-data.tar.gz +++ /dev/null diff --git a/cmd/restic/testdata/old-index-repo.tar.gz b/cmd/restic/testdata/old-index-repo.tar.gz Binary files differdeleted file mode 100644 index 9cfc38573..000000000 --- a/cmd/restic/testdata/old-index-repo.tar.gz +++ /dev/null diff --git a/cmd/restic/testdata/repo-restore-permissions-test.tar.gz b/cmd/restic/testdata/repo-restore-permissions-test.tar.gz Binary files differdeleted file mode 100644 index 36aa62dbf..000000000 --- a/cmd/restic/testdata/repo-restore-permissions-test.tar.gz +++ /dev/null diff --git a/cmd/restic/testdata/small-repo.tar.gz b/cmd/restic/testdata/small-repo.tar.gz Binary files differdeleted file mode 100644 index 83959f539..000000000 --- a/cmd/restic/testdata/small-repo.tar.gz +++ /dev/null |