New upstream version 0.9.1+ds

6 files changed, 94 insertions, 103 deletions

diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go
index 487d8d587..dc84ed280 100644
--- a/cmd/restic/cmd_backup.go
+++ b/cmd/restic/cmd_backup.go
@@ -336,6 +336,14 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		return err
 	}
 
+	timeStamp := time.Now()
+	if opts.TimeStamp != "" {
+		timeStamp, err = time.Parse(TimeFormat, opts.TimeStamp)
+		if err != nil {
+			return errors.Fatalf("error in time option: %v\n", err)
+		}
+	}
+
 	var t tomb.Tomb
 
 	p := ui.NewBackup(term, gopts.verbosity)
@@ -402,14 +410,6 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		return true
 	}
 
-	timeStamp := time.Now()
-	if opts.TimeStamp != "" {
-		timeStamp, err = time.Parse(TimeFormat, opts.TimeStamp)
-		if err != nil {
-			return errors.Fatalf("error in time option: %v\n", err)
-		}
-	}
-
 	var targetFS fs.FS = fs.Local{}
 	if opts.Stdin {
 		p.V("read data from stdin")
diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go
index b2a67da5d..d21453859 100644
--- a/cmd/restic/cmd_find.go
+++ b/cmd/restic/cmd_find.go
@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"path/filepath"
 	"strings"
 	"time"
 
@@ -11,7 +10,9 @@ import (
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/walker"
 )
 
 var cmdFind = &cobra.Command{
@@ -94,7 +95,7 @@ type statefulOutput struct {
 	hits     int
 }
 
-func (s *statefulOutput) PrintJSON(prefix string, node *restic.Node) {
+func (s *statefulOutput) PrintJSON(path string, node *restic.Node) {
 	type findNode restic.Node
 	b, err := json.Marshal(struct {
 		// Add these attributes
@@ -111,7 +112,7 @@ func (s *statefulOutput) PrintJSON(prefix string, node *restic.Node) {
 		Content            byte `json:"content,omitempty"`
 		Subtree            byte `json:"subtree,omitempty"`
 	}{
-		Path:        filepath.Join(prefix, node.Name),
+		Path:        path,
 		Permissions: node.Mode.String(),
 		findNode:    (*findNode)(node),
 	})
@@ -138,22 +139,22 @@ func (s *statefulOutput) PrintJSON(prefix string, node *restic.Node) {
 	s.hits++
 }
 
-func (s *statefulOutput) PrintNormal(prefix string, node *restic.Node) {
+func (s *statefulOutput) PrintNormal(path string, node *restic.Node) {
 	if s.newsn != s.oldsn {
 		if s.oldsn != nil {
 			Verbosef("\n")
 		}
 		s.oldsn = s.newsn
-		Verbosef("Found matching entries in snapshot %s\n", s.oldsn.ID())
+		Verbosef("Found matching entries in snapshot %s\n", s.oldsn.ID().Str())
 	}
-	Printf(formatNode(prefix, node, s.ListLong) + "\n")
+	Printf(formatNode(path, node, s.ListLong) + "\n")
 }
 
-func (s *statefulOutput) Print(prefix string, node *restic.Node) {
+func (s *statefulOutput) Print(path string, node *restic.Node) {
 	if s.JSON {
-		s.PrintJSON(prefix, node)
+		s.PrintJSON(path, node)
 	} else {
-		s.PrintNormal(prefix, node)
+		s.PrintNormal(path, node)
 	}
 }
 
@@ -174,74 +175,75 @@ func (s *statefulOutput) Finish() {
 
 // Finder bundles information needed to find a file or directory.
 type Finder struct {
-	repo     restic.Repository
-	pat      findPattern
-	out      statefulOutput
-	notfound restic.IDSet
+	repo        restic.Repository
+	pat         findPattern
+	out         statefulOutput
+	ignoreTrees restic.IDSet
 }
 
-func (f *Finder) findInTree(ctx context.Context, treeID restic.ID, prefix string) error {
-	if f.notfound.Has(treeID) {
-		debug.Log("%v skipping tree %v, has already been checked", prefix, treeID)
-		return nil
-	}
-
-	debug.Log("%v checking tree %v\n", prefix, treeID)
+func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error {
+	debug.Log("searching in snapshot %s\n  for entries within [%s %s]", sn.ID(), f.pat.oldest, f.pat.newest)
 
-	tree, err := f.repo.LoadTree(ctx, treeID)
-	if err != nil {
-		return err
+	if sn.Tree == nil {
+		return errors.Errorf("snapshot %v has no tree", sn.ID().Str())
 	}
 
-	var found bool
-	for _, node := range tree.Nodes {
-		debug.Log("  testing entry %q\n", node.Name)
+	f.out.newsn = sn
+	return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(nodepath string, node *restic.Node, err error) (bool, error) {
+		if err != nil {
+			return false, err
+		}
+
+		if node == nil {
+			return false, nil
+		}
 
 		name := node.Name
 		if f.pat.ignoreCase {
 			name = strings.ToLower(name)
 		}
 
-		m, err := filepath.Match(f.pat.pattern, name)
+		foundMatch, err := filter.Match(f.pat.pattern, nodepath)
 		if err != nil {
-			return err
+			return false, err
 		}
 
-		if m {
-			if !f.pat.oldest.IsZero() && node.ModTime.Before(f.pat.oldest) {
-				debug.Log("    ModTime is older than %s\n", f.pat.oldest)
-				continue
+		var (
+			ignoreIfNoMatch = true
+			errIfNoMatch    error
+		)
+		if node.Type == "dir" {
+			childMayMatch, err := filter.ChildMatch(f.pat.pattern, nodepath)
+			if err != nil {
+				return false, err
 			}
 
-			if !f.pat.newest.IsZero() && node.ModTime.After(f.pat.newest) {
-				debug.Log("    ModTime is newer than %s\n", f.pat.newest)
-				continue
+			if !childMayMatch {
+				ignoreIfNoMatch = true
+				errIfNoMatch = walker.SkipNode
+			} else {
+				ignoreIfNoMatch = false
 			}
-
-			debug.Log("    found match\n")
-			found = true
-			f.out.Print(prefix, node)
 		}
 
-		if node.Type == "dir" {
-			if err := f.findInTree(ctx, *node.Subtree, filepath.Join(prefix, node.Name)); err != nil {
-				return err
-			}
+		if !foundMatch {
+			return ignoreIfNoMatch, errIfNoMatch
 		}
-	}
 
-	if !found {
-		f.notfound.Insert(treeID)
-	}
-
-	return nil
-}
+		if !f.pat.oldest.IsZero() && node.ModTime.Before(f.pat.oldest) {
+			debug.Log("    ModTime is older than %s\n", f.pat.oldest)
+			return ignoreIfNoMatch, errIfNoMatch
+		}
 
-func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error {
-	debug.Log("searching in snapshot %s\n  for entries within [%s %s]", sn.ID(), f.pat.oldest, f.pat.newest)
+		if !f.pat.newest.IsZero() && node.ModTime.After(f.pat.newest) {
+			debug.Log("    ModTime is newer than %s\n", f.pat.newest)
+			return ignoreIfNoMatch, errIfNoMatch
+		}
 
-	f.out.newsn = sn
-	return f.findInTree(ctx, *sn.Tree, string(filepath.Separator))
+		debug.Log("    found match\n")
+		f.out.Print(nodepath, node)
+		return false, nil
+	})
 }
 
 func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
@@ -289,10 +291,10 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 	defer cancel()
 
 	f := &Finder{
-		repo:     repo,
-		pat:      pat,
-		out:      statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON},
-		notfound: restic.NewIDSet(),
+		repo:        repo,
+		pat:         pat,
+		out:         statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON},
+		ignoreTrees: restic.NewIDSet(),
 	}
 	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) {
 		if err = f.findInSnapshot(ctx, sn); err != nil {
diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go
index d4a768d70..f7996b438 100644
--- a/cmd/restic/cmd_ls.go
+++ b/cmd/restic/cmd_ls.go
@@ -2,13 +2,12 @@ package main
 
 import (
 	"context"
-	"path/filepath"
 
 	"github.com/spf13/cobra"
 
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/walker"
 )
 
 var cmdLs = &cobra.Command{
@@ -46,26 +45,6 @@ func init() {
 	flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
 }
 
-func printTree(ctx context.Context, repo *repository.Repository, id *restic.ID, prefix string) error {
-	tree, err := repo.LoadTree(ctx, *id)
-	if err != nil {
-		return err
-	}
-
-	for _, entry := range tree.Nodes {
-		Printf("%s\n", formatNode(prefix, entry, lsOptions.ListLong))
-
-		if entry.Type == "dir" && entry.Subtree != nil {
-			entryPath := prefix + string(filepath.Separator) + entry.Name
-			if err = printTree(ctx, repo, entry.Subtree, entryPath); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
 func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 	if len(args) == 0 && opts.Host == "" && len(opts.Tags) == 0 && len(opts.Paths) == 0 {
 		return errors.Fatal("Invalid arguments, either give one or more snapshot IDs or set filters.")
@@ -85,7 +64,18 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
 		Verbosef("snapshot %s of %v at %s):\n", sn.ID().Str(), sn.Paths, sn.Time)
 
-		if err = printTree(gopts.ctx, repo, sn.Tree, ""); err != nil {
+		err := walker.Walk(ctx, repo, *sn.Tree, nil, func(nodepath string, node *restic.Node, err error) (bool, error) {
+			if err != nil {
+				return false, err
+			}
+
+			if node == nil {
+				return false, nil
+			}
+			Printf("%s\n", formatNode(nodepath, node, lsOptions.ListLong))
+			return false, nil
+		})
+		if err != nil {
 			return err
 		}
 	}
diff --git a/cmd/restic/format.go b/cmd/restic/format.go
index 1f8ab366e..1de0335c9 100644
--- a/cmd/restic/format.go
+++ b/cmd/restic/format.go
@@ -3,7 +3,6 @@ package main
 import (
 	"fmt"
 	"os"
-	"path/filepath"
 	"time"
 
 	"github.com/restic/restic/internal/restic"
@@ -63,10 +62,9 @@ func formatDuration(d time.Duration) string {
 	return formatSeconds(sec)
 }
 
-func formatNode(prefix string, n *restic.Node, long bool) string {
-	nodepath := prefix + string(filepath.Separator) + n.Name
+func formatNode(path string, n *restic.Node, long bool) string {
 	if !long {
-		return nodepath
+		return path
 	}
 
 	var mode os.FileMode
@@ -92,6 +90,6 @@ func formatNode(prefix string, n *restic.Node, long bool) string {
 
 	return fmt.Sprintf("%s %5d %5d %6d %s %s%s",
 		mode|n.Mode, n.UID, n.GID, n.Size,
-		n.ModTime.Format(TimeFormat), nodepath,
+		n.ModTime.Format(TimeFormat), path,
 		target)
 }
diff --git a/cmd/restic/global.go b/cmd/restic/global.go
index 3a66323dc..96d1f9e36 100644
--- a/cmd/restic/global.go
+++ b/cmd/restic/global.go
@@ -561,17 +561,18 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 	}
 
 	// wrap the transport so that the throughput via HTTP is limited
-	rt = limiter.NewStaticLimiter(gopts.LimitUploadKb, gopts.LimitDownloadKb).Transport(rt)
+	lim := limiter.NewStaticLimiter(gopts.LimitUploadKb, gopts.LimitDownloadKb)
+	rt = lim.Transport(rt)
 
 	switch loc.Scheme {
 	case "local":
 		be, err = local.Open(cfg.(local.Config))
 		// wrap the backend in a LimitBackend so that the throughput is limited
-		be = limiter.LimitBackend(be, limiter.NewStaticLimiter(gopts.LimitUploadKb, gopts.LimitDownloadKb))
+		be = limiter.LimitBackend(be, lim)
 	case "sftp":
 		be, err = sftp.Open(cfg.(sftp.Config))
 		// wrap the backend in a LimitBackend so that the throughput is limited
-		be = limiter.LimitBackend(be, limiter.NewStaticLimiter(gopts.LimitUploadKb, gopts.LimitDownloadKb))
+		be = limiter.LimitBackend(be, lim)
 	case "s3":
 		be, err = s3.Open(cfg.(s3.Config), rt)
 	case "gs":
@@ -585,7 +586,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 	case "rest":
 		be, err = rest.Open(cfg.(rest.Config), rt)
 	case "rclone":
-		be, err = rclone.Open(cfg.(rclone.Config))
+		be, err = rclone.Open(cfg.(rclone.Config), lim)
 
 	default:
 		return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
@@ -648,7 +649,7 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 	case "rest":
 		return rest.Create(cfg.(rest.Config), rt)
 	case "rclone":
-		return rclone.Open(cfg.(rclone.Config))
+		return rclone.Open(cfg.(rclone.Config), nil)
 	}
 
 	debug.Log("invalid repository scheme: %v", s)
diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go
index 8ccf28b1e..c16097b97 100644
--- a/cmd/restic/integration_test.go
+++ b/cmd/restic/integration_test.go
@@ -387,23 +387,23 @@ func TestBackupExclude(t *testing.T) {
 	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
 	snapshots, snapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
 	files := testRunLs(t, env.gopts, snapshotID)
-	rtest.Assert(t, includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
+	rtest.Assert(t, includes(files, "/testdata/foo.tar.gz"),
 		"expected file %q in first snapshot, but it's not included", "foo.tar.gz")
 
 	opts.Excludes = []string{"*.tar.gz"}
 	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
 	snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
 	files = testRunLs(t, env.gopts, snapshotID)
-	rtest.Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
+	rtest.Assert(t, !includes(files, "/testdata/foo.tar.gz"),
 		"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
 
 	opts.Excludes = []string{"*.tar.gz", "private/secret"}
 	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
 	_, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
 	files = testRunLs(t, env.gopts, snapshotID)
-	rtest.Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
+	rtest.Assert(t, !includes(files, "/testdata/foo.tar.gz"),
 		"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
-	rtest.Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "private", "secret", "passwords.txt")),
+	rtest.Assert(t, !includes(files, "/testdata/private/secret/passwords.txt"),
 		"expected file %q not in first snapshot, but it's included", "passwords.txt")
 }