7 files changed, 107 insertions, 34 deletions

diff --git a/debian/changelog b/debian/changelog
index 7c9c472..8261687 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,16 @@
+pristine-tar (1.50) unstable; urgency=medium
+
+  [ Kevin Locke ]
+  * Handle octal escapes in quoted paths
+  * Use a single substitution to correctly unquote paths
+
+  [ Matthew Vernon ]
+  * Quote manifests and use tar --null (Closes: #1010024, #784635,
+    #736201, #719078 )
+  * Add a couple more test cases
+
+ -- Matthew Vernon <matthew@debian.org>  Fri, 04 Nov 2022 11:23:44 +0000
+
 pristine-tar (1.49) unstable; urgency=medium
 
   [ Steve McIntyre ]
diff --git a/pristine-tar b/pristine-tar
index 081dca1..7ec8d3e 100755
--- a/pristine-tar
+++ b/pristine-tar
@@ -362,6 +362,28 @@ Options:
 sub unquote_filename {
   my $filename = shift;
 
+  my %tar_escapes = (
+    'a' => "\a",
+    'b' => "\b",
+    'f' => "\f",
+    'n' => "\n",
+    'r' => "\r",
+    't' => "\t",
+    'v' => "\x11",
+    '\\' => '\\',
+  );
+  $filename =~ s/\\([abfnrtv\\]|[0-9]{3})/$tar_escapes{$1} or chr(oct($1))/ge;
+
+  return $filename;
+}
+
+#This is unquote_filename from pristine-tar 1.49; it isn't correct.
+#But we need to be able to identically mis-unquote paths in order to
+#generate identical tarballs to those buggy versions to apply deltas
+#against.
+sub buggy_unquote_filename {
+  my $filename = shift;
+
   $filename =~ s/\\a/\a/g;
   $filename =~ s/\\b/\b/g;
   $filename =~ s/\\f/\f/g;
@@ -430,7 +452,12 @@ sub recreatetarball {
   # in the tarball, since it can easily vary.
   my $full_sweep = 0;
   foreach my $file (@manifest) {
-    my $unquoted_file = unquote_filename($file);
+    my $unquoted_file;
+    if ($options{buggy_unquote}) {
+      $unquoted_file = buggy_unquote_filename($file);
+    } else {
+      $unquoted_file = unquote_filename($file);
+    }
 
     if (-l "$tempdir/workdir/$unquoted_file") {
       # Can't set timestamp of a symlink, so
@@ -464,7 +491,12 @@ sub recreatetarball {
   # Set file times only after modifying of the directory content is
   # done.
   foreach my $file (@manifest) {
-    my $unquoted_file = unquote_filename($file);
+    my $unquoted_file;
+    if ($options{buggy_unquote}) {
+      $unquoted_file = buggy_unquote_filename($file);
+    } else {
+      $unquoted_file = unquote_filename($file);
+    }
     if (-e "$tempdir/workdir/$unquoted_file") {
       utime(0, 0, "$tempdir/workdir/$unquoted_file") || die "utime: $file: $!";
     }
@@ -508,6 +540,26 @@ sub recreatetarball_helper {
   my %options = @_;
   my $tempdir = $recreatetarball_tempdir;
 
+  # Most manifests are quoted; we want to unquote and null-separate them
+  # to pass to tar --null --files-from. Unfortunately, some versions of
+  # pristine-tar wrote unquoted manifests; for many of these, unquoting
+  # is a no-op.
+  # So we try unquote-and-null-separate; if that fails, try just
+  # null-separate (there should be no unquoted manifests with paths with
+  # newlines in).
+  open(IN, "<", "$tempdir/manifest") || die "$tempdir/manifest: $!";
+  open(OUT, ">", "$tempdir/nulledmanifest") || die "$!";
+  while (<IN>) {
+      chomp;
+      if ($options{buggy_unquote}) {
+	print OUT buggy_unquote_filename($_) . "\0" if length $_;
+      } else {
+	print OUT unquote_filename($_) . "\0" if length $_;
+      }
+  }
+  close IN;
+  close OUT;
+
   my $ret = "$tempdir/recreatetarball";
   my @cmd = (
     $tar_program,     "cf",
@@ -516,14 +568,26 @@ sub recreatetarball_helper {
     0,                "--numeric-owner",
     "-C",             "$tempdir/workdir",
     "--no-recursion", "--mode",
-    "0644",           "--verbatim-files-from",
-    "--files-from",   "$tempdir/manifest"
+    "0644",           "--null",
+    "--files-from",   "$tempdir/nulledmanifest"
   );
   if (exists $options{tar_format}) {
     push @cmd, ("-H", $options{tar_format});
   }
 
-  doit(@cmd);
+  if (try_doit(@cmd) != 0) {
+      debug "tar failed with unquoted manifest, retrying without unquoting";
+      open(IN, "<", "$tempdir/manifest") || die "$tempdir/manifest: $!";
+      open(OUT, ">", "$tempdir/nulledmanifest") || die "$!";
+      while (<IN>) {
+	  chomp;
+	  print OUT "$_\0" if length $_;
+      }
+      close IN;
+      close OUT;
+      doit(@cmd);
+  }
+#  unlink("$tempdir/nulledmanifest") || die "unlink $tempdir/nulledmanifest: $!";
 
   return $ret;
 }
@@ -560,33 +624,6 @@ sub recreatetarball_broken_numeric_owner {
   return recreatetarball_helper();
 }
 
-sub recreatetarball_broken_verbatim {
-  # To fix #851286, the option --verbatim-files-from was added by
-  # default. But now some older older stored tarballs won't reproduce
-  # (#933031). Try again *without* that option to tar.
-  my %options = @_;
-  my $tempdir = $recreatetarball_tempdir;
-
-  my $ret = "$tempdir/recreatetarball";
-  my @cmd = (
-    $tar_program,     "cf",
-    $ret,             "--owner",
-    0,                "--group",
-    0,                "--numeric-owner",
-    "-C",             "$tempdir/workdir",
-    "--no-recursion", "--mode",
-    "0644",
-    "--files-from",   "$tempdir/manifest"
-  );
-  if (exists $options{tar_format}) {
-    push @cmd, ("-H", $options{tar_format});
-  }
-
-  doit(@cmd);
-
-  return $ret;
-}
-
 sub gentar {
   my $deltafile = shift;
   my $tarball   = shift;
@@ -624,6 +661,14 @@ sub gentar {
       %opts
     );
   };
+  push @try, sub {
+    recreatetarball(
+      $delta->{manifest}, getcwd,
+      clobber_source => 0,
+      buggy_unquote => 1,
+      %opts
+    );
+  };
   push @try, \&recreatetarball_longlink_100;
   push @try, sub {
     recreatetarball(
@@ -641,7 +686,6 @@ sub gentar {
       %opts
     );
   };
-  push @try, \&recreatetarball_broken_verbatim;
   push @try, \&recreatetarball_broken_numeric_owner;
 
   my $ok;
@@ -692,7 +736,7 @@ sub genmanifest {
     chomp;
     # ./ or / in the manifest just confuses tar
     s/^\.?\/+//;
-    print OUT unquote_filename($_) . "\n" if length $_;
+    print OUT "$_\n" if length $_;
   }
   close IN;
   close OUT;
diff --git a/test/samples/tarballs/hazard.tar.gz b/test/samples/tarballs/hazard.tar.gz
new file mode 100644
index 0000000..a08a3e6
--- /dev/null
+++ b/test/samples/tarballs/hazard.tar.gz
diff --git a/test/samples/tarballs/rclone_1.60.0.orig.tar.gz b/test/samples/tarballs/rclone_1.60.0.orig.tar.gz
new file mode 100644
index 0000000..373dc18
--- /dev/null
+++ b/test/samples/tarballs/rclone_1.60.0.orig.tar.gz
diff --git a/test/samples/tarballs/rlottie_git.tar.gz b/test/samples/tarballs/rlottie_git.tar.gz
new file mode 100644
index 0000000..c89daa8
--- /dev/null
+++ b/test/samples/tarballs/rlottie_git.tar.gz
diff --git a/test/test_bugs.sh b/test/test_bugs.sh
index a9cb654..00801ba 100644
--- a/test/test_bugs.sh
+++ b/test/test_bugs.sh
@@ -44,4 +44,12 @@ test_851286() {
   silent_run pristine-tar commit "$orig_tarball"
 }
 
+test_933031() {
+  # partially unquoted paths in manifests should work
+  silent_run tar xaf "$SAMPLES/tarballs/rlottie_git.tar.gz"
+  cd rlottie
+
+  silent_run pristine-tar checkout ../rlottie_933031_test.tar.gz
+}
+
 . shunit2
diff --git a/test/test_roundtrip.sh b/test/test_roundtrip.sh
index ab6c240..3ca6628 100644
--- a/test/test_roundtrip.sh
+++ b/test/test_roundtrip.sh
@@ -59,4 +59,12 @@ test_escaping() {
   assertWorksWithTarball $SAMPLES/tarballs/systemd_239.orig.tar.gz
 }
 
+test_highbit_paths() {
+  assertWorksWithTarball $SAMPLES/tarballs/rclone_1.60.0.orig.tar.gz
+}
+
+test_hyphen_and_newline() {
+  assertWorksWithTarball $SAMPLES/tarballs/hazard.tar.gz
+}
+
 . shunit2