From f8bf279fa55368a2840e62b5f5c14531e565f6aa Mon Sep 17 00:00:00 2001
From: Rafael Laboissiere <rafael@laboissiere.net>
Date: Fri, 29 Nov 2013 00:14:13 +0100
Subject: [PATCH] [uscan] Allow a different compression scheme when repacking
 upstream tarballs

The changes introduced by this commit were originally done by Andreas
Tille <tille@debian.org> and were discussed in the context of
Bug#685787.  I just merged the relevant changes from Andreas' private
Git repository into the master branch of the Git repository of
devscripts.

The --repack-compression option of uscan accepts now an argument that
indicates the type of the compression that will be used when repacking
the upstream tarball, either because there are files listed in the
Files-Excluded field of debian/copyright or because the --repack
option was given and there is need for repacking. The current
alternatives are gz, bz, xz, and lzma.

The documentation in scripts/uscan.1 has been changed accordingly and
the test in test/test_uscan has been extended to exercise the option
--repack-compression=xz, which will repack the gz upstream tarball
using xz.  For that to work, a build-dependency on xz-utils has also
been added.
---
 debian/control   |   3 +-
 scripts/uscan.1  |   6 +++
 scripts/uscan.pl | 115 ++++++++++++++++++++++++++++++++++++++-----------------
 test/test_uscan  |   8 ++--
 4 files changed, 93 insertions(+), 39 deletions(-)

diff --git a/debian/control b/debian/control
index d48f7f2..2c4b5ba 100644
--- a/debian/control
+++ b/debian/control
@@ -28,7 +28,8 @@ Build-Depends: debhelper (>= 9),
                python3-setuptools,
                shunit2 (>= 2.1.6),
                wdiff,
-               xsltproc
+               xsltproc,
+               xz-utils
 XS-Testsuite: autopkgtest
 Vcs-Browser: http://anonscm.debian.org/gitweb/?p=collab-maint/devscripts.git
 Vcs-Git: git://anonscm.debian.org/collab-maint/devscripts.git
diff --git a/scripts/uscan.1 b/scripts/uscan.1
index 17e7941..9adee41 100644
--- a/scripts/uscan.1
+++ b/scripts/uscan.1
@@ -401,6 +401,12 @@ a zip archive (i.e. it doesn't match a .tlz, .tlzm, .tlzma, .tar.lzma, .txz,
 installed in order to repack .zip archives, the xz-utils package must be
 installed to repack lzma or xz tar archives.
 .TP
+\fB\-\-repack-compression\fR \fICOMP\fR
+In the case where the upstream sources are repacked (either because
+\fB\-\-repack\fR option is given or \fIdebian/copyright\fR contains the
+field \fBFiles-Excluded\fR), it is possible to control the compression
+method via the \fICOMP\fR parameter (defaults to \fBgz\fR).
+.TP
 .B \-\-no\-symlink
 Don't make these symlinks and don't rename the files.
 .TP
diff --git a/scripts/uscan.pl b/scripts/uscan.pl
index a77385a..2f6e906 100755
--- a/scripts/uscan.pl
+++ b/scripts/uscan.pl
@@ -75,6 +75,7 @@ sub dehs_output ();
 sub quoted_regex_replace ($);
 sub safe_replace ($$);
 sub get_main_source_dir($$$$);
+sub compress_archive($$$);
 
 sub usage {
     print <<"EOF";
@@ -103,6 +104,9 @@ Options:
     --repack       Repack downloaded archives from orig.tar.bz2, orig.tar.lzma,
                    orig.tar.xz or orig.zip to orig.tar.gz
                    (does nothing if downloaded archive orig.tar.gz)
+    --repack-compression COMP
+                   When the upstream sources are repacked, use compression COMP
+                   for the resulting tarball
     --no-symlink   Don\'t make symlink or rename
     --verbose      Give verbose output
     --no-verbose   Don\'t give verbose output (default)
@@ -174,6 +178,8 @@ my $download_version;
 my $force_download = 0;
 my $report = 0; # report even on up-to-date packages?
 my $repack = 0; # repack .tar.bz2, .tar.lzma, .tar.xz or .zip to .tar.gz
+my $default_compression = 'gz' ;
+my $repack_compression = $default_compression;
 my $symlink = 'symlink';
 my $verbose = 0;
 my $check_dirname_level = 1;
@@ -272,7 +278,8 @@ if (@ARGV and $ARGV[0] =~ /^--no-?conf$/) {
 # Now read the command line arguments
 my $debug = 0;
 my ($opt_h, $opt_v, $opt_destdir, $opt_download, $opt_force_download,
-    $opt_report, $opt_passive, $opt_symlink, $opt_repack, $opt_exclusion);
+    $opt_report, $opt_passive, $opt_symlink, $opt_repack,
+    $opt_repack_compression, $opt_exclusion);
 my ($opt_verbose, $opt_level, $opt_regex, $opt_noconf);
 my ($opt_package, $opt_uversion, $opt_watchfile, $opt_dehs, $opt_timeout);
 my $opt_download_version;
@@ -292,6 +299,7 @@ GetOptions("help" => \$opt_h,
 	   "symlink!" => sub { $opt_symlink = $_[1] ? 'symlink' : 'no'; },
 	   "rename" => sub { $opt_symlink = 'rename'; },
 	   "repack" => sub { $opt_repack = 1; },
+	   "repack-compression=s" => \$opt_repack_compression,
 	   "package=s" => \$opt_package,
 	   "upstream-version=s" => \$opt_uversion,
 	   "watchfile=s" => \$opt_watchfile,
@@ -327,6 +335,16 @@ $timeout = $opt_timeout if defined $opt_timeout;
 $timeout = 20 unless defined $timeout and $timeout > 0;
 $symlink = $opt_symlink if defined $opt_symlink;
 $verbose = $opt_verbose if defined $opt_verbose;
+if ( defined $opt_repack_compression ) {
+    if ( $opt_repack_compression =~ /^gz$/  or
+         $opt_repack_compression =~ /^bz2$/ or
+         $opt_repack_compression =~ /^xz$/  or
+         $opt_repack_compression =~ /^lzma$/ ) {
+        $repack_compression = $opt_repack_compression;
+    } else {
+        print "-- Invalid compression $opt_repack_compression given.  Use default $default_compression instead.\n" if $verbose ;
+    }
+}
 $dehs = $opt_dehs if defined $opt_dehs;
 $exclusion = $opt_exclusion if defined $opt_exclusion;
 $user_agent_string = $opt_user_agent if defined $opt_user_agent;
@@ -1413,71 +1431,67 @@ EOF
 		 or uscan_die("$progname warning: OpenPGP signature did not verify.\n");
     }
 
-    if ($repack and $newfile_base =~ /^(.*)\.(tar\.bz2|tbz2?)$/) {
+    if ($repack and $newfile_base =~ /^(.*)\.(tar\.bz2|tbz2?)$/ and
+        $repack_compression !~ /^bz2$/ ) {
 	print "-- Repacking from bzip2 to gzip\n" if $verbose;
-	my $newfile_base_gz = "$1.tar.gz";
+	my $newfile_base_compression = "$1.tar.".$repack_compression;
 	my (undef, $fname) = tempfile(UNLINK => 1);
 	spawn(exec => ['bunzip2', '-c', "$destdir/$newfile_base"],
 	      to_file => $fname,
 	      wait_child => 1);
 	spawn(exec => ['gzip', '-n', '-9'],
 	      from_file => $fname,
-	      to_file => "$destdir/$newfile_base_gz",
+	      to_file => "$destdir/$newfile_base_compression",
 	      wait_child => 1);
 	unlink "$destdir/$newfile_base";
-	$newfile_base = $newfile_base_gz;
+	$newfile_base = $newfile_base_compression;
     }
 
-    if ($repack and $newfile_base =~ /^(.*)\.(tar\.lzma|tlz(?:ma?)?)$/) {
-	print "-- Repacking from lzma to gzip\n" if $verbose;
-	my $newfile_base_gz = "$1.tar.gz";
+    if ($repack and $newfile_base =~ /^(.*)\.(tar\.lzma|tlz(?:ma?)?)$/ and
+        $repack_compression !~ /^lzma$/ ) {
+	print "-- Repacking from lzma to $repack_compression\n" if $verbose;
+	my $newfile_base_compression = "$1.tar.".$repack_compression;
 	my (undef, $fname) = tempfile(UNLINK => 1);
 	spawn(exec => ['xz', '-F', 'lzma', '-cd', "$destdir/$newfile_base"],
 	      to_file => $fname,
 	      wait_child => 1);
-	spawn(exec => ['gzip', '-n', '-9'],
-	      from_file => $fname,
-	      to_file => "$destdir/$newfile_base_gz",
-	      wait_child => 1);
-	unlink "$destdir/$newfile_base";
-	$newfile_base = $newfile_base_gz;
+	compress_archive("$fname", "$destdir/$newfile_base_compression", $repack_compression);
+	$newfile_base = $newfile_base_compression;
     }
 
-    if ($repack and $newfile_base =~ /^(.*)\.(tar\.xz|txz)$/) {
-	print "-- Repacking from xz to gzip\n" if $verbose;
-	my $newfile_base_gz = "$1.tar.gz";
+    if ($repack and $newfile_base =~ /^(.*)\.(tar\.xz|txz)$/ and
+        $repack_compression !~ /^xz$/ ) {
+	print "-- Repacking from xz to $repack_compression\n" if $verbose;
+	my $newfile_base_compression = "$1.tar.".$repack_compression;
 	my (undef, $fname) = tempfile(UNLINK => 1);
 	spawn(exec => ['xz', '-cd', "$destdir/$newfile_base"],
 	      to_file => $fname,
 	      wait_child => 1);
-	spawn(exec => ['gzip', '-n', '-9'],
-	      from_file => $fname,
-	      to_file => "$destdir/$newfile_base_gz",
-	      wait_child => 1);
-	unlink "$destdir/$newfile_base";
-	$newfile_base = $newfile_base_gz;
+	compress_archive("$fname", "$destdir/$newfile_base_compression", $repack_compression);
+	$newfile_base = $newfile_base_compression;
     }
 
     if ($repack and $newfile_base =~ /^(.*)\.(zip|jar)$/) {
-	print "-- Repacking from zip to .tar.gz\n" if $verbose;
+	print "-- Repacking from zip to .tar.$repack_compression\n" if $verbose;
 
 	system('command -v unzip >/dev/null 2>&1') >> 8 == 0
 	  or uscan_die("unzip binary not found. You need to install the package unzip to be able to repack .zip upstream archives.\n");
 
-	my $newfile_base_gz = "$1.tar.gz";
+	my $compress_file_base = "$1.tar" ;
+	my $newfile_base_compression = "$compress_file_base.".$repack_compression;
 	my $tempdir = tempdir ( "uscanXXXX", TMPDIR => 1, CLEANUP => 1 );
 	my $globpattern = "*";
 	my $hidden = ".[!.]*";
 	my $absdestdir = abs_path($destdir);
 	system('unzip', '-q', '-a', '-d', $tempdir, "$destdir/$newfile_base") == 0
-	  or uscan_die("Repacking from zip or jar to tar.gz failed (could not unzip)\n");
+	  or uscan_die("Repacking from zip or jar to tar.$repack_compression failed (could not unzip)\n");
 	if (defined glob("$tempdir/$hidden")) {
 	    $globpattern .= " $hidden";
 	}
-	system("cd $tempdir; GZIP='-n -9' tar --owner=root --group=root --mode=a+rX -czf \"$absdestdir/$newfile_base_gz\" $globpattern") == 0
-	  or uscan_die("Repacking from zip or jar to tar.gz failed (could not create tarball)\n");
-	unlink "$destdir/$newfile_base";
-	$newfile_base = $newfile_base_gz;
+	system("cd $tempdir; tar --owner=root --group=root --mode=a+rX -cf \"$absdestdir/$compress_file_base\" $globpattern") == 0
+	  or uscan_die("Repacking from zip or jar to tar.$repack_compression failed (could not create tarball)\n");
+	compress_archive("$absdestdir/$compress_file_base", "$absdestdir/$newfile_base_compression", $repack_compression);
+	$newfile_base = $newfile_base_compression;
     }
 
     if ($newfile_base =~ /\.(tar\.gz|tgz
@@ -1530,11 +1544,11 @@ EOF
 	    if ( $nfiles_before == $nfiles_after ) {
 		print "-- Source tree remains identical - no need for repacking.\n" if $verbose;
 	    } else {
-		my $suffix = 'gz' ;
-		my $newfile_base_dfsg = "${pkg}_${newversion}${excludesuffix}.orig.tar.$suffix" ;
-		system("cd $tempdir; GZIP='-n -9' tar --owner=root --group=root --mode=a+rX -czf \"$absdestdir/$newfile_base_dfsg\" $globpattern") == 0
-		    or die("Excluding files failed (could not create tarball)\n");
-		$symlink = 'files-excluded' # prevent symlinking or renaming
+                my $newfile_base_dfsg = "${pkg}_${newversion}${excludesuffix}.orig.tar" ;
+                system("cd $tempdir; tar --owner=root --group=root --mode=a+rX -cf \"$absdestdir/$newfile_base_dfsg\" $globpattern") == 0
+                   or die("Excluding files failed (could not create tarball)\n");
+	        compress_archive("$absdestdir/$newfile_base_dfsg", "$absdestdir/$newfile_base_dfsg.$repack_compression", $repack_compression);
+                $symlink = 'files-excluded' # prevent symlinking or renaming
 	    }
 	}
     }
@@ -2180,3 +2194,34 @@ sub get_main_source_dir($$$$) {
     }
     return $main_source_dir;
 }
+
+sub compress_archive($$$) {
+    my ($from_file, $to_file, $compression) = @_;
+    if ( $compression =~ /^gz$/ ) {
+        spawn(exec => ['gzip', '-n', '-9'],
+            from_file => $from_file,
+            to_file => $to_file,
+            wait_child => 1);
+    } elsif ( $compression =~ /^bz2$/ ) {
+        # The actual options should be discussed - supporting small memory seems reasonable
+        spawn(exec => ['bzip2', '--small'],
+            from_file => $from_file,
+            to_file => $to_file,
+            wait_child => 1);
+    } elsif ( $compression =~ /^xz$/ ) {
+        # The actual options should be discussed - supporting small memory seems reasonable
+        spawn(exec => ['xz', '--memlimit=150MiB'],
+            from_file => $from_file,
+            to_file => $to_file,
+            wait_child => 1);
+    } elsif ( $compression =~ /^lzma$/ ) {
+        # The actual options should be discussed - no idea what might be reasonable here
+        spawn(exec => ['lzma'],
+            from_file => $from_file,
+            to_file => $to_file,
+            wait_child => 1);
+    } else {
+	die("Unknown compression method $compression.");
+    }
+    unlink "$from_file";
+}
diff --git a/test/test_uscan b/test/test_uscan
index aa8ccf7..d171f58 100755
--- a/test/test_uscan
+++ b/test/test_uscan
@@ -20,7 +20,7 @@ if test "$1" = --installed; then
     shift
 else
     top_srcdir=$(readlink -f "${0%/*}/..")
-    COMMAND="perl -I $top_srcdir $top_srcdir/scripts/uscan.pl --no-conf"
+    COMMAND="perl -I $top_srcdir $top_srcdir/scripts/uscan.pl --no-conf --repack-compression=xz"
 fi
 
 cleanup(){
@@ -80,10 +80,12 @@ END
 
     ( cd $TMPDIR/$PKG ; $COMMAND )
 
-    TARBALL=${PKG}_1+dfsg.orig.tar.gz
+    TARBALL=${PKG}_1+dfsg.orig.tar.xz
     assertTrue 'pristine tarball is not created' "[ -f $TMPDIR/$TARBALL ]"
+    assertNotNull 'pristine tarball is not xz-compressed'	\
+                  "$( file $TMPDIR/$TARBALL | grep 'XZ compressed data' )"
     assertNull 'file that must be excluded is present in the tarball'	\
-               "$( tar tzf $TMPDIR/$TARBALL  | grep exclude-this )"
+               "$( tar tJf $TMPDIR/$TARBALL  | grep exclude-this )"
 
     cleanup
 
-- 
1.8.5.rc3

