Support parallel building in makefile buildsystem

1) Add routine to Dh_Lib (used by dh and makefile.pm) which is capable of
detecting make jobserver and job control options from the MAKEFLAGS environment
variable. It also generates and returns a clean up MAKEFLAGS from these
options.

2) Add --parallel option to build system framework which allows source packages
to specify that they support parallel building. Optional value for this option is
the number of maximum parallel process to allow. However, the actual number of
parallel process (if any) for the specific build is determined from
DEB_BUILD_OPTIONS env variable as specified by Debian Policy.

By default (no --parallel option) parallel is neither enabled nor disabled
(depends on the external environment). However, dh may pass --parallel to
dh_auto_* implicitly in case 4) described below.

3) Add parallel support for makefile buildsystem. This implementation
forcefully starts a new make job server (or disables parallel) for the number
of process requested. If --parallel was not passed to the build system at all,
the build system will only clean up MAKEFLAGS from stale jobserver options to
avoid pointless make warnings.

4) If dh detects that it is being run by dpkg-buildpackage -jX and it is NOT
run with "+" prefix from debian/rules (i.e. jobserver is not reachable), it
enables --parallel implicitly. This closes: #532805.

Signed-off-by: Modestas Vainius <modestas@vainius.eu>

2 files changed, 288 insertions, 8 deletions

diff --git a/t/buildsystems/buildsystem_tests b/t/buildsystems/buildsystem_tests
index 8f7a275a..41c0f977 100755
--- a/t/buildsystems/buildsystem_tests
+++ b/t/buildsystems/buildsystem_tests
@@ -1,6 +1,6 @@
 #!/usr/bin/perl
 
-use Test::More tests => 228;
+use Test::More tests => 273;
 
 use strict;
 use warnings;
@@ -49,12 +49,32 @@ sub process_stdout {
 	my ($cmdline, $stdin) = @_;
 	my ($reader, $writer);
 
-	open2($reader, $writer, $cmdline) or die "Unable to exec $cmdline";
+	my $pid = open2($reader, $writer, $cmdline) or die "Unable to exec $cmdline";
 	print $writer $stdin if $stdin;
 	close $writer;
+	waitpid($pid, 0);
+	$? = $? >> 8; # exit status
 	return readlines($reader);
 }
 
+sub write_debian_rules {
+	my $contents=shift;
+	my $backup;
+
+	if (-f "debian/rules") {
+		(undef, $backup) = tempfile(DIR => ".", OPEN => 0);
+		rename "debian/rules", $backup;
+	}
+	# Write debian/rules if requested
+	if ($contents) {
+		open(my $f, ">", "debian/rules");
+		print $f $contents;;
+		close($f);
+		chmod 0755, "debian/rules";
+	}
+	return $backup;
+}
+
 ### Test Buildsystem class API methods
 is( $BS_CLASS->canonpath("path/to/the/./nowhere/../../somewhere"),
     "path/to/somewhere", "canonpath no1" );
@@ -384,17 +404,19 @@ EOF
 }
 
 $tmp = Cwd::getcwd();
-is_deeply( process_stdout("$^X -- - --builddirectory='autoconf/bld dir' --sourcedirectory autoconf",
+# NOTE: disabling parallel building explicitly (it might get automatically
+# enabled if run under dpkg-buildpackage -jX) to make output deterministic.
+is_deeply( process_stdout("$^X -- - --builddirectory='autoconf/bld dir' --sourcedirectory autoconf --parallel=1",
                           get_load_bs_source(undef, "configure")),
-    [ 'NAME=autoconf', 'builddir=autoconf/bld dir', "cwd=$tmp",  'makecmd=make', 'sourcedir=autoconf' ],
+    [ 'NAME=autoconf', 'builddir=autoconf/bld dir', "cwd=$tmp",  'makecmd=make', 'parallel=1', 'sourcedir=autoconf' ],
     "autoconf autoselection and sourcedir/builddir" );
 
-is_deeply( process_stdout("$^X -- - -Sautoconf -D autoconf", get_load_bs_source("autoconf", "build")),
-    [ 'NAME=autoconf', 'builddir=undef', "cwd=$tmp", 'makecmd=make', 'sourcedir=autoconf' ],
+is_deeply( process_stdout("$^X -- - -Sautoconf -D autoconf --parallel=1", get_load_bs_source("autoconf", "build")),
+    [ 'NAME=autoconf', 'builddir=undef', "cwd=$tmp", 'makecmd=make', 'parallel=1', 'sourcedir=autoconf' ],
     "forced autoconf and sourcedir" );
 
-is_deeply( process_stdout("$^X -- - -B -Sautoconf", get_load_bs_source("autoconf", "build")),
-    [ 'NAME=autoconf', "builddir=$default_builddir", "cwd=$tmp", 'makecmd=make', 'sourcedir=.' ],
+is_deeply( process_stdout("$^X -- - -B -Sautoconf --parallel=1", get_load_bs_source("autoconf", "build")),
+    [ 'NAME=autoconf', "builddir=$default_builddir", "cwd=$tmp", 'makecmd=make', 'parallel=1', 'sourcedir=.' ],
     "forced autoconf and default build directory" );
 
 # Build the autoconf test package
@@ -459,6 +481,243 @@ dh_auto_do_autoconf('autoconf');
 dh_auto_do_autoconf('autoconf', 'bld/dir', configure_args => [ "--extra-autoconf-configure-arg" ]);
 ok ( ! -e 'bld', "bld got deleted too" );
 
+#### Test parallel building and related options / routines
+@tmp = ( $ENV{MAKEFLAGS}, $ENV{DEB_BUILD_OPTIONS} );
+
+# Test get_make_jobserver_status() sub
+
+$ENV{MAKEFLAGS} = "--jobserver-fds=103,104 -j";
+is_deeply( [ get_make_jobserver_status() ], [ "jobserver-unavailable", undef ],
+	"get_make_jobserver_status(): unavailable jobserver, unset makeflags" );
+
+$ENV{MAKEFLAGS} = "-a --jobserver-fds=103,104 -j -b";
+is_deeply( [ get_make_jobserver_status() ], [ "jobserver-unavailable", "-a -b" ],
+	"get_make_jobserver_status(): unavailable jobserver, clean makeflags" );
+
+$ENV{MAKEFLAGS} = " --jobserver-fds=1,2 -j  ";
+is_deeply( [ get_make_jobserver_status() ], [ "jobserver", undef ],
+	"get_make_jobserver_status(): jobserver (available), clean makeflags" );
+
+$ENV{MAKEFLAGS} = "-a -j -b";
+is_deeply( [ get_make_jobserver_status() ], [ "jobs-0", "-a -b" ],
+	"get_make_jobserver_status(): -j" );
+
+$ENV{MAKEFLAGS} = "-a --jobs -b";
+is_deeply( [ get_make_jobserver_status() ], [ "jobs-0", "-a -b" ],
+	"get_make_jobserver_status(): --jobs" );
+
+$ENV{MAKEFLAGS} = "-j6";
+is_deeply( [ get_make_jobserver_status() ], [ "jobs-6", undef ],
+	"get_make_jobserver_status(): -j6" );
+
+$ENV{MAKEFLAGS} = "-a -j6 --jobs=7";
+is_deeply( [ get_make_jobserver_status() ], [ "jobs-7", "-a" ],
+	"get_make_jobserver_status(): -j6 --jobs=7" );
+
+$ENV{MAKEFLAGS} = "-j6 --jobserver-fds=5,6 --jobs=8";
+is_deeply( [ get_make_jobserver_status() ], [ "jobserver-unavailable", "-j6 --jobs=8" ],
+	"get_make_jobserver_status(): mixed jobserver and -j/--jobs" );
+
+# Test parallel building with makefile build system.
+$ENV{MAKEFLAGS} = "";
+$ENV{DEB_BUILD_OPTIONS} = "";
+
+sub do_parallel_mk {
+	my $dh_opts=shift || "";
+	my $make_opts=shift || "";
+	return process_stdout(
+		"LANG=C LC_ALL=C LC_MESSAGES=C $TOPDIR/dh_auto_build -Smakefile $dh_opts " .
+		"-- -s -f parallel.mk $make_opts 2>&1 >/dev/null", "");
+}
+
+sub test_isnt_parallel {
+	my ($got, $desc) = @_;
+	my @makemsgs = grep /^make[\d\[\]]*:/, @$got;
+	if (@makemsgs) {
+		like( $makemsgs[0], qr/Error 10/, $desc );
+	}
+	else {
+		ok( scalar(@makemsgs) > 0, $desc );
+	}
+}
+
+sub test_is_parallel {
+	my ($got, $desc) = @_;
+	is_deeply( $got, [] , $desc );
+	is( $?, 0, "(exit status=0) $desc");
+}
+
+test_isnt_parallel( do_parallel_mk(),
+	"No parallel by default" );
+test_isnt_parallel( do_parallel_mk("--parallel"),
+	"No parallel by default with --parallel" );
+
+$ENV{DEB_BUILD_OPTIONS}="parallel=5";
+test_isnt_parallel( do_parallel_mk(),
+	"DEB_BUILD_OPTIONS=parallel=5 without --parallel" );
+test_is_parallel( do_parallel_mk("--parallel"),
+	"DEB_BUILD_OPTIONS=parallel=5 with --parallel" );
+test_is_parallel( do_parallel_mk("--parallel=2"),
+	"DEB_BUILD_OPTIONS=parallel=5 with --parallel=2" );
+test_isnt_parallel( do_parallel_mk("--parallel=1"),
+	"DEB_BUILD_OPTIONS=parallel=5 with --parallel=1 (off)" );
+
+$ENV{MAKEFLAGS} = "--jobserver-fds=105,106 -j";
+$ENV{DEB_BUILD_OPTIONS}="";
+test_isnt_parallel( do_parallel_mk(),
+	"makefile.pm (no parallel): no make warnings about unavailable jobserver" );
+$ENV{DEB_BUILD_OPTIONS}="parallel=5";
+test_is_parallel( do_parallel_mk("--parallel"),
+	"DEB_BUILD_OPTIONS=parallel=5 with --parallel: no make warnings about unavail parent jobserver" );
+
+$ENV{MAKEFLAGS} = "-j2";
+$ENV{DEB_BUILD_OPTIONS}="";
+test_is_parallel( do_parallel_mk(),
+	"MAKEFLAGS=-j2 without --parallel: dh_auto_build honours MAKEFLAGS" );
+test_isnt_parallel( do_parallel_mk("--parallel=1"),
+	"MAKEFLAGS=-j2 with --parallel=1: dh_auto_build enforces -j1" );
+
+# Test dh dpkg-buildpackage -jX detection
+sub do_rules_for_parallel {
+	my $cmdline=shift || "";
+	my $stdin=shift || "";
+	return process_stdout("LANG=C LC_ALL=C LC_MESSAGES=C PATH=$TOPDIR:\$PATH " .
+		"make -f - $cmdline 2>&1 >/dev/null", $stdin);
+}
+
+# Simulate dpkg-buildpackage -j5
+doit("ln", "-s", "parallel.mk", "Makefile");
+
+sub test_dh_parallel {
+	my $extra_dsc=shift || "";
+	my $debian_rules=shift || "";
+	my $rules;
+	my $tmpfile;
+
+	$ENV{MAKEFLAGS} = "-j5";
+	$ENV{DEB_BUILD_OPTIONS} = "parallel=5";
+
+	# Write debian/rules if requested
+	$tmpfile = write_debian_rules($debian_rules);
+
+	$rules = <<'EOF';
+%:
+	@dh_clean > /dev/null 2>&1
+	@dh --buildsystem=makefile --after=dh_auto_configure --until=dh_auto_build $@
+	@dh_clean > /dev/null 2>&1
+EOF
+	test_is_parallel( do_rules_for_parallel("build", $rules),
+		"dh adds --parallel implicitly under dpkg-buildpackage -j5 $extra_dsc");
+
+	$ENV{MAKEFLAGS} = "";
+	test_isnt_parallel( do_rules_for_parallel("build", $rules),
+		"DEB_BUILD_OPTIONS=parallel=5 without MAKEFLAGS=-jX via dh $extra_dsc" );
+
+	$ENV{MAKEFLAGS} = "-j5";
+	$rules = <<'EOF';
+%:
+	@dh_clean > /dev/null 2>&1
+	@dh -j1 --buildsystem=makefile --after=dh_auto_configure --until=dh_auto_build $@
+	@dh_clean > /dev/null 2>&1
+EOF
+	test_isnt_parallel( do_rules_for_parallel("build", $rules),
+		"dh -j1 disables implicit parallel under dpkg-buildpackage -j5 $extra_dsc");
+
+	$rules = <<'EOF';
+%:
+	@dh_clean > /dev/null 2>&1
+	@dh -j --buildsystem=makefile --after=dh_auto_configure --until=dh_auto_build $@
+	@dh_clean > /dev/null 2>&1
+EOF
+	test_is_parallel( do_rules_for_parallel("build", $rules),
+		"dh -j under dpkg-buildpackage -j5 is parallel $extra_dsc");
+	$ENV{MAKEFLAGS} = "";
+	test_is_parallel( do_rules_for_parallel("build", $rules),
+		"dh -j is parallel only with DEB_BUILD_OPTIONS=parallel=5 $extra_dsc");
+
+	if (defined $tmpfile) {
+		rename($tmpfile, "debian/rules");
+	}
+	elsif ($debian_rules) {
+		unlink("debian/rules");
+	}
+}
+
+# dh should pass the same tests with and without overrides
+test_dh_parallel();
+test_dh_parallel("(with overrides)", <<'EOF');
+#!/usr/bin/make -f
+override_dh_auto_build:
+	@dh_auto_build -- -f parallel.mk
+EOF
+
+# Test if legacy punctuation hacks (+) work as before
+$ENV{MAKEFLAGS} = "-j5";
+$ENV{DEB_BUILD_OPTIONS} = "parallel=5";
+$tmp = write_debian_rules(<<'EOF');
+#!/usr/bin/make -f
+%:
+	@dh_clean > /dev/null 2>&1
+	@+dh --buildsystem=makefile --after=dh_auto_configure --until=dh_auto_build $@
+	@dh_clean > /dev/null 2>&1
+EOF
+test_is_parallel( do_rules_for_parallel("build", "include debian/rules"),
+	"legacy punctuation hacks: +dh, no override" );
+unlink "debian/rules";
+
+write_debian_rules(<<'EOF');
+#!/usr/bin/make -f
+override_dh_auto_build:
+	dh_auto_build
+%:
+	@dh_clean > /dev/null 2>&1
+	@+dh --buildsystem=makefile --after=dh_auto_configure --until=dh_auto_build $@
+	@dh_clean > /dev/null 2>&1
+EOF
+test_isnt_parallel( do_rules_for_parallel("build", "include debian/rules"),
+	"legacy punctuation hacks: +dh, override without +, no parallel, no make warnings" );
+unlink "debian/rules";
+
+write_debian_rules(<<'EOF');
+#!/usr/bin/make -f
+override_dh_auto_build:
+	+dh_auto_build
+%:
+	@dh_clean > /dev/null 2>&1
+	@+dh --buildsystem=makefile --after=dh_auto_configure --until=dh_auto_build $@
+	@dh_clean > /dev/null 2>&1
+EOF
+test_is_parallel( do_rules_for_parallel("build", "include debian/rules"),
+	"legacy punctuation hacks: +dh, override with +" );
+unlink "debian/rules";
+
+write_debian_rules(<<'EOF');
+#!/usr/bin/make -f
+override_dh_auto_build:
+	$(MAKE)
+%:
+	@dh_clean > /dev/null 2>&1
+	@+dh --buildsystem=makefile --after=dh_auto_configure --until=dh_auto_build $@
+	@dh_clean > /dev/null 2>&1
+EOF
+test_is_parallel( do_rules_for_parallel("build", "include debian/rules"),
+	"legacy punctuation hacks: +dh, override with \$(MAKE)" );
+unlink "debian/rules";
+
+if (defined $tmp) {
+	rename($tmp, "debian/rules");
+}
+else {
+	unlink("debian/rules");
+}
+
+# Clean up after parallel testing
+END {
+	system("rm", "-f", "Makefile");
+}
+$ENV{MAKEFLAGS} = $tmp[0] if defined $tmp[0];
+$ENV{DEB_BUILD_OPTIONS} = $tmp[1] if defined $tmp[1];
+
 END {
 	system("rm", "-rf", $tmpdir);
 	system("$TOPDIR/dh_clean");
diff --git a/t/buildsystems/parallel.mk b/t/buildsystems/parallel.mk
new file mode 100644
index 00000000..3e0d201d
--- /dev/null
+++ b/t/buildsystems/parallel.mk
@@ -0,0 +1,21 @@
+all: FIRST SECOND
+
+TMPFILE ?= $(CURDIR)/parallel.mk.lock
+
+rmtmpfile:
+	@rm -f "$(TMPFILE)"
+
+FIRST: rmtmpfile
+	@c=0; \
+    while [ $$c -le 5 ] && \
+          ([ ! -e "$(TMPFILE)" ] || [ "`cat "$(TMPFILE)"`" != "SECOND" ]); do \
+        c=$$(($$c+1)); \
+        sleep 0.1; \
+    done; \
+    rm -f "$(TMPFILE)"; \
+    if [ $$c -gt 5 ]; then exit 10; else exit 0; fi
+
+SECOND: rmtmpfile
+	@echo $@ > "$(TMPFILE)"
+
+.PHONY: all FIRST SECOND rmtmpfile