Support parallel building in makefile buildsystem

1) Add routine to Dh_Lib (used by dh and makefile.pm) which is capable of
detecting make jobserver and job control options from the MAKEFLAGS environment
variable. It also generates and returns a clean up MAKEFLAGS from these
options.

2) Add --parallel option to build system framework which allows source packages
to specify that they support parallel building. Optional value for this option is
the number of maximum parallel process to allow. However, the actual number of
parallel process (if any) for the specific build is determined from
DEB_BUILD_OPTIONS env variable as specified by Debian Policy.

By default (no --parallel option) parallel is neither enabled nor disabled
(depends on the external environment). However, dh may pass --parallel to
dh_auto_* implicitly in case 4) described below.

3) Add parallel support for makefile buildsystem. This implementation
forcefully starts a new make job server (or disables parallel) for the number
of process requested. If --parallel was not passed to the build system at all,
the build system will only clean up MAKEFLAGS from stale jobserver options to
avoid pointless make warnings.

4) If dh detects that it is being run by dpkg-buildpackage -jX and it is NOT
run with "+" prefix from debian/rules (i.e. jobserver is not reachable), it
enables --parallel implicitly. This closes: #532805.

Signed-off-by: Modestas Vainius <modestas@vainius.eu>

4 files changed, 126 insertions, 5 deletions

diff --git a/Debian/Debhelper/Buildsystem.pm b/Debian/Debhelper/Buildsystem.pm
index 62c45b5d..677e3bf9 100644
--- a/Debian/Debhelper/Buildsystem.pm
+++ b/Debian/Debhelper/Buildsystem.pm
@@ -47,7 +47,12 @@ sub DEFAULT_BUILD_DIRECTORY {
 #                  specified or empty, defaults to the current directory.
 # - builddir -     specifies build directory to use. Path is relative to the
 #                  current (top) directory. If undef or empty,
-#                  DEFAULT_BUILD_DIRECTORY directory will be used. 
+#                  DEFAULT_BUILD_DIRECTORY directory will be used.
+# - parallel -     number of parallel process to be spawned for building
+#                  sources. Parallel building needs to be supported by the
+#                  underlying build system for this option to be effective.
+#                  Defaults to undef (i.e. parallel disabled, but do not try to
+#                  enforce this limit by messing with environment).
 # Derived class can override the constructor to initialize common object
 # parameters. Do NOT use constructor to execute commands or otherwise
 # configure/setup build environment. There is absolutely no guarantee the
@@ -58,6 +63,7 @@ sub new {
 
 	my $this = bless({ sourcedir => '.',
 	                   builddir => undef,
+	                   parallel => undef,
 	                   cwd => Cwd::getcwd() }, $class);
 
 	if (exists $opts{sourcedir}) {
@@ -71,6 +77,9 @@ sub new {
 	if (exists $opts{builddir}) {
 		$this->_set_builddir($opts{builddir});
 	}
+	if (defined $opts{parallel} && $opts{parallel} >= 1) {
+		$this->{parallel} = $opts{parallel};
+	}
 	return $this;
 }
 
@@ -243,6 +252,11 @@ sub get_source_rel2builddir {
 	return $dir;
 }
 
+sub get_parallel {
+	my $this=shift;
+	return $this->{parallel};
+}
+
 # When given a relative path to the build directory, converts it
 # to the path that is relative to the source directory. If $path is
 # not given, returns a path to the build directory that is relative
diff --git a/Debian/Debhelper/Buildsystem/makefile.pm b/Debian/Debhelper/Buildsystem/makefile.pm
index 3809d594..6629d25b 100644
--- a/Debian/Debhelper/Buildsystem/makefile.pm
+++ b/Debian/Debhelper/Buildsystem/makefile.pm
@@ -7,7 +7,7 @@
 package Debian::Debhelper::Buildsystem::makefile;
 
 use strict;
-use Debian::Debhelper::Dh_Lib qw(escape_shell);
+use Debian::Debhelper::Dh_Lib qw(escape_shell get_make_jobserver_status);
 use base 'Debian::Debhelper::Buildsystem';
 
 sub get_makecmd_C {
@@ -30,13 +30,38 @@ sub exists_make_target {
 	return length($ret);
 }
 
+sub do_make {
+	my $this=shift;
+
+	# Always clean MAKEFLAGS from unavailable jobserver options. If parallel
+	# is enabled, do more extensive clean up from all job control specific
+	# options and start our own jobserver if parallel building (> 1) was
+	# requested.
+	my ($status, $makeflags) = get_make_jobserver_status();
+	if ($status eq "jobserver-unavailable" || defined $this->get_parallel()) {
+		if (defined $makeflags) {
+			$ENV{MAKEFLAGS} = $makeflags;
+		}
+		else {
+			delete $ENV{MAKEFLAGS} if exists $ENV{MAKEFLAGS};
+		}
+	}
+
+	# Start a new jobserver if parallel building was requested
+	if (defined $this->get_parallel()) {
+		unshift @_, "-j" . ($this->get_parallel() > 1 ? $this->get_parallel() : 1);
+	}
+
+	$this->doit_in_builddir($this->{makecmd}, @_);
+}
+
 sub make_first_existing_target {
 	my $this=shift;
 	my $targets=shift;
 
 	foreach my $target (@$targets) {
 		if ($this->exists_make_target($target)) {
-			$this->doit_in_builddir($this->{makecmd}, $target, @_);
+			$this->do_make($target, @_);
 			return $target;
 		}
 	}
@@ -71,7 +96,7 @@ sub check_auto_buildable {
 
 sub build {
 	my $this=shift;
-	$this->doit_in_builddir($this->{makecmd}, @_);
+	$this->do_make(@_);
 }
 
 sub test {
diff --git a/Debian/Debhelper/Dh_Buildsystems.pm b/Debian/Debhelper/Dh_Buildsystems.pm
index 49862675..39081458 100644
--- a/Debian/Debhelper/Dh_Buildsystems.pm
+++ b/Debian/Debhelper/Dh_Buildsystems.pm
@@ -30,6 +30,7 @@ my $opt_buildsys;
 my $opt_sourcedir;
 my $opt_builddir;
 my $opt_list;
+my $opt_parallel;
 
 sub create_buildsystem_instance {
 	my $system=shift;
@@ -47,6 +48,9 @@ sub create_buildsystem_instance {
 	if (!exists $bsopts{sourcedir} && defined $opt_sourcedir) {
 		$bsopts{sourcedir} = ($opt_sourcedir eq "") ? undef : $opt_sourcedir;
 	}
+	if (!exists $bsopts{parallel}) {
+		$bsopts{parallel} = $opt_parallel;
+	}
 	return $module->new(%bsopts);
 }
 
@@ -122,9 +126,47 @@ sub buildsystems_init {
 
 	    "l" => \$opt_list,
 	    "list" => \$opt_list,
+
+	    "j:i" => \$opt_parallel,
+	    "parallel:i" => \$opt_parallel,
 	);
 	$args{options}{$_} = $options{$_} foreach keys(%options);
 	Debian::Debhelper::Dh_Lib::init(%args);
+
+	# Post-process parallel building option. Initially $opt_parallel may have
+	# such values:
+	# * undef - no --parallel option was specified. This tells buildsystem class
+	#   not to mess with MAKEFLAGS (with the exception of cleaning MAKEFLAGS
+	#   from pointless unavailable jobserver options to avoid warnings) nor
+	#   enable parallel.
+	# * 1 - --parallel=1 option was specified, hence the package should never be
+	#   built in parallel mode. Cleans MAKEFLAGS if needed.
+	# * 0 - --parallel was specified without interger argument meaning package
+	#   does not want to enforce limit on maximum number of parallel processes.
+	# * N > 1 - --parallel=N was specified where N is the maximum number parallel
+	#   processes the package wants to enforce.
+	# Taken DEB_BUILD_OPTIONS and all this into account, set $opt_parallel to the
+	# number of parallel processes to be used for *this* build.
+	if (defined $opt_parallel) {
+		if ($opt_parallel >= 0 && exists $ENV{DEB_BUILD_OPTIONS}) {
+			# Parse parallel=n tag
+			my $n;
+			foreach my $opt (split(/\s+/, $ENV{DEB_BUILD_OPTIONS})) {
+				$n = $1 if $opt =~ /^parallel=(\d+)$/;
+			}
+			if (defined $n && $n > 0) {
+				$opt_parallel = $n if $opt_parallel == 0 || $n < $opt_parallel;
+			}
+			else {
+				# Invalid value in the parallel tag. Disable.
+				$opt_parallel = 1;
+			}
+		}
+		else {
+			# In case invalid number was passed
+			$opt_parallel = 1;
+		}
+	}
 }
 
 sub buildsystems_list {
diff --git a/Debian/Debhelper/Dh_Lib.pm b/Debian/Debhelper/Dh_Lib.pm
index 2d1934be..ebf7db7e 100644
--- a/Debian/Debhelper/Dh_Lib.pm
+++ b/Debian/Debhelper/Dh_Lib.pm
@@ -16,7 +16,7 @@ use vars qw(@ISA @EXPORT %dh);
 	    &compat &addsubstvar &delsubstvar &excludefile &package_arch
 	    &is_udeb &udeb_filename &debhelper_script_subst &escape_shell
 	    &inhibit_log &load_log &write_log &dpkg_architecture_value
-	    &sourcepackage);
+	    &sourcepackage &get_make_jobserver_status);
 
 my $max_compat=7;
 
@@ -205,6 +205,46 @@ sub _error_exitcode {
 	}
 }
 
+# A helper subroutine for detecting (based on MAKEFLAGS) if make jobserver 
+# is enabled, if it is available or MAKEFLAGS contains "jobs" option.
+# It returns current status (jobserver, jobserver-unavailable or jobs-N where
+# N is number of jobs, 0 if infinite) and MAKEFLAGS cleaned up from 
+# job control options.
+sub get_make_jobserver_status {
+	my $jobsre = qr/(?:^|\s)(?:(?:-j\s*|--jobs(?:=|\s+))(\d+)?|--jobs)\b/;
+	my $status = "";
+	my $makeflags;
+
+	if (exists $ENV{MAKEFLAGS}) {
+		$makeflags = $ENV{MAKEFLAGS};
+		if ($makeflags =~ /(?:^|\s)--jobserver-fds=(\d+)/) {
+			$status = "jobserver";
+			if (!open(my $in, "<&", "$1")) {
+				# Job server is unavailable
+				$status .= "-unavailable";
+			}
+			else {
+				close $in;
+			}
+			# Clean makeflags up
+			$makeflags =~ s/(?:^|\s)--jobserver-fds=\S+//g;
+			$makeflags =~ s/(?:^|\s)-j\b//g;
+		}
+		elsif (my @m = ($makeflags =~ /$jobsre/g)) {
+			# Job count is specified in MAKEFLAGS. Whenever make reads it, a new
+			# jobserver will be started. Job count returned is 0 if infinite.
+			$status = "jobs-" . (defined $m[$#m] ? $m[$#m] : "0");
+			# Clean makeflags up from "jobs" option(s)
+			$makeflags =~ s/$jobsre//g;
+		}
+	}
+	if ($status) {
+		# MAKEFLAGS could be unset if it is empty
+		$makeflags = undef if $makeflags =~ /^\s*$/;
+	}
+	return wantarray ? ($status, $makeflags) : $status;
+}
+
 # Run a command that may have a huge number of arguments, like xargs does.
 # Pass in a reference to an array containing the arguments, and then other
 # parameters that are the command and any parameters that should be passed to