summaryrefslogtreecommitdiff
path: root/t
diff options
context:
space:
mode:
authorModestas Vainius <modestas@vainius.eu>2009-10-28 15:49:34 -0400
committerJoey Hess <joey@gnu.kitenet.net>2009-10-28 15:49:34 -0400
commitdace0773fe5f66fdf040f54383322e21a65fa1e8 (patch)
treed03e084c1980708e31773b1b422cfec4e5546c7d /t
parent834d95aaba24e5cee1c5a74078a09443f5e122a7 (diff)
Support parallel building in makefile buildsystem
1) Add routine to Dh_Lib (used by dh and makefile.pm) which is capable of detecting make jobserver and job control options from the MAKEFLAGS environment variable. It also generates and returns a clean up MAKEFLAGS from these options. 2) Add --parallel option to build system framework which allows source packages to specify that they support parallel building. Optional value for this option is the number of maximum parallel process to allow. However, the actual number of parallel process (if any) for the specific build is determined from DEB_BUILD_OPTIONS env variable as specified by Debian Policy. By default (no --parallel option) parallel is neither enabled nor disabled (depends on the external environment). However, dh may pass --parallel to dh_auto_* implicitly in case 4) described below. 3) Add parallel support for makefile buildsystem. This implementation forcefully starts a new make job server (or disables parallel) for the number of process requested. If --parallel was not passed to the build system at all, the build system will only clean up MAKEFLAGS from stale jobserver options to avoid pointless make warnings. 4) If dh detects that it is being run by dpkg-buildpackage -jX and it is NOT run with "+" prefix from debian/rules (i.e. jobserver is not reachable), it enables --parallel implicitly. This closes: #532805. Signed-off-by: Modestas Vainius <modestas@vainius.eu>
Diffstat (limited to 't')
-rwxr-xr-xt/buildsystems/buildsystem_tests275
-rw-r--r--t/buildsystems/parallel.mk21
2 files changed, 288 insertions, 8 deletions
diff --git a/t/buildsystems/buildsystem_tests b/t/buildsystems/buildsystem_tests
index 8f7a275..41c0f97 100755
--- a/t/buildsystems/buildsystem_tests
+++ b/t/buildsystems/buildsystem_tests
@@ -1,6 +1,6 @@
#!/usr/bin/perl
-use Test::More tests => 228;
+use Test::More tests => 273;
use strict;
use warnings;
@@ -49,12 +49,32 @@ sub process_stdout {
my ($cmdline, $stdin) = @_;
my ($reader, $writer);
- open2($reader, $writer, $cmdline) or die "Unable to exec $cmdline";
+ my $pid = open2($reader, $writer, $cmdline) or die "Unable to exec $cmdline";
print $writer $stdin if $stdin;
close $writer;
+ waitpid($pid, 0);
+ $? = $? >> 8; # exit status
return readlines($reader);
}
+sub write_debian_rules {
+ my $contents=shift;
+ my $backup;
+
+ if (-f "debian/rules") {
+ (undef, $backup) = tempfile(DIR => ".", OPEN => 0);
+ rename "debian/rules", $backup;
+ }
+ # Write debian/rules if requested
+ if ($contents) {
+ open(my $f, ">", "debian/rules");
+ print $f $contents;;
+ close($f);
+ chmod 0755, "debian/rules";
+ }
+ return $backup;
+}
+
### Test Buildsystem class API methods
is( $BS_CLASS->canonpath("path/to/the/./nowhere/../../somewhere"),
"path/to/somewhere", "canonpath no1" );
@@ -384,17 +404,19 @@ EOF
}
$tmp = Cwd::getcwd();
-is_deeply( process_stdout("$^X -- - --builddirectory='autoconf/bld dir' --sourcedirectory autoconf",
+# NOTE: disabling parallel building explicitly (it might get automatically
+# enabled if run under dpkg-buildpackage -jX) to make output deterministic.
+is_deeply( process_stdout("$^X -- - --builddirectory='autoconf/bld dir' --sourcedirectory autoconf --parallel=1",
get_load_bs_source(undef, "configure")),
- [ 'NAME=autoconf', 'builddir=autoconf/bld dir', "cwd=$tmp", 'makecmd=make', 'sourcedir=autoconf' ],
+ [ 'NAME=autoconf', 'builddir=autoconf/bld dir', "cwd=$tmp", 'makecmd=make', 'parallel=1', 'sourcedir=autoconf' ],
"autoconf autoselection and sourcedir/builddir" );
-is_deeply( process_stdout("$^X -- - -Sautoconf -D autoconf", get_load_bs_source("autoconf", "build")),
- [ 'NAME=autoconf', 'builddir=undef', "cwd=$tmp", 'makecmd=make', 'sourcedir=autoconf' ],
+is_deeply( process_stdout("$^X -- - -Sautoconf -D autoconf --parallel=1", get_load_bs_source("autoconf", "build")),
+ [ 'NAME=autoconf', 'builddir=undef', "cwd=$tmp", 'makecmd=make', 'parallel=1', 'sourcedir=autoconf' ],
"forced autoconf and sourcedir" );
-is_deeply( process_stdout("$^X -- - -B -Sautoconf", get_load_bs_source("autoconf", "build")),
- [ 'NAME=autoconf', "builddir=$default_builddir", "cwd=$tmp", 'makecmd=make', 'sourcedir=.' ],
+is_deeply( process_stdout("$^X -- - -B -Sautoconf --parallel=1", get_load_bs_source("autoconf", "build")),
+ [ 'NAME=autoconf', "builddir=$default_builddir", "cwd=$tmp", 'makecmd=make', 'parallel=1', 'sourcedir=.' ],
"forced autoconf and default build directory" );
# Build the autoconf test package
@@ -459,6 +481,243 @@ dh_auto_do_autoconf('autoconf');
dh_auto_do_autoconf('autoconf', 'bld/dir', configure_args => [ "--extra-autoconf-configure-arg" ]);
ok ( ! -e 'bld', "bld got deleted too" );
+#### Test parallel building and related options / routines
+@tmp = ( $ENV{MAKEFLAGS}, $ENV{DEB_BUILD_OPTIONS} );
+
+# Test get_make_jobserver_status() sub
+
+$ENV{MAKEFLAGS} = "--jobserver-fds=103,104 -j";
+is_deeply( [ get_make_jobserver_status() ], [ "jobserver-unavailable", undef ],
+ "get_make_jobserver_status(): unavailable jobserver, unset makeflags" );
+
+$ENV{MAKEFLAGS} = "-a --jobserver-fds=103,104 -j -b";
+is_deeply( [ get_make_jobserver_status() ], [ "jobserver-unavailable", "-a -b" ],
+ "get_make_jobserver_status(): unavailable jobserver, clean makeflags" );
+
+$ENV{MAKEFLAGS} = " --jobserver-fds=1,2 -j ";
+is_deeply( [ get_make_jobserver_status() ], [ "jobserver", undef ],
+ "get_make_jobserver_status(): jobserver (available), clean makeflags" );
+
+$ENV{MAKEFLAGS} = "-a -j -b";
+is_deeply( [ get_make_jobserver_status() ], [ "jobs-0", "-a -b" ],
+ "get_make_jobserver_status(): -j" );
+
+$ENV{MAKEFLAGS} = "-a --jobs -b";
+is_deeply( [ get_make_jobserver_status() ], [ "jobs-0", "-a -b" ],
+ "get_make_jobserver_status(): --jobs" );
+
+$ENV{MAKEFLAGS} = "-j6";
+is_deeply( [ get_make_jobserver_status() ], [ "jobs-6", undef ],
+ "get_make_jobserver_status(): -j6" );
+
+$ENV{MAKEFLAGS} = "-a -j6 --jobs=7";
+is_deeply( [ get_make_jobserver_status() ], [ "jobs-7", "-a" ],
+ "get_make_jobserver_status(): -j6 --jobs=7" );
+
+$ENV{MAKEFLAGS} = "-j6 --jobserver-fds=5,6 --jobs=8";
+is_deeply( [ get_make_jobserver_status() ], [ "jobserver-unavailable", "-j6 --jobs=8" ],
+ "get_make_jobserver_status(): mixed jobserver and -j/--jobs" );
+
+# Test parallel building with makefile build system.
+$ENV{MAKEFLAGS} = "";
+$ENV{DEB_BUILD_OPTIONS} = "";
+
+sub do_parallel_mk {
+ my $dh_opts=shift || "";
+ my $make_opts=shift || "";
+ return process_stdout(
+ "LANG=C LC_ALL=C LC_MESSAGES=C $TOPDIR/dh_auto_build -Smakefile $dh_opts " .
+ "-- -s -f parallel.mk $make_opts 2>&1 >/dev/null", "");
+}
+
+sub test_isnt_parallel {
+ my ($got, $desc) = @_;
+ my @makemsgs = grep /^make[\d\[\]]*:/, @$got;
+ if (@makemsgs) {
+ like( $makemsgs[0], qr/Error 10/, $desc );
+ }
+ else {
+ ok( scalar(@makemsgs) > 0, $desc );
+ }
+}
+
+sub test_is_parallel {
+ my ($got, $desc) = @_;
+ is_deeply( $got, [] , $desc );
+ is( $?, 0, "(exit status=0) $desc");
+}
+
+test_isnt_parallel( do_parallel_mk(),
+ "No parallel by default" );
+test_isnt_parallel( do_parallel_mk("--parallel"),
+ "No parallel by default with --parallel" );
+
+$ENV{DEB_BUILD_OPTIONS}="parallel=5";
+test_isnt_parallel( do_parallel_mk(),
+ "DEB_BUILD_OPTIONS=parallel=5 without --parallel" );
+test_is_parallel( do_parallel_mk("--parallel"),
+ "DEB_BUILD_OPTIONS=parallel=5 with --parallel" );
+test_is_parallel( do_parallel_mk("--parallel=2"),
+ "DEB_BUILD_OPTIONS=parallel=5 with --parallel=2" );
+test_isnt_parallel( do_parallel_mk("--parallel=1"),
+ "DEB_BUILD_OPTIONS=parallel=5 with --parallel=1 (off)" );
+
+$ENV{MAKEFLAGS} = "--jobserver-fds=105,106 -j";
+$ENV{DEB_BUILD_OPTIONS}="";
+test_isnt_parallel( do_parallel_mk(),
+ "makefile.pm (no parallel): no make warnings about unavailable jobserver" );
+$ENV{DEB_BUILD_OPTIONS}="parallel=5";
+test_is_parallel( do_parallel_mk("--parallel"),
+ "DEB_BUILD_OPTIONS=parallel=5 with --parallel: no make warnings about unavail parent jobserver" );
+
+$ENV{MAKEFLAGS} = "-j2";
+$ENV{DEB_BUILD_OPTIONS}="";
+test_is_parallel( do_parallel_mk(),
+ "MAKEFLAGS=-j2 without --parallel: dh_auto_build honours MAKEFLAGS" );
+test_isnt_parallel( do_parallel_mk("--parallel=1"),
+ "MAKEFLAGS=-j2 with --parallel=1: dh_auto_build enforces -j1" );
+
+# Test dh dpkg-buildpackage -jX detection
+sub do_rules_for_parallel {
+ my $cmdline=shift || "";
+ my $stdin=shift || "";
+ return process_stdout("LANG=C LC_ALL=C LC_MESSAGES=C PATH=$TOPDIR:\$PATH " .
+ "make -f - $cmdline 2>&1 >/dev/null", $stdin);
+}
+
+# Simulate dpkg-buildpackage -j5
+doit("ln", "-s", "parallel.mk", "Makefile");
+
+sub test_dh_parallel {
+ my $extra_dsc=shift || "";
+ my $debian_rules=shift || "";
+ my $rules;
+ my $tmpfile;
+
+ $ENV{MAKEFLAGS} = "-j5";
+ $ENV{DEB_BUILD_OPTIONS} = "parallel=5";
+
+ # Write debian/rules if requested
+ $tmpfile = write_debian_rules($debian_rules);
+
+ $rules = <<'EOF';
+%:
+ @dh_clean > /dev/null 2>&1
+ @dh --buildsystem=makefile --after=dh_auto_configure --until=dh_auto_build $@
+ @dh_clean > /dev/null 2>&1
+EOF
+ test_is_parallel( do_rules_for_parallel("build", $rules),
+ "dh adds --parallel implicitly under dpkg-buildpackage -j5 $extra_dsc");
+
+ $ENV{MAKEFLAGS} = "";
+ test_isnt_parallel( do_rules_for_parallel("build", $rules),
+ "DEB_BUILD_OPTIONS=parallel=5 without MAKEFLAGS=-jX via dh $extra_dsc" );
+
+ $ENV{MAKEFLAGS} = "-j5";
+ $rules = <<'EOF';
+%:
+ @dh_clean > /dev/null 2>&1
+ @dh -j1 --buildsystem=makefile --after=dh_auto_configure --until=dh_auto_build $@
+ @dh_clean > /dev/null 2>&1
+EOF
+ test_isnt_parallel( do_rules_for_parallel("build", $rules),
+ "dh -j1 disables implicit parallel under dpkg-buildpackage -j5 $extra_dsc");
+
+ $rules = <<'EOF';
+%:
+ @dh_clean > /dev/null 2>&1
+ @dh -j --buildsystem=makefile --after=dh_auto_configure --until=dh_auto_build $@
+ @dh_clean > /dev/null 2>&1
+EOF
+ test_is_parallel( do_rules_for_parallel("build", $rules),
+ "dh -j under dpkg-buildpackage -j5 is parallel $extra_dsc");
+ $ENV{MAKEFLAGS} = "";
+ test_is_parallel( do_rules_for_parallel("build", $rules),
+ "dh -j is parallel only with DEB_BUILD_OPTIONS=parallel=5 $extra_dsc");
+
+ if (defined $tmpfile) {
+ rename($tmpfile, "debian/rules");
+ }
+ elsif ($debian_rules) {
+ unlink("debian/rules");
+ }
+}
+
+# dh should pass the same tests with and without overrides
+test_dh_parallel();
+test_dh_parallel("(with overrides)", <<'EOF');
+#!/usr/bin/make -f
+override_dh_auto_build:
+ @dh_auto_build -- -f parallel.mk
+EOF
+
+# Test if legacy punctuation hacks (+) work as before
+$ENV{MAKEFLAGS} = "-j5";
+$ENV{DEB_BUILD_OPTIONS} = "parallel=5";
+$tmp = write_debian_rules(<<'EOF');
+#!/usr/bin/make -f
+%:
+ @dh_clean > /dev/null 2>&1
+ @+dh --buildsystem=makefile --after=dh_auto_configure --until=dh_auto_build $@
+ @dh_clean > /dev/null 2>&1
+EOF
+test_is_parallel( do_rules_for_parallel("build", "include debian/rules"),
+ "legacy punctuation hacks: +dh, no override" );
+unlink "debian/rules";
+
+write_debian_rules(<<'EOF');
+#!/usr/bin/make -f
+override_dh_auto_build:
+ dh_auto_build
+%:
+ @dh_clean > /dev/null 2>&1
+ @+dh --buildsystem=makefile --after=dh_auto_configure --until=dh_auto_build $@
+ @dh_clean > /dev/null 2>&1
+EOF
+test_isnt_parallel( do_rules_for_parallel("build", "include debian/rules"),
+ "legacy punctuation hacks: +dh, override without +, no parallel, no make warnings" );
+unlink "debian/rules";
+
+write_debian_rules(<<'EOF');
+#!/usr/bin/make -f
+override_dh_auto_build:
+ +dh_auto_build
+%:
+ @dh_clean > /dev/null 2>&1
+ @+dh --buildsystem=makefile --after=dh_auto_configure --until=dh_auto_build $@
+ @dh_clean > /dev/null 2>&1
+EOF
+test_is_parallel( do_rules_for_parallel("build", "include debian/rules"),
+ "legacy punctuation hacks: +dh, override with +" );
+unlink "debian/rules";
+
+write_debian_rules(<<'EOF');
+#!/usr/bin/make -f
+override_dh_auto_build:
+ $(MAKE)
+%:
+ @dh_clean > /dev/null 2>&1
+ @+dh --buildsystem=makefile --after=dh_auto_configure --until=dh_auto_build $@
+ @dh_clean > /dev/null 2>&1
+EOF
+test_is_parallel( do_rules_for_parallel("build", "include debian/rules"),
+ "legacy punctuation hacks: +dh, override with \$(MAKE)" );
+unlink "debian/rules";
+
+if (defined $tmp) {
+ rename($tmp, "debian/rules");
+}
+else {
+ unlink("debian/rules");
+}
+
+# Clean up after parallel testing
+END {
+ system("rm", "-f", "Makefile");
+}
+$ENV{MAKEFLAGS} = $tmp[0] if defined $tmp[0];
+$ENV{DEB_BUILD_OPTIONS} = $tmp[1] if defined $tmp[1];
+
END {
system("rm", "-rf", $tmpdir);
system("$TOPDIR/dh_clean");
diff --git a/t/buildsystems/parallel.mk b/t/buildsystems/parallel.mk
new file mode 100644
index 0000000..3e0d201
--- /dev/null
+++ b/t/buildsystems/parallel.mk
@@ -0,0 +1,21 @@
+all: FIRST SECOND
+
+TMPFILE ?= $(CURDIR)/parallel.mk.lock
+
+rmtmpfile:
+ @rm -f "$(TMPFILE)"
+
+FIRST: rmtmpfile
+ @c=0; \
+ while [ $$c -le 5 ] && \
+ ([ ! -e "$(TMPFILE)" ] || [ "`cat "$(TMPFILE)"`" != "SECOND" ]); do \
+ c=$$(($$c+1)); \
+ sleep 0.1; \
+ done; \
+ rm -f "$(TMPFILE)"; \
+ if [ $$c -gt 5 ]; then exit 10; else exit 0; fi
+
+SECOND: rmtmpfile
+ @echo $@ > "$(TMPFILE)"
+
+.PHONY: all FIRST SECOND rmtmpfile