Merge pull request #115 from jblunck/livebuild

Support for building Debian Live Systems

9 files changed, 342 insertions, 5 deletions

diff --git a/Build.pm b/Build.pm
index d681420..6705513 100644
--- a/Build.pm
+++ b/Build.pm
@@ -11,6 +11,7 @@ our $do_rpm;
 our $do_deb;
 our $do_kiwi;
 our $do_arch;
+our $do_livebuild;
 
 sub import {
   for (@_) {
@@ -18,8 +19,9 @@ sub import {
     $do_deb = 1 if $_ eq ':deb';
     $do_kiwi = 1 if $_ eq ':kiwi';
     $do_arch = 1 if $_ eq ':arch';
+    $do_livebuild = 1 if $_ eq ':livebuild';
   }
-  $do_rpm = $do_deb = $do_kiwi = $do_arch = 1 if !$do_rpm && !$do_deb && !$do_kiwi && !$do_arch;
+  $do_rpm = $do_deb = $do_kiwi = $do_arch = $do_livebuild = 1 if !$do_rpm && !$do_deb && !$do_kiwi && !$do_arch;
   if ($do_deb) {
     require Build::Deb;
   }
@@ -29,6 +31,9 @@ sub import {
   if ($do_arch) {
     require Build::Arch;
   }
+  if ($do_livebuild) {
+    require Build::LiveBuild;
+  }
 }
 
 package Build::Features;
@@ -327,7 +332,7 @@ sub read_config {
   }
   if (!$config->{'binarytype'}) {
     $config->{'binarytype'} = 'rpm' if $config->{'type'} eq 'spec' || $config->{'type'} eq 'kiwi';
-    $config->{'binarytype'} = 'deb' if $config->{'type'} eq 'dsc';
+    $config->{'binarytype'} = 'deb' if $config->{'type'} eq 'dsc' || $config->{'type'} eq 'livebuild';
     $config->{'binarytype'} = 'arch' if $config->{'type'} eq 'arch';
     $config->{'binarytype'} ||= 'UNDEFINED';
   }
@@ -918,6 +923,7 @@ sub parse {
   return Build::Kiwi::parse($cf, $fn, @args) if $do_kiwi && $fn =~ /\.kiwi$/;
   return Build::Arch::parse($cf, $fn, @args) if $do_arch && $fn =~ /(^|\/|-)PKGBUILD$/;
   return parse_preinstallimage($cf, $fn, @args) if $fn =~ /(^|\/|-)_preinstallimage$/;
+  return Build::LiveBuild::parse($cf, $fn, @args) if $do_livebuild && $fn =~ /\.livebuild$/;
   return undef;
 }
 
diff --git a/Build/LiveBuild.pm b/Build/LiveBuild.pm
new file mode 100644
index 0000000..b7076ff
--- /dev/null
+++ b/Build/LiveBuild.pm
@@ -0,0 +1,108 @@
+#
+# Author: Jan Blunck <jblunck@infradead.org>
+#
+# This file is part of build.
+#
+# build is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# build is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with build.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+package Build::LiveBuild;
+
+use strict;
+use Archive::Tar;
+
+sub filter {
+  my ( $content ) = @_;
+
+  return '' unless defined $content;
+
+  $content =~ s/^#.*$//mg;
+  $content =~ s/^!.*$//mg;
+  $content =~ s/^\s*//mg;
+  return $content;
+}
+
+sub parse_package_list {
+  my ( $content ) = @_;
+  my @packages = split /\n/, filter($content);
+
+  return @packages;
+};
+
+sub parse_archive {
+  my ( $content ) = @_;
+  my @repos = ( );
+
+  my @lines = split /\n/, filter($content);
+  for (@lines) {
+    next if $_ =~ /^deb-src /;
+
+    die("bad path using not obs:/ URL: $_\n") unless $_ =~ /^deb\s+obs:\/\/\/?([^\s\/]+)\/([^\s\/]+)\/?\s+.*$/;
+    push @repos, "$1/$2";
+  }
+
+  return @repos;
+}
+
+sub unify {
+  my %h = map {$_ => 1} @_;
+  return grep(delete($h{$_}), @_);
+}
+
+sub parse {
+  my ($config, $filename, @args) = @_;
+
+  # TODO: check that filename exists
+
+  # check that filename is a tar
+  my $tar = Archive::Tar->new;
+  $tar->read($filename) || die "Read failed: $filename";
+
+  # check that directory layout matches live-build directory structure
+
+  # TODO: add dependency injection package based on $LB_DISTRIBUTION
+  my @packages = ( 'live-build-desc-wheezy' );
+
+  my @lb4_requirements = (
+    'live-boot', 'live-config', 'e2fsprogs', 'squashfs-tools', 'mtd-tools',
+    'dosfstools', 'parted', 'grub', 'syslinux', 'syslinux-common',
+    'librsvg2-bin', 'xorriso', 'zsync', 'apt-utils', 'dctrl-tools',
+    'debconf', 'wget' );
+
+  push @packages, @lb4_requirements;
+
+  for my $file( $tar->list_files('') ) {
+    next unless $file =~ /^config\/package-lists\/.*/;
+    push @packages, parse_package_list($tar->get_content($file));
+  }
+
+  my @repos;
+  for my $file( $tar->list_files('') ) {
+    next unless $file =~ /^config\/archives\/.*\.list.*/;
+    push @repos, parse_archive($tar->get_content($file));
+  }
+
+  my $ret = {};
+  ( $ret->{'name'} = $filename ) =~ s/\.[^.]+$//;
+  $ret->{'deps'} = [ unify(@packages) ];
+  $ret->{'path'} = [ unify(@repos) ];
+  for (@{$ret->{'path'}}) {
+    my @s = split('/', $_, 2);
+    $_ = {'project' => $s[0], 'repository' => $s[1]};
+  }
+
+  return $ret;
+}
+
+1;
diff --git a/build-recipe b/build-recipe
index e50c2a7..d6ffe69 100644
--- a/build-recipe
+++ b/build-recipe
@@ -6,7 +6,7 @@
 
 KIWI_PARAMETERS=
 
-for i in spec dsc kiwi arch preinstallimage mock ; do
+for i in spec dsc kiwi arch preinstallimage mock livebuild; do
     . "$BUILD_DIR/build-recipe-$i"
 done
 
@@ -55,6 +55,7 @@ recipe_set_buildtype() {
         *.kiwi) BUILDTYPE=kiwi ;;
         PKGBUILD) BUILDTYPE=arch ;;
         _preinstallimage) BUILDTYPE=preinstallimage ;;
+        *.livebuild) BUILDTYPE=livebuild ;;
     esac
     if test -z "$BUILDTYPE" ; then
        echo "I don't know how to build $RECIPEFILE"
diff --git a/build-recipe-livebuild b/build-recipe-livebuild
new file mode 100644
index 0000000..7b6f095
--- /dev/null
+++ b/build-recipe-livebuild
@@ -0,0 +1,123 @@
+#
+# Debian live-build specific functions.
+#
+# Author: Jan Blunck <jblunck@infradead.org>
+#
+# This file is part of build.
+#
+# build is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# build is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with build.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+recipe_setup_livebuild() {
+
+    TOPDIR=/usr/src/packages
+    rm -rf "$BUILD_ROOT$TOPDIR"
+    for i in OTHER SOURCES LIVEBUILD_ROOT ; do
+	mkdir -p "$BUILD_ROOT$TOPDIR/$i"
+    done
+    chown -R "$ABUILD_UID:$ABUILD_GID" "$BUILD_ROOT$TOPDIR"
+    if test "$MYSRCDIR" = $BUILD_ROOT/.build-srcdir ; then
+	mv "$MYSRCDIR"/* $BUILD_ROOT$TOPDIR/SOURCES/
+    else
+	if test -z "$LINKSOURCES" ; then
+	    cp -dLR "$MYSRCDIR"/* $BUILD_ROOT$TOPDIR/SOURCES/
+	else
+	    cp -lR "$MYSRCDIR"/* $BUILD_ROOT$TOPDIR/SOURCES/
+	fi
+	if test "$?" != 0 ; then
+	    echo "source copy failed"
+	    cleanup_and_exit 1
+	fi
+    fi
+}
+
+recipe_prepare_livebuild() {
+    :
+}
+
+# This script expects that the $BUILD_ROOT is a Debian installation with
+# live-build already installed!
+#
+# Variables:
+# $BUILD_ROOT the Debian chroot
+# $TOPDIR/SOURCES includes the live-build config tarball
+# $TOPDIR/LIVEBUILD_ROOT where live-build will be called
+# $RECIPEFILE the name of the live-build config tarball
+
+recipe_build_livebuild() {
+
+    echo "Creating repository metadata"
+    cat > $BUILD_ROOT/.build.run_livebuild.tmp.sh <<EOF
+cd $TOPDIR/SOURCES/repos || exit 1
+apt-ftparchive packages . > Packages
+gzip -c9 Packages > Packages.gz
+apt-ftparchive sources . > Sources
+gzip -c9 Sources > Sources.gz
+apt-ftparchive release . > Release
+EOF
+    chroot $BUILD_ROOT su -c "sh /.build.run_livebuild.tmp.sh" - root
+    local RESULT=$?
+    rm -f $BUILD_ROOT/.build.run_livebuild.tmp.sh
+    [ "${RESULT}" != 0 ] && cleanup_and_exit 1
+
+    # Expand live-build configuration to $TOPDIR/LIVEBUILD_ROOT
+    echo "Expanding live-build configuration"
+    tar -xvf $BUILD_ROOT/$TOPDIR/SOURCES/$RECIPEFILE \
+	-C $BUILD_ROOT/$TOPDIR/LIVEBUILD_ROOT || cleanup_and_exit 1
+
+    if [ -f $BUILD_ROOT/$TOPDIR/SOURCES/livebuild_pre_run ] ; then
+	cp $BUILD_ROOT/$TOPDIR/SOURCES/livebuild_pre_run \
+	    $BUILD_ROOT/.build.livebuild_pre_run
+	chmod +x $BUILD_ROOT/.build.livebuild_pre_run
+	echo "Running package livebuild_pre_run hook"
+	chroot $BUILD_ROOT su -c "/.build.livebuild_pre_run" - root \
+	    < /dev/null || cleanup_and_exit 1
+    elif [ -x $BUILD_ROOT/usr/lib/build/livebuild_pre_run ] ; then
+	echo "Running OBS build livebuild_pre_run hook"
+	chroot $BUILD_ROOT su -c "/usr/lib/build/livebuild_pre_run" - root \
+	    < /dev/null || cleanup_and_exit 1
+    fi
+
+    chroot $BUILD_ROOT su -c "cd $TOPDIR/LIVEBUILD_ROOT && lb build" - root \
+	< /dev/null || cleanup_and_exit 1
+
+    # Move created product to destination
+    for i in $BUILD_ROOT/$TOPDIR/LIVEBUILD_ROOT/* ; do
+	test -f "$i" || continue
+	case "${i##*/}" in
+	    *.iso)
+		# all created files share the same name without suffix
+		mv ${i%%.iso}.* $BUILD_ROOT/$TOPDIR/OTHER/.
+		BUILD_SUCCEEDED=true
+		;;
+	    *)
+		;;
+	esac
+    done
+
+    # Fail the build if no ISO was created
+    if [ -z "$(ls $BUILD_ROOT/$TOPDIR/OTHER/*.iso)" ] ; then
+	echo "No ISO image found"
+	cleanup_and_exit 1
+    fi
+}
+
+recipe_resultdirs_livebuild() {
+    # our results are already in OTHERS
+    echo ""
+}
+
+# Local Variables:
+# mode: Shell-script
+# End:
diff --git a/build-vm b/build-vm
index 4f1e5e2..124926f 100644
--- a/build-vm
+++ b/build-vm
@@ -601,6 +601,8 @@ vm_first_stage() {
 	mkdir "$BUILD_ROOT/.build-srcdir"
 	if test "$BUILDTYPE" = kiwi ; then
 	    cp -pRL "$MYSRCDIR"/* $BUILD_ROOT/.build-srcdir
+        elif test "$BUILDTYPE" = livebuild ; then
+            cp -pRL "$MYSRCDIR"/* $BUILD_ROOT/.build-srcdir
 	else
 	    cp -p "$MYSRCDIR"/* $BUILD_ROOT/.build-srcdir
 	fi
diff --git a/expanddeps b/expanddeps
index 60dac44..9d61645 100755
--- a/expanddeps
+++ b/expanddeps
@@ -62,8 +62,8 @@ while (@ARGV)  {
 $archs = '' unless defined $archs;
 die("you must specfiy a depfile!\n") unless defined $rpmdeps;
 
-my @extradeps = grep {!/(^|\/)(?:PKGBUILD|_preinstallimage)$/ && !/\.(?:spec|dsc|kiwi)$/} @ARGV;
-my @recipes = grep {/(^|\/)(?:PKGBUILD|_preinstallimage)$/ || /\.(?:spec|dsc|kiwi)$/} @ARGV;
+my @extradeps = grep {!/(^|\/)(?:PKGBUILD|_preinstallimage)$/ && !/\.(?:spec|dsc|kiwi|livebuild)$/} @ARGV;
+my @recipes = grep {/(^|\/)(?:PKGBUILD|_preinstallimage)$/ || /\.(?:spec|dsc|kiwi|livebuild)$/} @ARGV;
 die("can only work with at most one recipe file\n") if @recipes > 1;
 my $recipe = $recipes[0];
 
@@ -260,6 +260,12 @@ if ($recipe) {
     }
     push @kdeps, grep {/^kiwi-.*:/} @{$d->{'deps'} || []};
     $d = { 'deps' => \@kdeps, 'subpacks' => [] };
+  } elsif ($recipe =~ /\.livebuild$/) {
+    # just set up live-build root for now
+    $d = {
+      'deps' => [ 'live-build' ],
+      'subpacks' => [],
+    };
   } else {
     $d = Build::parse($cf, $recipe);
   }
diff --git a/livebuild_pre_run.template b/livebuild_pre_run.template
new file mode 100755
index 0000000..d88dad6
--- /dev/null
+++ b/livebuild_pre_run.template
@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+# This is a template for a livebuild_pre_run script. These scripts are
+# executed by build_livebuild.sh in the chroot environment.
+#
+
+fix_debootstrap()
+{
+    # debootstrap in Debian 7.0 does not like dash
+
+    if [ -x /usr/sbin/debootstrap ] ; then
+	sed -i 's|^#!/bin/sh|#!/bin/bash|' /usr/sbin/debootstrap
+    fi
+}
+
+fix_lb_bootstrap_archive-keys()
+{
+    if [ -e /usr/lib/live/build/bootstrap_archive-keys ] ; then
+	sed -i '/apt-get update/{ s/^/#/ }' \
+	    /usr/lib/live/build/bootstrap_archive-keys
+    fi
+}
+
+#
+# main
+#
+
+: ${TOPDIR:=/usr/src/packages}
+
+# Distribution and live-build specific hooks
+fix_debootstrap
+fix_lb_bootstrap_archive-keys
+
+# Expand configuration based on defaults
+cd $TOPDIR/LIVEBUILD_ROOT && lb config || exit 1
+
+# Replace all occurances of LB_MIRROR with local repository
+sed -i "s|^\(LB_MIRROR_[^=]\+=\).*|\1\"file:$TOPDIR/SOURCES/repos/\"|" \
+    $TOPDIR/LIVEBUILD_ROOT/config/bootstrap
+sed -i "s|^\(LB_PARENT_MIRROR_[^=]\+=\).*|\1\"file:$TOPDIR/SOURCES/repos/\"|" \
+    $TOPDIR/LIVEBUILD_ROOT/config/bootstrap
+
+# Prevent debootstrap from cleaning our cache
+sed -i 's|^\(LB_CACHE_PACKAGES=\).*|\1"false"|' \
+    $TOPDIR/LIVEBUILD_ROOT/config/common
+
+# Disable GPG checking
+sed -i 's|^\(LB_APT_SECURE=\).*|\1"false"|' \
+    $TOPDIR/LIVEBUILD_ROOT/config/common
diff --git a/t/live-build b/t/live-build
new file mode 100755
index 0000000..022f9d3
--- /dev/null
+++ b/t/live-build
@@ -0,0 +1,42 @@
+#!/usr/bin/perl -w -I ..
+
+use strict;
+use Test::More tests => 3;
+use Build::LiveBuild;
+
+use Data::Dumper;
+use Digest::MD5 qw(md5_hex);
+
+my $VAR = '
+
+
+a
+
+b
+#comment
+
+! whatever
+';
+
+is(Build::LiveBuild::filter($VAR), 'a
+b
+');
+
+
+my $DEB_ARCHIVE = '
+# comment
+deb obs://openSUSE.org:Debian:7.0/standard wheezy main contrib
+deb-src obs://openSUSE.org:Debian:7.0/standard wheezy main contrib
+
+';
+my $DEB_ARCHIVE_RESULT = "\$VAR1 = 'openSUSE.org:Debian:7.0/standard';
+";
+
+is(Dumper(Build::LiveBuild::parse_archive($DEB_ARCHIVE)), $DEB_ARCHIVE_RESULT);
+
+my $config = {};
+$Data::Dumper::Sortkeys = 1;
+is(md5_hex(Dumper(Build::LiveBuild::parse( $config, 'standard.livebuild'))),
+   '0e68b6473bc8dbf00e0c6f1b53af3efd');
+
+#print Dumper(Build::LiveBuild::parse( $config, 'standard.livebuild'));
diff --git a/t/standard.livebuild b/t/standard.livebuild
new file mode 100644
index 0000000..af0d4ed
--- /dev/null
+++ b/t/standard.livebuild