Update upstream source from tag 'upstream/1.0.0+git20191223.292995b'

Update to upstream version '1.0.0+git20191223.292995b'
with Debian dir f7f3a5ade924e6cb45402aac0b363e5f82aa5ae4

17 files changed, 192 insertions, 16 deletions

diff --git a/actions/debootstrap_action.go b/actions/debootstrap_action.go
index b4d6730..77ee63d 100644
--- a/actions/debootstrap_action.go
+++ b/actions/debootstrap_action.go
@@ -3,6 +3,10 @@ Debootstrap Action
 
 Construct the target rootfs with debootstrap tool.
 
+Please keep in mind -- file `/etc/resolv.conf` will be removed after execution.
+Most of the OS scripts used by `debootstrap` copy `resolv.conf` from the host,
+and this may lead to incorrect configuration when becoming part of the created rootfs.
+
 Yaml syntax:
  - action: debootstrap
    mirror: URL
@@ -174,6 +178,14 @@ func (d *DebootstrapAction) Run(context *debos.DebosContext) error {
 	}
 	srclist.Close()
 
+	/* Cleanup resolv.conf after debootstrap */
+	resolvconf := path.Join(context.Rootdir, "/etc/resolv.conf")
+	if _, err = os.Stat(resolvconf); !os.IsNotExist(err) {
+		if err = os.Remove(resolvconf); err != nil {
+			return err
+		}
+	}
+
 	c := debos.NewChrootCommandForContext(*context)
 
 	return c.Run("apt clean", "/usr/bin/apt-get", "clean")
diff --git a/actions/image_partition_action.go b/actions/image_partition_action.go
index b4f0cd2..0850990 100644
--- a/actions/image_partition_action.go
+++ b/actions/image_partition_action.go
@@ -41,6 +41,7 @@ Yaml syntax for partitions:
 	   fs: filesystem
 	   start: offset
 	   end: offset
+	   features: list of filesystem features
 	   flags: list of flags
 	   fsck: bool
 
@@ -62,6 +63,9 @@ form -- '32MB', '1GB' or as disk percentage -- '100%'.
 
 Optional properties:
 
+- features -- list of additional filesystem features which need to be enabled
+for partition.
+
 - flags -- list of additional flags for partition compatible with parted(8)
 'set' command.
 
@@ -138,14 +142,15 @@ import (
 )
 
 type Partition struct {
-	number int
-	Name   string
-	Start  string
-	End    string
-	FS     string
-	Flags  []string
-	Fsck   bool "fsck"
-	FSUUID string
+	number   int
+	Name     string
+	Start    string
+	End      string
+	FS       string
+	Flags    []string
+	Features []string
+	Fsck     bool "fsck"
+	FSUUID   string
 }
 
 type Mountpoint struct {
@@ -268,6 +273,9 @@ func (i ImagePartitionAction) formatPartition(p *Partition, context debos.DebosC
 	case "btrfs":
 		// Force formatting to prevent failure in case if partition was formatted already
 		cmdline = append(cmdline, "mkfs.btrfs", "-L", p.Name, "-f")
+		if len(p.Features) > 0 {
+			cmdline = append(cmdline, "-O", strings.Join(p.Features, ","))
+		}
 	case "hfs":
 		cmdline = append(cmdline, "mkfs.hfs", "-h", "-v", p.Name)
 	case "hfsplus":
@@ -279,6 +287,9 @@ func (i ImagePartitionAction) formatPartition(p *Partition, context debos.DebosC
 	case "none":
 	default:
 		cmdline = append(cmdline, fmt.Sprintf("mkfs.%s", p.FS), "-L", p.Name)
+		if len(p.Features) > 0 {
+			cmdline = append(cmdline, "-O", strings.Join(p.Features, ","))
+		}
 	}
 
 	if len(cmdline) != 0 {
diff --git a/cmd/debos/debos.go b/cmd/debos/debos.go
index 611162a..0f3edc9 100644
--- a/cmd/debos/debos.go
+++ b/cmd/debos/debos.go
@@ -72,6 +72,7 @@ func main() {
 		Verbose       bool              `short:"v" long:"verbose" description:"Verbose output"`
 		PrintRecipe   bool              `long:"print-recipe" description:"Print final recipe"`
 		DryRun        bool              `long:"dry-run" description:"Compose final recipe to build but without any real work started"`
+		DisableFakeMachine bool         `long:"disable-fakemachine" description:"Do not use fakemachine."`
 	}
 
 	// These are the environment variables that will be detected on the
@@ -143,7 +144,7 @@ func main() {
 	/* If fakemachine is supported the outer fake machine will never use the
 	 * scratchdir, so just set it to /scratch as a dummy to prevent the
 	 * outer debos creating a temporary direction */
-	if fakemachine.InMachine() || fakemachine.Supported() {
+	if !options.DisableFakeMachine && (fakemachine.InMachine() || fakemachine.Supported()) {
 		context.Scratchdir = "/scratch"
 	} else {
 		log.Printf("fakemachine not supported, running on the host!")
@@ -212,7 +213,7 @@ func main() {
 		return
 	}
 
-	if !fakemachine.InMachine() && fakemachine.Supported() {
+	if !options.DisableFakeMachine && !fakemachine.InMachine() && fakemachine.Supported() {
 		m := fakemachine.NewMachine()
 		var args []string
 
diff --git a/commands.go b/commands.go
index 0ea2b9a..52704c5 100644
--- a/commands.go
+++ b/commands.go
@@ -2,8 +2,10 @@ package debos
 
 import (
 	"bytes"
+	"crypto/sha256"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -115,9 +117,101 @@ func (cmd *Command) AddBindMount(source, target string) {
 	cmd.bindMounts = append(cmd.bindMounts, mount)
 }
 
+func (cmd *Command) saveResolvConf() (*[sha256.Size]byte, error) {
+	hostconf := "/etc/resolv.conf"
+	chrootedconf := path.Join(cmd.Chroot, hostconf)
+	savedconf := chrootedconf + ".debos"
+	var sum [sha256.Size]byte
+
+	if cmd.ChrootMethod == CHROOT_METHOD_NONE {
+		return nil, nil
+	}
+
+	// There may not be an existing resolv.conf
+	if _, err := os.Lstat(chrootedconf); !os.IsNotExist(err) {
+		if err = os.Rename(chrootedconf, savedconf); err != nil {
+			return nil, err
+		}
+	}
+
+	/* Expect a relatively small file here */
+	data, err := ioutil.ReadFile(hostconf)
+	if err != nil {
+		return nil, err
+	}
+	out := []byte("# Automatically generated by Debos\n")
+	out = append(out, data...)
+
+	sum = sha256.Sum256(out)
+
+	err = ioutil.WriteFile(chrootedconf, out, 0644)
+	if err != nil {
+		return nil, err
+	}
+
+	return &sum, nil
+}
+
+func (cmd *Command) restoreResolvConf(sum *[sha256.Size]byte) error {
+	hostconf := "/etc/resolv.conf"
+	chrootedconf := path.Join(cmd.Chroot, hostconf)
+	savedconf := chrootedconf + ".debos"
+
+	if cmd.ChrootMethod == CHROOT_METHOD_NONE || sum == nil {
+		return nil
+	}
+
+	// Remove the original copy anyway
+	defer os.Remove(savedconf)
+
+	fi, err := os.Lstat(chrootedconf)
+
+	// resolv.conf was removed during the command call
+	// Nothing to do with it -- file has been changed anyway
+	if os.IsNotExist(err) {
+		return nil
+	}
+
+	mode := fi.Mode()
+	switch {
+	case mode.IsRegular():
+		// Try to calculate checksum
+		data, err := ioutil.ReadFile(chrootedconf)
+		if err != nil {
+			return err
+		}
+		currentsum := sha256.Sum256(data)
+
+		// Leave the changed resolv.conf untouched
+		if bytes.Compare(currentsum[:], (*sum)[:]) == 0 {
+			// Remove the generated version
+			if err := os.Remove(chrootedconf); err != nil {
+				return err
+			}
+
+			if _, err := os.Lstat(savedconf); !os.IsNotExist(err) {
+				// Restore the original version
+				if err = os.Rename(savedconf, chrootedconf); err != nil {
+					return err
+				}
+			}
+		}
+	case mode&os.ModeSymlink != 0:
+		// If the 'resolv.conf' is a symlink
+		// Nothing to do with it -- file has been changed anyway
+	default:
+		// File is not regular or symlink
+		// Let's get out here with verbose message
+		log.Printf("Warning: /etc/resolv.conf inside the chroot is not a regular file")
+	}
+
+	return nil
+}
+
 func (cmd Command) Run(label string, cmdline ...string) error {
 	q := newQemuHelper(cmd)
 	q.Setup()
+	defer q.Cleanup()
 
 	var options []string
 	switch cmd.ChrootMethod {
@@ -128,7 +222,8 @@ func (cmd Command) Run(label string, cmdline ...string) error {
 		options = append(options, cmd.Chroot)
 		options = append(options, cmdline...)
 	case CHROOT_METHOD_NSPAWN:
-		options = append(options, "systemd-nspawn", "-q", "-D", cmd.Chroot)
+		// We use own resolv.conf handling
+		options = append(options, "systemd-nspawn", "-q", "--resolv-conf=off", "-D", cmd.Chroot)
 		for _, e := range cmd.extraEnv {
 			options = append(options, "--setenv", e)
 
@@ -147,6 +242,8 @@ func (cmd Command) Run(label string, cmdline ...string) error {
 	exe.Stdout = w
 	exe.Stderr = w
 
+	defer w.flush()
+
 	if len(cmd.extraEnv) > 0 && cmd.ChrootMethod != CHROOT_METHOD_NSPAWN {
 		exe.Env = append(os.Environ(), cmd.extraEnv...)
 	}
@@ -158,11 +255,22 @@ func (cmd Command) Run(label string, cmdline ...string) error {
 		defer services.Allow()
 	}
 
-	err := exe.Run()
-	w.flush()
-	q.Cleanup()
+	// Save the original resolv.conf and copy version from host
+	resolvsum, err := cmd.saveResolvConf()
+	if err != nil {
+		return err
+	}
+
+	if err = exe.Run(); err != nil {
+		return err
+	}
+
+	// Restore the original resolv.conf if not changed
+	if err = cmd.restoreResolvConf(resolvsum); err != nil {
+		return err
+	}
 
-	return err
+	return nil
 }
 
 type qemuHelper struct {
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 16059d9..252c295 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -64,6 +64,7 @@ RUN apt-get update && \
         dosfstools \
         e2fsprogs \
         gzip \
+        pigz \
         libostree-1-1 \
         linux-image-amd64 \
         parted \
@@ -72,6 +73,7 @@ RUN apt-get update && \
         qemu-user-static \
         systemd \
         systemd-container \
+        unzip \
         xz-utils && \
     rm -rf /var/lib/apt/lists/*
 
diff --git a/docker/recipes.test.yml b/docker/recipes.test.yml
index 18d4aca..fde59b5 100644
--- a/docker/recipes.test.yml
+++ b/docker/recipes.test.yml
@@ -11,4 +11,4 @@ services:
             source: ./tests
             target: /recipes
         working_dir: /recipes
-        command: simple-recipe.yaml
+        entrypoint: ./tests.sh
diff --git a/docker/tests/base.tar.gz b/docker/tests/base.tar.gz
deleted file mode 100644
index cf787ff..0000000
--- a/docker/tests/base.tar.gz
+++ /dev/null
diff --git a/docker/tests/separatedirs/base.tar.gz b/docker/tests/separatedirs/base.tar.gz
new file mode 100644
index 0000000..0c8dc6d
--- /dev/null
+++ b/docker/tests/separatedirs/base.tar.gz
diff --git a/docker/tests/separatedirs/main.yaml b/docker/tests/separatedirs/main.yaml
new file mode 100644
index 0000000..f99d0e5
--- /dev/null
+++ b/docker/tests/separatedirs/main.yaml
@@ -0,0 +1,14 @@
+architecture: amd64
+
+actions:
+  - action: unpack
+    origin: recipe
+    compression: gz
+    file: base.tar.gz
+
+  - action: recipe
+    description: Load recipe from separate dir
+    recipe: ../subdirs/overlay-a/overlay-A.yaml
+
+  - action: run
+    command: find ${ROOTDIR}
diff --git a/docker/tests/A/A/A/A/a.txt b/docker/tests/simple/A/A/A/A/a.txt
index 7898192..7898192 100644
--- a/docker/tests/A/A/A/A/a.txt
+++ b/docker/tests/simple/A/A/A/A/a.txt
diff --git a/docker/tests/simple/base.tar.gz b/docker/tests/simple/base.tar.gz
new file mode 100644
index 0000000..0c8dc6d
--- /dev/null
+++ b/docker/tests/simple/base.tar.gz
diff --git a/docker/tests/simple-recipe.yaml b/docker/tests/simple/main.yaml
index b4f99f2..b4f99f2 100644
--- a/docker/tests/simple-recipe.yaml
+++ b/docker/tests/simple/main.yaml
diff --git a/docker/tests/subdirs/base.tar.gz b/docker/tests/subdirs/base.tar.gz
new file mode 100644
index 0000000..0c8dc6d
--- /dev/null
+++ b/docker/tests/subdirs/base.tar.gz
diff --git a/docker/tests/subdirs/main.yaml b/docker/tests/subdirs/main.yaml
new file mode 100644
index 0000000..abea663
--- /dev/null
+++ b/docker/tests/subdirs/main.yaml
@@ -0,0 +1,14 @@
+architecture: amd64
+
+actions:
+  - action: unpack
+    origin: recipe
+    compression: gz
+    file: base.tar.gz
+
+  - action: recipe
+    description: Load recipe from subdir
+    recipe: overlay-a/overlay-A.yaml
+
+  - action: run
+    command: find ${ROOTDIR}
diff --git a/docker/tests/subdirs/overlay-a/A/A/A/A/a.txt b/docker/tests/subdirs/overlay-a/A/A/A/A/a.txt
new file mode 100644
index 0000000..7898192
--- /dev/null
+++ b/docker/tests/subdirs/overlay-a/A/A/A/A/a.txt
@@ -0,0 +1 @@
+a
diff --git a/docker/tests/subdirs/overlay-a/overlay-A.yaml b/docker/tests/subdirs/overlay-a/overlay-A.yaml
new file mode 100644
index 0000000..f044176
--- /dev/null
+++ b/docker/tests/subdirs/overlay-a/overlay-A.yaml
@@ -0,0 +1,5 @@
+architecture: amd64
+
+actions:
+   - action: overlay
+     source: A
diff --git a/docker/tests/tests.sh b/docker/tests/tests.sh
new file mode 100755
index 0000000..9239d72
--- /dev/null
+++ b/docker/tests/tests.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+for dir in simple subdirs separatedirs ; do
+    pushd ${dir}
+    debos main.yaml
+    popd
+done