New upstream version 1.0.0+git20181105.b02e058

10 files changed, 264 insertions, 22 deletions

diff --git a/README.md b/README.md
index 87e374a..bf49524 100644
--- a/README.md
+++ b/README.md
@@ -12,6 +12,7 @@ Application Options:
           --debug-shell     Fall into interactive shell on error
       -s, --shell=          Redefine interactive shell binary (default: bash)
           --scratchsize=    Size of disk backed scratch space
+      -e, --environ-var=    Environment variables
 
 
 ## Description
@@ -45,12 +46,12 @@ https://godoc.org/github.com/go-debos/debos/actions
 
 ## Installation (under Debian)
 
-    sudo apt install golang
-    sudo apt install libglib2.0-dev libostree-dev
+    sudo apt install golang git libglib2.0-dev libostree-dev qemu-system-x86 \
+         qemu-user-static debootstrap systemd-container
     export GOPATH=/opt/src/gocode # or whatever suites your needs
     go get -u github.com/go-debos/debos/cmd/debos
     /opt/src/gocode/bin/debos --help
-    
+
 ## Simple example
 
 The following example will create a arm64 image, install several
@@ -96,7 +97,41 @@ this:
 This example builds a customized image for a Raspberry Pi 3.
 https://github.com/go-debos/debos-recipes
 
+## Environment variables
+
+debos read a predefined list of environment variables from the host and
+propagates it to fakemachine. The set of environment variables is defined by
+environ_vars on cmd/debos/debos.go. Currently the list of environment variables
+includes the proxy environment variables as documented at:
+
+https://wiki.archlinux.org/index.php/proxy_settings
+
+The list of environment variables currently exported to fakemachine is:
+
+    http_proxy, https_proxy, ftp_proxy, rsync_proxy, all_proxy, no_proxy
+
+While the elements of environ_vars are in lower case, for each element both
+lower and upper case variants are probed on the host, and if found propagated
+to fakemachine. So if the host has the environment variables HTTP_PROXY and
+no_proxy defined, both will be propagated to fakemachine respecting the case.
+
+The command line options --environ-var and -e can be used to specify,
+overwrite, and unset environment variables for fakemachine with the syntax:
+
+$ debos -e ENVIRONVAR:VALUE ...
+
+To unset an enviroment variable, or in other words, to prevent an environment
+variable to be propagated to fakemachine, use the same syntax without a value.
+debos accept multiple -e simultaneously.
+
+## Proxy configuration
+
+While the proxy related environment variables are exported from the host to
+fakemachine, there are two known sources of issues:
+
+* Using localhost will not work from fakemachine. Prefer using an address that is valid on your network. debos will warn if environment variables contain localhost.
+
+* In case you are running applications and/or scripts inside fakemachine you may need to check which are the proxy environment variables they use. Different apps are known to use different environment variable names and different case for environment variable names.
 
 ## See also
 fakemachine at https://github.com/go-debos/fakemachine
-
diff --git a/action.go b/action.go
index b15aabb..b934e9b 100644
--- a/action.go
+++ b/action.go
@@ -36,6 +36,7 @@ type DebosContext struct {
 	DebugShell      string
 	Origins         map[string]string
 	State           DebosState
+	EnvironVars     map[string]string
 }
 
 type Action interface {
diff --git a/actions/debootstrap_action.go b/actions/debootstrap_action.go
index bfbf3dd..b4d6730 100644
--- a/actions/debootstrap_action.go
+++ b/actions/debootstrap_action.go
@@ -21,10 +21,13 @@ Optional properties:
 - check-gpg -- verify GPG signatures on Release files, true by default
 
 - mirror -- URL with Debian-compatible repository
+ If no mirror is specified debos will use http://deb.debian.org/debian as default.
 
 - variant -- name of the bootstrap script variant to use
 
 - components -- list of components to use for packages selection.
+ If no components are specified debos will use main as default.
+
 Example:
  components: [ main, contrib ]
 
@@ -64,8 +67,12 @@ func NewDebootstrapAction() *DebootstrapAction {
 	d.MergedUsr = true
 	// Be secure by default
 	d.CheckGpg = true
-	return &d
+	// Use main as default component
+	d.Components = []string {"main"}
+	// Set generic default mirror
+	d.Mirror = "http://deb.debian.org/debian"
 
+	return &d
 }
 
 func (d *DebootstrapAction) RunSecondStage(context debos.DebosContext) error {
@@ -99,6 +106,8 @@ func (d *DebootstrapAction) Run(context *debos.DebosContext) error {
 
 	if d.MergedUsr {
 		cmdline = append(cmdline, "--merged-usr")
+	} else {
+		cmdline = append(cmdline, "--no-merged-usr")
 	}
 
 	if !d.CheckGpg {
diff --git a/actions/image_partition_action.go b/actions/image_partition_action.go
index cd832ac..ce63464 100644
--- a/actions/image_partition_action.go
+++ b/actions/image_partition_action.go
@@ -112,6 +112,7 @@ import (
 	"fmt"
 	"github.com/docker/go-units"
 	"github.com/go-debos/fakemachine"
+	"gopkg.in/freddierice/go-losetup.v1"
 	"log"
 	"os"
 	"os/exec"
@@ -119,6 +120,7 @@ import (
 	"path/filepath"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/go-debos/debos"
 )
@@ -149,6 +151,7 @@ type ImagePartitionAction struct {
 	Partitions       []Partition
 	Mountpoints      []Mountpoint
 	size             int64
+	loopDev          losetup.Device
 	usingLoop        bool
 }
 
@@ -227,6 +230,14 @@ func (i ImagePartitionAction) formatPartition(p *Partition, context debos.DebosC
 	case "btrfs":
 		// Force formatting to prevent failure in case if partition was formatted already
 		cmdline = append(cmdline, "mkfs.btrfs", "-L", p.Name, "-f")
+	case "hfs":
+		cmdline = append(cmdline, "mkfs.hfs", "-h", "-v", p.Name)
+	case "hfsplus":
+		cmdline = append(cmdline, "mkfs.hfsplus", "-v", p.Name)
+	case "hfsx":
+		cmdline = append(cmdline, "mkfs.hfsplus", "-s", "-v", p.Name)
+		// hfsx is case-insensitive hfs+, should be treated as "normal" hfs+ from now on
+		p.FS = "hfsplus"
 	case "none":
 	default:
 		cmdline = append(cmdline, fmt.Sprintf("mkfs.%s", p.FS), "-L", p.Name)
@@ -266,11 +277,11 @@ func (i *ImagePartitionAction) PreNoMachine(context *debos.DebosContext) error {
 
 	img.Close()
 
-	loop, err := exec.Command("losetup", "-f", "--show", i.ImageName).Output()
+	i.loopDev, err = losetup.Attach(i.ImageName, 0, false)
 	if err != nil {
 		return fmt.Errorf("Failed to setup loop device")
 	}
-	context.Image = strings.TrimSpace(string(loop[:]))
+	context.Image = i.loopDev.Path()
 	i.usingLoop = true
 
 	return nil
@@ -317,6 +328,8 @@ func (i ImagePartitionAction) Run(context *debos.DebosContext) error {
 		switch p.FS {
 		case "vfat":
 			command = append(command, "fat32")
+		case "hfsplus":
+			command = append(command, "hfs+")
 		case "none":
 		default:
 			command = append(command, p.FS)
@@ -378,11 +391,34 @@ func (i ImagePartitionAction) Cleanup(context *debos.DebosContext) error {
 	for idx := len(i.Mountpoints) - 1; idx >= 0; idx-- {
 		m := i.Mountpoints[idx]
 		mntpath := path.Join(context.ImageMntDir, m.Mountpoint)
-		syscall.Unmount(mntpath, 0)
+		err := syscall.Unmount(mntpath, 0)
+		if err != nil {
+			log.Printf("Warning: Failed to get unmount %s: %s", m.Mountpoint, err)
+			log.Printf("Unmount failure can cause images being incomplete!")
+			return err
+		}
 	}
 
 	if i.usingLoop {
-		exec.Command("losetup", "-d", context.Image).Run()
+		err := i.loopDev.Detach()
+		if err != nil {
+			log.Printf("WARNING: Failed to detach loop device: %s", err)
+			return err
+		}
+
+		for t := 0; t < 60; t++ {
+			err = i.loopDev.Remove()
+			if err == nil {
+				break
+			}
+			log.Printf("Loop dev couldn't remove %s, waiting", err)
+			time.Sleep(time.Second)
+		}
+
+		if err != nil {
+			log.Printf("WARNING: Failed to remove loop device: %s", err)
+			return err
+		}
 	}
 
 	return nil
diff --git a/actions/ostree_deploy_action.go b/actions/ostree_deploy_action.go
index d69600d..dab2265 100644
--- a/actions/ostree_deploy_action.go
+++ b/actions/ostree_deploy_action.go
@@ -52,6 +52,7 @@ import (
 	"io"
 	"os"
 	"path"
+	"runtime"
 	"strings"
 
 	"github.com/go-debos/debos"
@@ -191,5 +192,6 @@ func (ot *OstreeDeployAction) Run(context *debos.DebosContext) error {
 		return err
 	}
 
+	runtime.GC()
 	return nil
 }
diff --git a/actions/raw_action.go b/actions/raw_action.go
index d3e66fe..af8c31f 100644
--- a/actions/raw_action.go
+++ b/actions/raw_action.go
@@ -116,6 +116,7 @@ func (raw *RawAction) Run(context *debos.DebosContext) error {
 	if err != nil {
 		return fmt.Errorf("Failed to open %s: %v", devicePath, err)
 	}
+	defer target.Close()
 
 	offset, err := strconv.ParseInt(raw.Offset, 0, 64)
 	if err != nil {
@@ -123,7 +124,12 @@ func (raw *RawAction) Run(context *debos.DebosContext) error {
 	}
 	bytes, err := target.WriteAt(content, offset)
 	if bytes != len(content) {
-		return errors.New("Couldn't write complete data")
+		return fmt.Errorf("Couldn't write complete data %v", err)
+	}
+
+	err = target.Sync()
+	if err != nil {
+		return fmt.Errorf("Couldn't sync content %v", err)
 	}
 
 	return nil
diff --git a/actions/run_action.go b/actions/run_action.go
index c6115cc..7bae989 100644
--- a/actions/run_action.go
+++ b/actions/run_action.go
@@ -10,6 +10,7 @@ Yaml syntax:
    postprocess: bool
    script: script name
    command: command line
+   label: string
 
 Properties 'command' and 'script' are mutually exclusive.
 
@@ -26,6 +27,9 @@ access to the filesystem ($ROOTDIR), the image if any ($IMAGE), the
 recipe directory ($RECIPEDIR) and the artifact directory ($ARTIFACTDIR).
 In both cases it is run with root privileges.
 
+- label -- if non-empty, this string is used to label output. If empty,
+a label is derived from the command or script.
+
 - postprocess -- if set script or command is executed after all other commands and
 has access to the image file.
 
@@ -49,6 +53,7 @@ type RunAction struct {
 	PostProcess      bool
 	Script           string
 	Command          string
+	Label            string
 }
 
 func (run *RunAction) Verify(context *debos.DebosContext) error {
@@ -103,6 +108,10 @@ func (run *RunAction) doRun(context debos.DebosContext) error {
 		label = run.Command
 	}
 
+	if run.Label != "" {
+		label = run.Label
+	}
+
 	// Command/script with options passed as single string
 	cmdline = append([]string{"sh", "-c"}, cmdline...)
 
diff --git a/cmd/debos/debos.go b/cmd/debos/debos.go
index f0f0ce3..20f5253 100644
--- a/cmd/debos/debos.go
+++ b/cmd/debos/debos.go
@@ -6,6 +6,7 @@ import (
 	"log"
 	"os"
 	"path"
+	"strings"
 
 	"github.com/docker/go-units"
 	"github.com/go-debos/debos"
@@ -25,6 +26,35 @@ func checkError(context *debos.DebosContext, err error, a debos.Action, stage st
 	return 1
 }
 
+func do_run(r recipe.Recipe, context *debos.DebosContext) int {
+	for _, a := range r.Actions {
+		err := a.Run(context)
+
+		// This does not stop the call of stacked Cleanup methods for other Actions
+		// Stack Cleanup methods
+		defer a.Cleanup(context)
+
+		// Check the state of Run method
+		if exitcode := checkError(context, err, a, "Run"); exitcode != 0 {
+			return exitcode
+		}
+	}
+
+	return 0
+}
+
+func warnLocalhost(variable string, value string) {
+	message := `WARNING: Environment variable %[1]s contains a reference to
+		    localhost. This may not work when running from fakemachine.
+		    Consider using an address that is valid on your network.`
+
+	if strings.Contains(value, "localhost") ||
+	   strings.Contains(value, "127.0.0.1") ||
+	   strings.Contains(value, "::1") {
+		log.Printf(message, variable)
+	}
+}
+
 func main() {
 	var context debos.DebosContext
 	var options struct {
@@ -37,6 +67,19 @@ func main() {
 		CPUs          int               `short:"c" long:"cpus" description:"Number of CPUs to use for build VM (default: 2)"`
 		Memory        string            `short:"m" long:"memory" description:"Amount of memory for build VM (default: 2048MB)"`
 		ShowBoot      bool              `long:"show-boot" description:"Show boot/console messages from the fake machine"`
+		EnvironVars   map[string]string `short:"e" long:"environ-var" description:"Environment variables (use -e VARIABLE:VALUE syntax)"`
+	}
+
+	// These are the environment variables that will be detected on the
+	// host and propagated to fakemachine. These are listed lower case, but
+	// they are detected and configured in both lower case and upper case.
+	var environ_vars = [...]string {
+		"http_proxy",
+		"https_proxy",
+		"ftp_proxy",
+		"rsync_proxy",
+		"all_proxy",
+		"no_proxy",
 	}
 
 	var exitcode int = 0
@@ -117,6 +160,34 @@ func main() {
 
 	context.State = debos.Success
 
+	// Initialize environment variables map
+	context.EnvironVars = make(map[string]string)
+
+	// First add variables from host
+	for _, e := range environ_vars {
+		lowerVar := strings.ToLower(e) // lowercase not really needed
+		lowerVal := os.Getenv(lowerVar)
+		if lowerVal != "" {
+			context.EnvironVars[lowerVar] = lowerVal
+		}
+
+		upperVar := strings.ToUpper(e)
+		upperVal := os.Getenv(upperVar)
+		if upperVal != "" {
+			context.EnvironVars[upperVar] = upperVal
+		}
+	}
+
+	// Then add/overwrite with variables from command line
+	for k, v := range options.EnvironVars {
+		// Allows the user to unset environ variables with -e
+		if v == "" {
+			delete(context.EnvironVars, k)
+		} else {
+			context.EnvironVars[k] = v
+		}
+	}
+
 	for _, a := range r.Actions {
 		err = a.Verify(&context)
 		if exitcode = checkError(&context, err, a, "Verify"); exitcode != 0 {
@@ -158,6 +229,16 @@ func main() {
 
 		m.SetShowBoot(options.ShowBoot)
 
+		// Puts in a format that is compatible with output of os.Environ()
+		if context.EnvironVars != nil {
+			EnvironString := []string{}
+			for k, v := range context.EnvironVars {
+				warnLocalhost(k, v)
+				EnvironString = append(EnvironString, fmt.Sprintf("%s=%s", k, v))
+			}
+			m.SetEnviron(EnvironString) // And save the resulting environ vars on m
+		}
+
 		m.AddVolume(context.Artifactdir)
 		args = append(args, "--artifactdir", context.Artifactdir)
 
@@ -165,6 +246,10 @@ func main() {
 			args = append(args, "--template-var", fmt.Sprintf("%s:\"%s\"", k, v))
 		}
 
+		for k, v := range options.EnvironVars {
+			args = append(args, "--environ-var", fmt.Sprintf("%s:\"%s\"", k, v))
+		}
+
 		m.AddVolume(context.RecipeDir)
 		args = append(args, file)
 
@@ -226,17 +311,9 @@ func main() {
 		}
 	}
 
-	for _, a := range r.Actions {
-		err = a.Run(&context)
-
-		// This does not stop the call of stacked Cleanup methods for other Actions
-		// Stack Cleanup methods
-		defer a.Cleanup(&context)
-
-		// Check the state of Run method
-		if exitcode = checkError(&context, err, a, "Run"); exitcode != 0 {
-			return
-		}
+	exitcode = do_run(r, &context)
+	if exitcode != 0 {
+		return
 	}
 
 	if !fakemachine.InMachine() {
diff --git a/commands.go b/commands.go
index 96e2de1..aa562e3 100644
--- a/commands.go
+++ b/commands.go
@@ -69,6 +69,12 @@ func (w *commandWrapper) flush() {
 func NewChrootCommandForContext(context DebosContext) Command {
 	c := Command{Architecture: context.Architecture, Chroot: context.Rootdir, ChrootMethod: CHROOT_METHOD_NSPAWN}
 
+	if context.EnvironVars != nil {
+		for k, v := range context.EnvironVars {
+			c.AddEnv(fmt.Sprintf("%s=%s", k, v))
+		}
+	}
+
 	if context.Image != "" {
 		path, err := RealPath(context.Image)
 		if err == nil {
diff --git a/doc/man/debos.1 b/doc/man/debos.1
index 6e58a40..9af586c 100644
--- a/doc/man/debos.1
+++ b/doc/man/debos.1
@@ -1,4 +1,4 @@
-.\" Automatically generated by Pandoc 2.2.1
+.\" Automatically generated by Pandoc 2.0.6
 .\"
 .TH "debos" "1" "" "" ""
 .hy
@@ -27,6 +27,7 @@ Application Options:
 \ \ \ \ \ \ \-\-debug\-shell\ \ \ \ \ Fall\ into\ interactive\ shell\ on\ error
 \ \ \-s,\ \-\-shell=\ \ \ \ \ \ \ \ \ \ Redefine\ interactive\ shell\ binary\ (default:\ bash)
 \ \ \ \ \ \ \-\-scratchsize=\ \ \ \ Size\ of\ disk\ backed\ scratch\ space
+\ \ \-e,\ \-\-environ\-var=\ \ \ \ Environment\ variables
 \f[]
 .fi
 .SH DESCRIPTION
@@ -72,6 +73,17 @@ unpack: unpack files from archive in the filesystem
 .PP
 A full syntax description of all the debos actions can be found at:
 https://godoc.org/github.com/go\-debos/debos/actions
+.SH INSTALLATION (UNDER DEBIAN)
+.IP
+.nf
+\f[C]
+sudo\ apt\ install\ golang
+sudo\ apt\ install\ libglib2.0\-dev\ libostree\-dev
+export\ GOPATH=/opt/src/gocode\ #\ or\ whatever\ suites\ your\ needs
+go\ get\ \-u\ github.com/go\-debos/debos/cmd/debos
+/opt/src/gocode/bin/debos\ \-\-help
+\f[]
+.fi
 .SH SIMPLE EXAMPLE
 .PP
 The following example will create a arm64 image, install several
@@ -127,6 +139,55 @@ debos\ \-t\ image:"debian\-arm64.tgz"\ example.yaml
 .PP
 This example builds a customized image for a Raspberry Pi 3.
 https://github.com/go\-debos/debos\-recipes
+.SH ENVIRONMENT VARIABLES
+.PP
+debos read a predefined list of environment variables from the host and
+propagates it to fakemachine.
+The set of environment variables is defined by environ_vars on
+cmd/debos/debos.go.
+Currently the list of environment variables includes the proxy
+environment variables as documented at:
+.PP
+https://wiki.archlinux.org/index.php/proxy_settings
+.PP
+The list of environment variables currently exported to fakemachine is:
+.IP
+.nf
+\f[C]
+http_proxy,\ https_proxy,\ ftp_proxy,\ rsync_proxy,\ all_proxy,\ no_proxy
+\f[]
+.fi
+.PP
+While the elements of environ_vars are in lower case, for each element
+both lower and upper case variants are probed on the host, and if found
+propagated to fakemachine.
+So if the host has the environment variables HTTP_PROXY and no_proxy
+defined, both will be propagated to fakemachine respecting the case.
+.PP
+The command line options \[en]environ\-var and \-e can be used to
+specify, overwrite, and unset environment variables for fakemachine with
+the syntax:
+.PP
+$ debos \-e ENVIRONVAR:VALUE \&...
+.PP
+To unset an enviroment variable, or in other words, to prevent an
+environment variable to be propagated to fakemachine, use the same
+syntax without a value.
+debos accept multiple \-e simultaneously.
+.SH PROXY CONFIGURATION
+.PP
+While the proxy related environment variables are exported from the host
+to fakemachine, there are two known sources of issues:
+.IP \[bu] 2
+Using localhost will not work from fakemachine.
+Prefer using an address that is valid on your network.
+debos will warn if environment variables contain localhost.
+.IP \[bu] 2
+In case you are running applications and/or scripts inside fakemachine
+you may need to check which are the proxy environment variables they
+use.
+Different apps are known to use different environment variable names and
+different case for environment variable names.
 .SH SEE ALSO
 .PP
 fakemachine at https://github.com/go\-debos/fakemachine