From fef5746ee329eb560d740132895ed29a44e44d10 Mon Sep 17 00:00:00 2001 From: Andrej Shadura Date: Thu, 16 Jan 2020 15:25:17 +0100 Subject: New upstream version 1.0.0+git20191223.292995b --- actions/debootstrap_action.go | 12 +++ actions/image_partition_action.go | 27 ++++-- cmd/debos/debos.go | 5 +- commands.go | 118 ++++++++++++++++++++++++-- docker/Dockerfile | 2 + docker/recipes.test.yml | 2 +- docker/tests/A/A/A/A/a.txt | 1 - docker/tests/base.tar.gz | Bin 289 -> 0 bytes docker/tests/separatedirs/base.tar.gz | Bin 0 -> 631 bytes docker/tests/separatedirs/main.yaml | 14 +++ docker/tests/simple-recipe.yaml | 12 --- docker/tests/simple/A/A/A/A/a.txt | 1 + docker/tests/simple/base.tar.gz | Bin 0 -> 631 bytes docker/tests/simple/main.yaml | 12 +++ docker/tests/subdirs/base.tar.gz | Bin 0 -> 631 bytes docker/tests/subdirs/main.yaml | 14 +++ docker/tests/subdirs/overlay-a/A/A/A/A/a.txt | 1 + docker/tests/subdirs/overlay-a/overlay-A.yaml | 5 ++ docker/tests/tests.sh | 8 ++ 19 files changed, 205 insertions(+), 29 deletions(-) delete mode 100644 docker/tests/A/A/A/A/a.txt delete mode 100644 docker/tests/base.tar.gz create mode 100644 docker/tests/separatedirs/base.tar.gz create mode 100644 docker/tests/separatedirs/main.yaml delete mode 100644 docker/tests/simple-recipe.yaml create mode 100644 docker/tests/simple/A/A/A/A/a.txt create mode 100644 docker/tests/simple/base.tar.gz create mode 100644 docker/tests/simple/main.yaml create mode 100644 docker/tests/subdirs/base.tar.gz create mode 100644 docker/tests/subdirs/main.yaml create mode 100644 docker/tests/subdirs/overlay-a/A/A/A/A/a.txt create mode 100644 docker/tests/subdirs/overlay-a/overlay-A.yaml create mode 100755 docker/tests/tests.sh diff --git a/actions/debootstrap_action.go b/actions/debootstrap_action.go index b4d6730..77ee63d 100644 --- a/actions/debootstrap_action.go +++ b/actions/debootstrap_action.go @@ -3,6 +3,10 @@ Debootstrap Action Construct the target rootfs with debootstrap tool. +Please keep in mind -- file `/etc/resolv.conf` will be removed after execution. +Most of the OS scripts used by `debootstrap` copy `resolv.conf` from the host, +and this may lead to incorrect configuration when becoming part of the created rootfs. + Yaml syntax: - action: debootstrap mirror: URL @@ -174,6 +178,14 @@ func (d *DebootstrapAction) Run(context *debos.DebosContext) error { } srclist.Close() + /* Cleanup resolv.conf after debootstrap */ + resolvconf := path.Join(context.Rootdir, "/etc/resolv.conf") + if _, err = os.Stat(resolvconf); !os.IsNotExist(err) { + if err = os.Remove(resolvconf); err != nil { + return err + } + } + c := debos.NewChrootCommandForContext(*context) return c.Run("apt clean", "/usr/bin/apt-get", "clean") diff --git a/actions/image_partition_action.go b/actions/image_partition_action.go index b4f0cd2..0850990 100644 --- a/actions/image_partition_action.go +++ b/actions/image_partition_action.go @@ -41,6 +41,7 @@ Yaml syntax for partitions: fs: filesystem start: offset end: offset + features: list of filesystem features flags: list of flags fsck: bool @@ -62,6 +63,9 @@ form -- '32MB', '1GB' or as disk percentage -- '100%'. Optional properties: +- features -- list of additional filesystem features which need to be enabled +for partition. + - flags -- list of additional flags for partition compatible with parted(8) 'set' command. @@ -138,14 +142,15 @@ import ( ) type Partition struct { - number int - Name string - Start string - End string - FS string - Flags []string - Fsck bool "fsck" - FSUUID string + number int + Name string + Start string + End string + FS string + Flags []string + Features []string + Fsck bool "fsck" + FSUUID string } type Mountpoint struct { @@ -268,6 +273,9 @@ func (i ImagePartitionAction) formatPartition(p *Partition, context debos.DebosC case "btrfs": // Force formatting to prevent failure in case if partition was formatted already cmdline = append(cmdline, "mkfs.btrfs", "-L", p.Name, "-f") + if len(p.Features) > 0 { + cmdline = append(cmdline, "-O", strings.Join(p.Features, ",")) + } case "hfs": cmdline = append(cmdline, "mkfs.hfs", "-h", "-v", p.Name) case "hfsplus": @@ -279,6 +287,9 @@ func (i ImagePartitionAction) formatPartition(p *Partition, context debos.DebosC case "none": default: cmdline = append(cmdline, fmt.Sprintf("mkfs.%s", p.FS), "-L", p.Name) + if len(p.Features) > 0 { + cmdline = append(cmdline, "-O", strings.Join(p.Features, ",")) + } } if len(cmdline) != 0 { diff --git a/cmd/debos/debos.go b/cmd/debos/debos.go index 611162a..0f3edc9 100644 --- a/cmd/debos/debos.go +++ b/cmd/debos/debos.go @@ -72,6 +72,7 @@ func main() { Verbose bool `short:"v" long:"verbose" description:"Verbose output"` PrintRecipe bool `long:"print-recipe" description:"Print final recipe"` DryRun bool `long:"dry-run" description:"Compose final recipe to build but without any real work started"` + DisableFakeMachine bool `long:"disable-fakemachine" description:"Do not use fakemachine."` } // These are the environment variables that will be detected on the @@ -143,7 +144,7 @@ func main() { /* If fakemachine is supported the outer fake machine will never use the * scratchdir, so just set it to /scratch as a dummy to prevent the * outer debos creating a temporary direction */ - if fakemachine.InMachine() || fakemachine.Supported() { + if !options.DisableFakeMachine && (fakemachine.InMachine() || fakemachine.Supported()) { context.Scratchdir = "/scratch" } else { log.Printf("fakemachine not supported, running on the host!") @@ -212,7 +213,7 @@ func main() { return } - if !fakemachine.InMachine() && fakemachine.Supported() { + if !options.DisableFakeMachine && !fakemachine.InMachine() && fakemachine.Supported() { m := fakemachine.NewMachine() var args []string diff --git a/commands.go b/commands.go index 0ea2b9a..52704c5 100644 --- a/commands.go +++ b/commands.go @@ -2,8 +2,10 @@ package debos import ( "bytes" + "crypto/sha256" "fmt" "io" + "io/ioutil" "log" "os" "os/exec" @@ -115,9 +117,101 @@ func (cmd *Command) AddBindMount(source, target string) { cmd.bindMounts = append(cmd.bindMounts, mount) } +func (cmd *Command) saveResolvConf() (*[sha256.Size]byte, error) { + hostconf := "/etc/resolv.conf" + chrootedconf := path.Join(cmd.Chroot, hostconf) + savedconf := chrootedconf + ".debos" + var sum [sha256.Size]byte + + if cmd.ChrootMethod == CHROOT_METHOD_NONE { + return nil, nil + } + + // There may not be an existing resolv.conf + if _, err := os.Lstat(chrootedconf); !os.IsNotExist(err) { + if err = os.Rename(chrootedconf, savedconf); err != nil { + return nil, err + } + } + + /* Expect a relatively small file here */ + data, err := ioutil.ReadFile(hostconf) + if err != nil { + return nil, err + } + out := []byte("# Automatically generated by Debos\n") + out = append(out, data...) + + sum = sha256.Sum256(out) + + err = ioutil.WriteFile(chrootedconf, out, 0644) + if err != nil { + return nil, err + } + + return &sum, nil +} + +func (cmd *Command) restoreResolvConf(sum *[sha256.Size]byte) error { + hostconf := "/etc/resolv.conf" + chrootedconf := path.Join(cmd.Chroot, hostconf) + savedconf := chrootedconf + ".debos" + + if cmd.ChrootMethod == CHROOT_METHOD_NONE || sum == nil { + return nil + } + + // Remove the original copy anyway + defer os.Remove(savedconf) + + fi, err := os.Lstat(chrootedconf) + + // resolv.conf was removed during the command call + // Nothing to do with it -- file has been changed anyway + if os.IsNotExist(err) { + return nil + } + + mode := fi.Mode() + switch { + case mode.IsRegular(): + // Try to calculate checksum + data, err := ioutil.ReadFile(chrootedconf) + if err != nil { + return err + } + currentsum := sha256.Sum256(data) + + // Leave the changed resolv.conf untouched + if bytes.Compare(currentsum[:], (*sum)[:]) == 0 { + // Remove the generated version + if err := os.Remove(chrootedconf); err != nil { + return err + } + + if _, err := os.Lstat(savedconf); !os.IsNotExist(err) { + // Restore the original version + if err = os.Rename(savedconf, chrootedconf); err != nil { + return err + } + } + } + case mode&os.ModeSymlink != 0: + // If the 'resolv.conf' is a symlink + // Nothing to do with it -- file has been changed anyway + default: + // File is not regular or symlink + // Let's get out here with verbose message + log.Printf("Warning: /etc/resolv.conf inside the chroot is not a regular file") + } + + return nil +} + func (cmd Command) Run(label string, cmdline ...string) error { q := newQemuHelper(cmd) q.Setup() + defer q.Cleanup() var options []string switch cmd.ChrootMethod { @@ -128,7 +222,8 @@ func (cmd Command) Run(label string, cmdline ...string) error { options = append(options, cmd.Chroot) options = append(options, cmdline...) case CHROOT_METHOD_NSPAWN: - options = append(options, "systemd-nspawn", "-q", "-D", cmd.Chroot) + // We use own resolv.conf handling + options = append(options, "systemd-nspawn", "-q", "--resolv-conf=off", "-D", cmd.Chroot) for _, e := range cmd.extraEnv { options = append(options, "--setenv", e) @@ -147,6 +242,8 @@ func (cmd Command) Run(label string, cmdline ...string) error { exe.Stdout = w exe.Stderr = w + defer w.flush() + if len(cmd.extraEnv) > 0 && cmd.ChrootMethod != CHROOT_METHOD_NSPAWN { exe.Env = append(os.Environ(), cmd.extraEnv...) } @@ -158,11 +255,22 @@ func (cmd Command) Run(label string, cmdline ...string) error { defer services.Allow() } - err := exe.Run() - w.flush() - q.Cleanup() + // Save the original resolv.conf and copy version from host + resolvsum, err := cmd.saveResolvConf() + if err != nil { + return err + } + + if err = exe.Run(); err != nil { + return err + } + + // Restore the original resolv.conf if not changed + if err = cmd.restoreResolvConf(resolvsum); err != nil { + return err + } - return err + return nil } type qemuHelper struct { diff --git a/docker/Dockerfile b/docker/Dockerfile index 16059d9..252c295 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -64,6 +64,7 @@ RUN apt-get update && \ dosfstools \ e2fsprogs \ gzip \ + pigz \ libostree-1-1 \ linux-image-amd64 \ parted \ @@ -72,6 +73,7 @@ RUN apt-get update && \ qemu-user-static \ systemd \ systemd-container \ + unzip \ xz-utils && \ rm -rf /var/lib/apt/lists/* diff --git a/docker/recipes.test.yml b/docker/recipes.test.yml index 18d4aca..fde59b5 100644 --- a/docker/recipes.test.yml +++ b/docker/recipes.test.yml @@ -11,4 +11,4 @@ services: source: ./tests target: /recipes working_dir: /recipes - command: simple-recipe.yaml + entrypoint: ./tests.sh diff --git a/docker/tests/A/A/A/A/a.txt b/docker/tests/A/A/A/A/a.txt deleted file mode 100644 index 7898192..0000000 --- a/docker/tests/A/A/A/A/a.txt +++ /dev/null @@ -1 +0,0 @@ -a diff --git a/docker/tests/base.tar.gz b/docker/tests/base.tar.gz deleted file mode 100644 index cf787ff..0000000 Binary files a/docker/tests/base.tar.gz and /dev/null differ diff --git a/docker/tests/separatedirs/base.tar.gz b/docker/tests/separatedirs/base.tar.gz new file mode 100644 index 0000000..0c8dc6d Binary files /dev/null and b/docker/tests/separatedirs/base.tar.gz differ diff --git a/docker/tests/separatedirs/main.yaml b/docker/tests/separatedirs/main.yaml new file mode 100644 index 0000000..f99d0e5 --- /dev/null +++ b/docker/tests/separatedirs/main.yaml @@ -0,0 +1,14 @@ +architecture: amd64 + +actions: + - action: unpack + origin: recipe + compression: gz + file: base.tar.gz + + - action: recipe + description: Load recipe from separate dir + recipe: ../subdirs/overlay-a/overlay-A.yaml + + - action: run + command: find ${ROOTDIR} diff --git a/docker/tests/simple-recipe.yaml b/docker/tests/simple-recipe.yaml deleted file mode 100644 index b4f99f2..0000000 --- a/docker/tests/simple-recipe.yaml +++ /dev/null @@ -1,12 +0,0 @@ -architecture: amd64 - -actions: - - action: unpack - compression: gz - file: base.tar.gz - - - action: overlay - source: A - - - action: run - command: find ${ROOTDIR} diff --git a/docker/tests/simple/A/A/A/A/a.txt b/docker/tests/simple/A/A/A/A/a.txt new file mode 100644 index 0000000..7898192 --- /dev/null +++ b/docker/tests/simple/A/A/A/A/a.txt @@ -0,0 +1 @@ +a diff --git a/docker/tests/simple/base.tar.gz b/docker/tests/simple/base.tar.gz new file mode 100644 index 0000000..0c8dc6d Binary files /dev/null and b/docker/tests/simple/base.tar.gz differ diff --git a/docker/tests/simple/main.yaml b/docker/tests/simple/main.yaml new file mode 100644 index 0000000..b4f99f2 --- /dev/null +++ b/docker/tests/simple/main.yaml @@ -0,0 +1,12 @@ +architecture: amd64 + +actions: + - action: unpack + compression: gz + file: base.tar.gz + + - action: overlay + source: A + + - action: run + command: find ${ROOTDIR} diff --git a/docker/tests/subdirs/base.tar.gz b/docker/tests/subdirs/base.tar.gz new file mode 100644 index 0000000..0c8dc6d Binary files /dev/null and b/docker/tests/subdirs/base.tar.gz differ diff --git a/docker/tests/subdirs/main.yaml b/docker/tests/subdirs/main.yaml new file mode 100644 index 0000000..abea663 --- /dev/null +++ b/docker/tests/subdirs/main.yaml @@ -0,0 +1,14 @@ +architecture: amd64 + +actions: + - action: unpack + origin: recipe + compression: gz + file: base.tar.gz + + - action: recipe + description: Load recipe from subdir + recipe: overlay-a/overlay-A.yaml + + - action: run + command: find ${ROOTDIR} diff --git a/docker/tests/subdirs/overlay-a/A/A/A/A/a.txt b/docker/tests/subdirs/overlay-a/A/A/A/A/a.txt new file mode 100644 index 0000000..7898192 --- /dev/null +++ b/docker/tests/subdirs/overlay-a/A/A/A/A/a.txt @@ -0,0 +1 @@ +a diff --git a/docker/tests/subdirs/overlay-a/overlay-A.yaml b/docker/tests/subdirs/overlay-a/overlay-A.yaml new file mode 100644 index 0000000..f044176 --- /dev/null +++ b/docker/tests/subdirs/overlay-a/overlay-A.yaml @@ -0,0 +1,5 @@ +architecture: amd64 + +actions: + - action: overlay + source: A diff --git a/docker/tests/tests.sh b/docker/tests/tests.sh new file mode 100755 index 0000000..9239d72 --- /dev/null +++ b/docker/tests/tests.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +for dir in simple subdirs separatedirs ; do + pushd ${dir} + debos main.yaml + popd +done -- cgit v1.2.3