diff options
author | Emanuele Aina <emanuele.aina@collabora.com> | 2019-09-09 06:58:33 +0000 |
---|---|---|
committer | Emanuele Aina <emanuele.aina@collabora.com> | 2019-09-09 06:58:33 +0000 |
commit | 8c16f187783f193a7e5a87d9ef1281cd8ca92c1e (patch) | |
tree | efd3b745eec6410d08dc53dd7ecb43a7be5cdd32 | |
parent | 3f13e87bb45d0d791c86338f9aa0853a9f41a992 (diff) | |
parent | afdeba754f8766de3b12211953eced5304991cde (diff) |
Update upstream source from tag 'upstream/1.0.0+git20190906.f5be960'
Update to upstream version '1.0.0+git20190906.f5be960'
with Debian dir ff8d0024ed4cb787f364d24c9d56bc359f4d73e9
-rw-r--r-- | README.md | 9 | ||||
-rw-r--r-- | TODO | 2 | ||||
-rw-r--r-- | actions/apt_action.go | 1 | ||||
-rw-r--r-- | actions/image_partition_action.go | 32 | ||||
-rw-r--r-- | actions/ostree_deploy_action.go | 8 | ||||
-rw-r--r-- | actions/overlay_action.go | 2 | ||||
-rw-r--r-- | actions/pack_action.go | 6 | ||||
-rw-r--r-- | actions/recipe_action.go | 10 | ||||
-rw-r--r-- | actions/recipe_test.go | 22 | ||||
-rw-r--r-- | actions/run_action.go | 4 | ||||
-rw-r--r-- | archiver.go | 2 | ||||
-rw-r--r-- | commands.go | 4 | ||||
-rw-r--r-- | doc/examples/example.yaml | 4 | ||||
-rw-r--r-- | docker/Dockerfile | 80 | ||||
-rw-r--r-- | docker/README.md | 36 | ||||
-rw-r--r-- | docker/recipes.test.yml | 14 | ||||
-rw-r--r-- | docker/tests/A/A/A/A/a.txt | 1 | ||||
-rw-r--r-- | docker/tests/base.tar.gz | bin | 0 -> 289 bytes | |||
-rw-r--r-- | docker/tests/simple-recipe.yaml | 12 | ||||
-rw-r--r-- | docker/unit-tests.test.yml | 10 |
20 files changed, 220 insertions, 39 deletions
@@ -48,6 +48,15 @@ Some of the actions provided by debos to customize and produce images are: A full syntax description of all the debos actions can be found at: https://godoc.org/github.com/go-debos/debos/actions +## Installation (Docker container) + +Official debos container is available: +``` +docker pull godebos/debos +``` + +See [docker/README.md](https://github.com/go-debos/debos/blob/master/docker/README.md) for usage. + ## Installation (under Debian) sudo apt install golang git libglib2.0-dev libostree-dev qemu-system-x86 \ @@ -41,8 +41,6 @@ TODO * Make actions using (host) commands check their existance early -* Ensure we copy xattrs? - * Fix race in qemu-helper (if qemu-user-static gets installed in the system chroot things will get confused) diff --git a/actions/apt_action.go b/actions/apt_action.go index 6f3988f..9136687 100644 --- a/actions/apt_action.go +++ b/actions/apt_action.go @@ -18,6 +18,7 @@ Mandatory properties: Optional properties: - recommends -- boolean indicating if suggested packages will be installed + - unauthenticated -- boolean indicating if unauthenticated packages can be installed */ package actions diff --git a/actions/image_partition_action.go b/actions/image_partition_action.go index fcfd93e..b4f0cd2 100644 --- a/actions/image_partition_action.go +++ b/actions/image_partition_action.go @@ -42,6 +42,7 @@ Yaml syntax for partitions: start: offset end: offset flags: list of flags + fsck: bool Mandatory properties: @@ -64,6 +65,9 @@ Optional properties: - flags -- list of additional flags for partition compatible with parted(8) 'set' command. +- fsck -- if set to `false` -- then set fs_passno (man fstab) to 0 meaning no filesystem +checks in boot time. By default is set to `true` allowing checks on boot. + Yaml syntax for mount points: mountpoints: @@ -82,6 +86,7 @@ should be mounted. Optional properties: - options -- list of options to be added to appropriate entry in fstab file. + - buildtime -- if set to true then the mountpoint only used during the debos run. No entry in `/etc/fstab' will be created. The mountpoints directory will be removed from the image, so it is recommended @@ -139,6 +144,7 @@ type Partition struct { End string FS string Flags []string + Fsck bool "fsck" FSUUID string } @@ -163,6 +169,16 @@ type ImagePartitionAction struct { usingLoop bool } +func (p *Partition) UnmarshalYAML(unmarshal func(interface{}) error) error { + type rawPartition Partition + part := rawPartition{Fsck: true} + if err := unmarshal(&part); err != nil { + return err + } + *p = Partition(part) + return nil +} + func (i *ImagePartitionAction) generateFSTab(context *debos.DebosContext) error { context.ImageFSTab.Reset() @@ -176,9 +192,19 @@ func (i *ImagePartitionAction) generateFSTab(context *debos.DebosContext) error if m.part.FSUUID == "" { return fmt.Errorf("Missing fs UUID for partition %s!?!", m.part.Name) } - context.ImageFSTab.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0\t0\n", + + fs_passno := 0 + + if m.part.Fsck { + if m.Mountpoint == "/" { + fs_passno = 1 + } else { + fs_passno = 2 + } + } + context.ImageFSTab.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0\t%d\n", m.part.FSUUID, m.Mountpoint, m.part.FS, - strings.Join(options, ","))) + strings.Join(options, ","), fs_passno)) } return nil @@ -238,7 +264,7 @@ func (i ImagePartitionAction) formatPartition(p *Partition, context debos.DebosC cmdline := []string{} switch p.FS { case "vfat": - cmdline = append(cmdline, "mkfs.vfat", "-n", p.Name) + cmdline = append(cmdline, "mkfs.vfat", "-F32", "-n", p.Name) case "btrfs": // Force formatting to prevent failure in case if partition was formatted already cmdline = append(cmdline, "mkfs.btrfs", "-L", p.Name, "-f") diff --git a/actions/ostree_deploy_action.go b/actions/ostree_deploy_action.go index 7d6e6bd..7353691 100644 --- a/actions/ostree_deploy_action.go +++ b/actions/ostree_deploy_action.go @@ -55,7 +55,6 @@ import ( "io" "os" "path" - "runtime" "strings" "github.com/go-debos/debos" @@ -198,6 +197,11 @@ func (ot *OstreeDeployAction) Run(context *debos.DebosContext) error { return err } - runtime.GC() + /* libostree keeps some information, like repo lock file descriptor, in + * thread specific variables. As GC can be run from another thread, it + * may not been able to access this, preventing to free them correctly. + * To prevent this, explicitly dereference libostree objects. */ + dstRepo.Unref() + sysroot.Unref() return nil } diff --git a/actions/overlay_action.go b/actions/overlay_action.go index f17ecb3..b5e4924 100644 --- a/actions/overlay_action.go +++ b/actions/overlay_action.go @@ -20,7 +20,7 @@ Optional properties: - destination -- absolute path in the target rootfs where 'source' will be copied. All existing files will be overwritten. -If destination isn't set '/' of the rootfs will be usedi. +If destination isn't set '/' of the rootfs will be used. */ package actions diff --git a/actions/pack_action.go b/actions/pack_action.go index 1cb1af0..b8f28a7 100644 --- a/actions/pack_action.go +++ b/actions/pack_action.go @@ -34,6 +34,8 @@ func (pf *PackAction) Run(context *debos.DebosContext) error { pf.LogStart() outfile := path.Join(context.Artifactdir, pf.File) - log.Printf("Compression to %s\n", outfile) - return debos.Command{}.Run("Packing", "tar", "czf", outfile, "-C", context.Rootdir, ".") + log.Printf("Compressing to %s\n", outfile) + return debos.Command{}.Run("Packing", "tar", "czf", outfile, + "--xattrs", "--xattrs-include=*.*", + "-C", context.Rootdir, ".") } diff --git a/actions/recipe_action.go b/actions/recipe_action.go index d9500dd..9fa3b1c 100644 --- a/actions/recipe_action.go +++ b/actions/recipe_action.go @@ -1,7 +1,15 @@ /* Recipe Action -Include a recipe. +This action includes the recipe at the given path, and can optionally +override or set template variables. + +To ensure compatibility, both the parent recipe and all included recipes have +to be for the same architecture. For convenience the parent architecture is +passed in the "architecture" template variable. + +Limitations of combined recipes are equivalent to limitations within a +single recipe (e.g. there can only be one image partition action). Yaml syntax: - action: recipe diff --git a/actions/recipe_test.go b/actions/recipe_test.go index de9d6ea..ef2a755 100644 --- a/actions/recipe_test.go +++ b/actions/recipe_test.go @@ -228,16 +228,6 @@ actions: command: ok.sh `, } - var recipeIncluded = subRecipe { - "included.yaml", - ` -architecture: amd64 - -actions: - - action: run - command: ok.sh -`, - } // test recipes var tests = []testSubRecipe { @@ -289,18 +279,6 @@ actions: recipeArmhf, "Expect architecture 'amd64' but got 'armhf'", }, - { - // Test included_recipe prevents parsing OK - ` -architecture: amd64 - -actions: - - action: recipe - recipe: included.yaml -`, - recipeIncluded, - "", // Do not expect failure - }, } for _, test := range tests { diff --git a/actions/run_action.go b/actions/run_action.go index 7bae989..8d36228 100644 --- a/actions/run_action.go +++ b/actions/run_action.go @@ -98,8 +98,8 @@ func (run *RunAction) doRun(context debos.DebosContext) error { script[0] = debos.CleanPathAt(script[0], context.RecipeDir) if run.Chroot { scriptpath := path.Dir(script[0]) - cmd.AddBindMount(scriptpath, "/script") - script[0] = strings.Replace(script[0], scriptpath, "/script", 1) + cmd.AddBindMount(scriptpath, "/tmp/script") + script[0] = strings.Replace(script[0], scriptpath, "/tmp/script", 1) } cmdline = []string{strings.Join(script, " ")} label = path.Base(run.Script) diff --git a/archiver.go b/archiver.go index e67fdf6..c54f89c 100644 --- a/archiver.go +++ b/archiver.go @@ -99,6 +99,8 @@ func (tar *ArchiveTar) Unpack(destination string) error { } command = append(command, "-C", destination) command = append(command, "-x") + command = append(command, "--xattrs") + command = append(command, "--xattrs-include=*.*") if compression, ok := tar.options["tarcompression"]; ok { if unpackTarOpt := tarOptions(compression.(string)); len(unpackTarOpt) > 0 { diff --git a/commands.go b/commands.go index aa562e3..0ea2b9a 100644 --- a/commands.go +++ b/commands.go @@ -13,8 +13,8 @@ import ( type ChrootEnterMethod int const ( - CHROOT_METHOD_NONE = iota // use nspawn to create the chroot environment - CHROOT_METHOD_NSPAWN // No chroot in use + CHROOT_METHOD_NONE = iota // No chroot in use + CHROOT_METHOD_NSPAWN // use nspawn to create the chroot environment CHROOT_METHOD_CHROOT // use chroot to create the chroot environment ) diff --git a/doc/examples/example.yaml b/doc/examples/example.yaml index c0b6f55..9375323 100644 --- a/doc/examples/example.yaml +++ b/doc/examples/example.yaml @@ -1,12 +1,12 @@ {{- $architecture := or .architecture "arm64" -}} -{{- $suite := or .suite "stretch" -}} +{{- $suite := or .suite "buster" -}} {{ $image := or .image (printf "debian-%s-%s.tgz" $suite $architecture) }} architecture: {{ $architecture }} actions: - action: debootstrap - suite: "buster" + suite: {{ $suite }} components: - main - contrib diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..16059d9 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,80 @@ +# Global ARGs shared by all stages +ARG DEBIAN_FRONTEND=noninteractive +ARG GOPATH=/usr/local/go + +### first stage - builder ### +FROM debian:buster-slim as builder + +ARG DEBIAN_FRONTEND +ARG GOPATH +ENV GOPATH=${GOPATH} + +# install debos build dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + gcc \ + git \ + golang-go \ + libc6-dev \ + libostree-dev && \ + rm -rf /var/lib/apt/lists/* + +# Build debos +COPY . $GOPATH/src/github.com/go-debos/debos +WORKDIR $GOPATH/src/github.com/go-debos/debos/cmd/debos +RUN go get -d ./... && \ + go get -d github.com/stretchr/testify && \ + go install + +### second stage - runner ### +FROM debian:buster-slim as runner + +ARG DEBIAN_FRONTEND +ARG GOPATH + +# Set HOME to a writable directory in case something wants to cache things +ENV HOME=/tmp + +LABEL org.label-schema.name "debos" +LABEL org.label-schema.description "Debian OS builder" +LABEL org.label-schema.vcs-url = "https://github.com/go-debos/debos" +LABEL org.label-schema.docker.cmd 'docker run \ + --rm \ + --interactive \ + --tty \ + --device /dev/kvm \ + --user $(id -u) \ + --workdir /recipes \ + --mount "type=bind,source=$(pwd),destination=/recipes" \ + --security-opt label=disable' + +# debos runtime dependencies +# ca-certificates is required to validate HTTPS certificates when getting debootstrap release file +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + apt-transport-https \ + binfmt-support \ + bmap-tools \ + btrfs-progs \ + busybox \ + bzip2 \ + ca-certificates \ + debootstrap \ + dosfstools \ + e2fsprogs \ + gzip \ + libostree-1-1 \ + linux-image-amd64 \ + parted \ + pkg-config \ + qemu-system-x86 \ + qemu-user-static \ + systemd \ + systemd-container \ + xz-utils && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=builder $GOPATH/bin/debos /usr/local/bin/debos + +ENTRYPOINT ["/usr/local/bin/debos"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..7354617 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,36 @@ +# debos + +Docker container for ['debos' tool](https://github.com/go-debos/debos). + +## Installation +``` +docker pull godebos/debos +``` + +Debos needs virtualization to be enabled on the host and shared with the container. + +Check that `kvm` is enabled and writable by the user running the docker container by running ```ls /dev/kvm``` + +## Usage +/!\ This container should be used as an executable, i.e. there is no need to add `debos` after `godebos/debos`. + +To build `recipe.yaml`: +``` +cd <PATH_TO_RECIPE_DIR> +docker run --rm --interactive --tty --device /dev/kvm --user $(id -u) --workdir /recipes --mount "type=bind,source=$(pwd),destination=/recipes" --security-opt label=disable godebos/debos <RECIPE.yaml> +``` + +## Container build +To build the debos container image from current git branch: +``` +docker build -f docker/Dockerfile -t godebos/debos . +``` + +## Tests + +### unit tests +Run unit test with debos-docker: +``` +cd docker +docker-compose -f unit-tests.test.yml up --build +``` diff --git a/docker/recipes.test.yml b/docker/recipes.test.yml new file mode 100644 index 0000000..18d4aca --- /dev/null +++ b/docker/recipes.test.yml @@ -0,0 +1,14 @@ +version: '3.4' + +services: + sut: + build: + context: .. + dockerfile: docker/Dockerfile + target: runner + volumes: + - type: bind + source: ./tests + target: /recipes + working_dir: /recipes + command: simple-recipe.yaml diff --git a/docker/tests/A/A/A/A/a.txt b/docker/tests/A/A/A/A/a.txt new file mode 100644 index 0000000..7898192 --- /dev/null +++ b/docker/tests/A/A/A/A/a.txt @@ -0,0 +1 @@ +a diff --git a/docker/tests/base.tar.gz b/docker/tests/base.tar.gz Binary files differnew file mode 100644 index 0000000..cf787ff --- /dev/null +++ b/docker/tests/base.tar.gz diff --git a/docker/tests/simple-recipe.yaml b/docker/tests/simple-recipe.yaml new file mode 100644 index 0000000..b4f99f2 --- /dev/null +++ b/docker/tests/simple-recipe.yaml @@ -0,0 +1,12 @@ +architecture: amd64 + +actions: + - action: unpack + compression: gz + file: base.tar.gz + + - action: overlay + source: A + + - action: run + command: find ${ROOTDIR} diff --git a/docker/unit-tests.test.yml b/docker/unit-tests.test.yml new file mode 100644 index 0000000..b5c3e3f --- /dev/null +++ b/docker/unit-tests.test.yml @@ -0,0 +1,10 @@ +version: '3.4' + +services: + sut: + build: + context: .. + dockerfile: docker/Dockerfile + target: builder + working_dir: /usr/local/go/src/github.com/go-debos/debos/actions + command: go test |