summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorEmanuele Aina <emanuele.aina@collabora.com>2019-09-09 06:58:33 +0000
committerEmanuele Aina <emanuele.aina@collabora.com>2019-09-09 06:58:33 +0000
commit8c16f187783f193a7e5a87d9ef1281cd8ca92c1e (patch)
treeefd3b745eec6410d08dc53dd7ecb43a7be5cdd32
parent3f13e87bb45d0d791c86338f9aa0853a9f41a992 (diff)
parentafdeba754f8766de3b12211953eced5304991cde (diff)
Update upstream source from tag 'upstream/1.0.0+git20190906.f5be960'
Update to upstream version '1.0.0+git20190906.f5be960' with Debian dir ff8d0024ed4cb787f364d24c9d56bc359f4d73e9
-rw-r--r--README.md9
-rw-r--r--TODO2
-rw-r--r--actions/apt_action.go1
-rw-r--r--actions/image_partition_action.go32
-rw-r--r--actions/ostree_deploy_action.go8
-rw-r--r--actions/overlay_action.go2
-rw-r--r--actions/pack_action.go6
-rw-r--r--actions/recipe_action.go10
-rw-r--r--actions/recipe_test.go22
-rw-r--r--actions/run_action.go4
-rw-r--r--archiver.go2
-rw-r--r--commands.go4
-rw-r--r--doc/examples/example.yaml4
-rw-r--r--docker/Dockerfile80
-rw-r--r--docker/README.md36
-rw-r--r--docker/recipes.test.yml14
-rw-r--r--docker/tests/A/A/A/A/a.txt1
-rw-r--r--docker/tests/base.tar.gzbin0 -> 289 bytes
-rw-r--r--docker/tests/simple-recipe.yaml12
-rw-r--r--docker/unit-tests.test.yml10
20 files changed, 220 insertions, 39 deletions
diff --git a/README.md b/README.md
index 03b6d54..eee1e3c 100644
--- a/README.md
+++ b/README.md
@@ -48,6 +48,15 @@ Some of the actions provided by debos to customize and produce images are:
A full syntax description of all the debos actions can be found at:
https://godoc.org/github.com/go-debos/debos/actions
+## Installation (Docker container)
+
+Official debos container is available:
+```
+docker pull godebos/debos
+```
+
+See [docker/README.md](https://github.com/go-debos/debos/blob/master/docker/README.md) for usage.
+
## Installation (under Debian)
sudo apt install golang git libglib2.0-dev libostree-dev qemu-system-x86 \
diff --git a/TODO b/TODO
index 31b1519..21ca64e 100644
--- a/TODO
+++ b/TODO
@@ -41,8 +41,6 @@ TODO
* Make actions using (host) commands check their existance early
-* Ensure we copy xattrs?
-
* Fix race in qemu-helper (if qemu-user-static gets installed in the system
chroot things will get confused)
diff --git a/actions/apt_action.go b/actions/apt_action.go
index 6f3988f..9136687 100644
--- a/actions/apt_action.go
+++ b/actions/apt_action.go
@@ -18,6 +18,7 @@ Mandatory properties:
Optional properties:
- recommends -- boolean indicating if suggested packages will be installed
+
- unauthenticated -- boolean indicating if unauthenticated packages can be installed
*/
package actions
diff --git a/actions/image_partition_action.go b/actions/image_partition_action.go
index fcfd93e..b4f0cd2 100644
--- a/actions/image_partition_action.go
+++ b/actions/image_partition_action.go
@@ -42,6 +42,7 @@ Yaml syntax for partitions:
start: offset
end: offset
flags: list of flags
+ fsck: bool
Mandatory properties:
@@ -64,6 +65,9 @@ Optional properties:
- flags -- list of additional flags for partition compatible with parted(8)
'set' command.
+- fsck -- if set to `false` -- then set fs_passno (man fstab) to 0 meaning no filesystem
+checks in boot time. By default is set to `true` allowing checks on boot.
+
Yaml syntax for mount points:
mountpoints:
@@ -82,6 +86,7 @@ should be mounted.
Optional properties:
- options -- list of options to be added to appropriate entry in fstab file.
+
- buildtime -- if set to true then the mountpoint only used during the debos run.
No entry in `/etc/fstab' will be created.
The mountpoints directory will be removed from the image, so it is recommended
@@ -139,6 +144,7 @@ type Partition struct {
End string
FS string
Flags []string
+ Fsck bool "fsck"
FSUUID string
}
@@ -163,6 +169,16 @@ type ImagePartitionAction struct {
usingLoop bool
}
+func (p *Partition) UnmarshalYAML(unmarshal func(interface{}) error) error {
+ type rawPartition Partition
+ part := rawPartition{Fsck: true}
+ if err := unmarshal(&part); err != nil {
+ return err
+ }
+ *p = Partition(part)
+ return nil
+}
+
func (i *ImagePartitionAction) generateFSTab(context *debos.DebosContext) error {
context.ImageFSTab.Reset()
@@ -176,9 +192,19 @@ func (i *ImagePartitionAction) generateFSTab(context *debos.DebosContext) error
if m.part.FSUUID == "" {
return fmt.Errorf("Missing fs UUID for partition %s!?!", m.part.Name)
}
- context.ImageFSTab.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0\t0\n",
+
+ fs_passno := 0
+
+ if m.part.Fsck {
+ if m.Mountpoint == "/" {
+ fs_passno = 1
+ } else {
+ fs_passno = 2
+ }
+ }
+ context.ImageFSTab.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0\t%d\n",
m.part.FSUUID, m.Mountpoint, m.part.FS,
- strings.Join(options, ",")))
+ strings.Join(options, ","), fs_passno))
}
return nil
@@ -238,7 +264,7 @@ func (i ImagePartitionAction) formatPartition(p *Partition, context debos.DebosC
cmdline := []string{}
switch p.FS {
case "vfat":
- cmdline = append(cmdline, "mkfs.vfat", "-n", p.Name)
+ cmdline = append(cmdline, "mkfs.vfat", "-F32", "-n", p.Name)
case "btrfs":
// Force formatting to prevent failure in case if partition was formatted already
cmdline = append(cmdline, "mkfs.btrfs", "-L", p.Name, "-f")
diff --git a/actions/ostree_deploy_action.go b/actions/ostree_deploy_action.go
index 7d6e6bd..7353691 100644
--- a/actions/ostree_deploy_action.go
+++ b/actions/ostree_deploy_action.go
@@ -55,7 +55,6 @@ import (
"io"
"os"
"path"
- "runtime"
"strings"
"github.com/go-debos/debos"
@@ -198,6 +197,11 @@ func (ot *OstreeDeployAction) Run(context *debos.DebosContext) error {
return err
}
- runtime.GC()
+ /* libostree keeps some information, like repo lock file descriptor, in
+ * thread specific variables. As GC can be run from another thread, it
+ * may not been able to access this, preventing to free them correctly.
+ * To prevent this, explicitly dereference libostree objects. */
+ dstRepo.Unref()
+ sysroot.Unref()
return nil
}
diff --git a/actions/overlay_action.go b/actions/overlay_action.go
index f17ecb3..b5e4924 100644
--- a/actions/overlay_action.go
+++ b/actions/overlay_action.go
@@ -20,7 +20,7 @@ Optional properties:
- destination -- absolute path in the target rootfs where 'source' will be copied.
All existing files will be overwritten.
-If destination isn't set '/' of the rootfs will be usedi.
+If destination isn't set '/' of the rootfs will be used.
*/
package actions
diff --git a/actions/pack_action.go b/actions/pack_action.go
index 1cb1af0..b8f28a7 100644
--- a/actions/pack_action.go
+++ b/actions/pack_action.go
@@ -34,6 +34,8 @@ func (pf *PackAction) Run(context *debos.DebosContext) error {
pf.LogStart()
outfile := path.Join(context.Artifactdir, pf.File)
- log.Printf("Compression to %s\n", outfile)
- return debos.Command{}.Run("Packing", "tar", "czf", outfile, "-C", context.Rootdir, ".")
+ log.Printf("Compressing to %s\n", outfile)
+ return debos.Command{}.Run("Packing", "tar", "czf", outfile,
+ "--xattrs", "--xattrs-include=*.*",
+ "-C", context.Rootdir, ".")
}
diff --git a/actions/recipe_action.go b/actions/recipe_action.go
index d9500dd..9fa3b1c 100644
--- a/actions/recipe_action.go
+++ b/actions/recipe_action.go
@@ -1,7 +1,15 @@
/*
Recipe Action
-Include a recipe.
+This action includes the recipe at the given path, and can optionally
+override or set template variables.
+
+To ensure compatibility, both the parent recipe and all included recipes have
+to be for the same architecture. For convenience the parent architecture is
+passed in the "architecture" template variable.
+
+Limitations of combined recipes are equivalent to limitations within a
+single recipe (e.g. there can only be one image partition action).
Yaml syntax:
- action: recipe
diff --git a/actions/recipe_test.go b/actions/recipe_test.go
index de9d6ea..ef2a755 100644
--- a/actions/recipe_test.go
+++ b/actions/recipe_test.go
@@ -228,16 +228,6 @@ actions:
command: ok.sh
`,
}
- var recipeIncluded = subRecipe {
- "included.yaml",
- `
-architecture: amd64
-
-actions:
- - action: run
- command: ok.sh
-`,
- }
// test recipes
var tests = []testSubRecipe {
@@ -289,18 +279,6 @@ actions:
recipeArmhf,
"Expect architecture 'amd64' but got 'armhf'",
},
- {
- // Test included_recipe prevents parsing OK
- `
-architecture: amd64
-
-actions:
- - action: recipe
- recipe: included.yaml
-`,
- recipeIncluded,
- "", // Do not expect failure
- },
}
for _, test := range tests {
diff --git a/actions/run_action.go b/actions/run_action.go
index 7bae989..8d36228 100644
--- a/actions/run_action.go
+++ b/actions/run_action.go
@@ -98,8 +98,8 @@ func (run *RunAction) doRun(context debos.DebosContext) error {
script[0] = debos.CleanPathAt(script[0], context.RecipeDir)
if run.Chroot {
scriptpath := path.Dir(script[0])
- cmd.AddBindMount(scriptpath, "/script")
- script[0] = strings.Replace(script[0], scriptpath, "/script", 1)
+ cmd.AddBindMount(scriptpath, "/tmp/script")
+ script[0] = strings.Replace(script[0], scriptpath, "/tmp/script", 1)
}
cmdline = []string{strings.Join(script, " ")}
label = path.Base(run.Script)
diff --git a/archiver.go b/archiver.go
index e67fdf6..c54f89c 100644
--- a/archiver.go
+++ b/archiver.go
@@ -99,6 +99,8 @@ func (tar *ArchiveTar) Unpack(destination string) error {
}
command = append(command, "-C", destination)
command = append(command, "-x")
+ command = append(command, "--xattrs")
+ command = append(command, "--xattrs-include=*.*")
if compression, ok := tar.options["tarcompression"]; ok {
if unpackTarOpt := tarOptions(compression.(string)); len(unpackTarOpt) > 0 {
diff --git a/commands.go b/commands.go
index aa562e3..0ea2b9a 100644
--- a/commands.go
+++ b/commands.go
@@ -13,8 +13,8 @@ import (
type ChrootEnterMethod int
const (
- CHROOT_METHOD_NONE = iota // use nspawn to create the chroot environment
- CHROOT_METHOD_NSPAWN // No chroot in use
+ CHROOT_METHOD_NONE = iota // No chroot in use
+ CHROOT_METHOD_NSPAWN // use nspawn to create the chroot environment
CHROOT_METHOD_CHROOT // use chroot to create the chroot environment
)
diff --git a/doc/examples/example.yaml b/doc/examples/example.yaml
index c0b6f55..9375323 100644
--- a/doc/examples/example.yaml
+++ b/doc/examples/example.yaml
@@ -1,12 +1,12 @@
{{- $architecture := or .architecture "arm64" -}}
-{{- $suite := or .suite "stretch" -}}
+{{- $suite := or .suite "buster" -}}
{{ $image := or .image (printf "debian-%s-%s.tgz" $suite $architecture) }}
architecture: {{ $architecture }}
actions:
- action: debootstrap
- suite: "buster"
+ suite: {{ $suite }}
components:
- main
- contrib
diff --git a/docker/Dockerfile b/docker/Dockerfile
new file mode 100644
index 0000000..16059d9
--- /dev/null
+++ b/docker/Dockerfile
@@ -0,0 +1,80 @@
+# Global ARGs shared by all stages
+ARG DEBIAN_FRONTEND=noninteractive
+ARG GOPATH=/usr/local/go
+
+### first stage - builder ###
+FROM debian:buster-slim as builder
+
+ARG DEBIAN_FRONTEND
+ARG GOPATH
+ENV GOPATH=${GOPATH}
+
+# install debos build dependencies
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ ca-certificates \
+ gcc \
+ git \
+ golang-go \
+ libc6-dev \
+ libostree-dev && \
+ rm -rf /var/lib/apt/lists/*
+
+# Build debos
+COPY . $GOPATH/src/github.com/go-debos/debos
+WORKDIR $GOPATH/src/github.com/go-debos/debos/cmd/debos
+RUN go get -d ./... && \
+ go get -d github.com/stretchr/testify && \
+ go install
+
+### second stage - runner ###
+FROM debian:buster-slim as runner
+
+ARG DEBIAN_FRONTEND
+ARG GOPATH
+
+# Set HOME to a writable directory in case something wants to cache things
+ENV HOME=/tmp
+
+LABEL org.label-schema.name "debos"
+LABEL org.label-schema.description "Debian OS builder"
+LABEL org.label-schema.vcs-url = "https://github.com/go-debos/debos"
+LABEL org.label-schema.docker.cmd 'docker run \
+ --rm \
+ --interactive \
+ --tty \
+ --device /dev/kvm \
+ --user $(id -u) \
+ --workdir /recipes \
+ --mount "type=bind,source=$(pwd),destination=/recipes" \
+ --security-opt label=disable'
+
+# debos runtime dependencies
+# ca-certificates is required to validate HTTPS certificates when getting debootstrap release file
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ apt-transport-https \
+ binfmt-support \
+ bmap-tools \
+ btrfs-progs \
+ busybox \
+ bzip2 \
+ ca-certificates \
+ debootstrap \
+ dosfstools \
+ e2fsprogs \
+ gzip \
+ libostree-1-1 \
+ linux-image-amd64 \
+ parted \
+ pkg-config \
+ qemu-system-x86 \
+ qemu-user-static \
+ systemd \
+ systemd-container \
+ xz-utils && \
+ rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder $GOPATH/bin/debos /usr/local/bin/debos
+
+ENTRYPOINT ["/usr/local/bin/debos"]
diff --git a/docker/README.md b/docker/README.md
new file mode 100644
index 0000000..7354617
--- /dev/null
+++ b/docker/README.md
@@ -0,0 +1,36 @@
+# debos
+
+Docker container for ['debos' tool](https://github.com/go-debos/debos).
+
+## Installation
+```
+docker pull godebos/debos
+```
+
+Debos needs virtualization to be enabled on the host and shared with the container.
+
+Check that `kvm` is enabled and writable by the user running the docker container by running ```ls /dev/kvm```
+
+## Usage
+/!\ This container should be used as an executable, i.e. there is no need to add `debos` after `godebos/debos`.
+
+To build `recipe.yaml`:
+```
+cd <PATH_TO_RECIPE_DIR>
+docker run --rm --interactive --tty --device /dev/kvm --user $(id -u) --workdir /recipes --mount "type=bind,source=$(pwd),destination=/recipes" --security-opt label=disable godebos/debos <RECIPE.yaml>
+```
+
+## Container build
+To build the debos container image from current git branch:
+```
+docker build -f docker/Dockerfile -t godebos/debos .
+```
+
+## Tests
+
+### unit tests
+Run unit test with debos-docker:
+```
+cd docker
+docker-compose -f unit-tests.test.yml up --build
+```
diff --git a/docker/recipes.test.yml b/docker/recipes.test.yml
new file mode 100644
index 0000000..18d4aca
--- /dev/null
+++ b/docker/recipes.test.yml
@@ -0,0 +1,14 @@
+version: '3.4'
+
+services:
+ sut:
+ build:
+ context: ..
+ dockerfile: docker/Dockerfile
+ target: runner
+ volumes:
+ - type: bind
+ source: ./tests
+ target: /recipes
+ working_dir: /recipes
+ command: simple-recipe.yaml
diff --git a/docker/tests/A/A/A/A/a.txt b/docker/tests/A/A/A/A/a.txt
new file mode 100644
index 0000000..7898192
--- /dev/null
+++ b/docker/tests/A/A/A/A/a.txt
@@ -0,0 +1 @@
+a
diff --git a/docker/tests/base.tar.gz b/docker/tests/base.tar.gz
new file mode 100644
index 0000000..cf787ff
--- /dev/null
+++ b/docker/tests/base.tar.gz
Binary files differ
diff --git a/docker/tests/simple-recipe.yaml b/docker/tests/simple-recipe.yaml
new file mode 100644
index 0000000..b4f99f2
--- /dev/null
+++ b/docker/tests/simple-recipe.yaml
@@ -0,0 +1,12 @@
+architecture: amd64
+
+actions:
+ - action: unpack
+ compression: gz
+ file: base.tar.gz
+
+ - action: overlay
+ source: A
+
+ - action: run
+ command: find ${ROOTDIR}
diff --git a/docker/unit-tests.test.yml b/docker/unit-tests.test.yml
new file mode 100644
index 0000000..b5c3e3f
--- /dev/null
+++ b/docker/unit-tests.test.yml
@@ -0,0 +1,10 @@
+version: '3.4'
+
+services:
+ sut:
+ build:
+ context: ..
+ dockerfile: docker/Dockerfile
+ target: builder
+ working_dir: /usr/local/go/src/github.com/go-debos/debos/actions
+ command: go test