From 9c6de646795022f856ed7c22d28e2d63e4c0ca22 Mon Sep 17 00:00:00 2001 From: Andrej Shadura Date: Wed, 14 Oct 2020 22:18:30 +0200 Subject: New upstream version 1.0.0+git20200909.d2ab9d3 --- actions/debootstrap_action.go | 68 ++++++++++++++++- actions/filesystem_deploy_action.go | 9 ++- actions/image_partition_action.go | 142 +++++++++++++++++++++++++++++++----- actions/run_action.go | 16 +++- commands.go | 5 +- docker/Dockerfile | 13 +++- 6 files changed, 222 insertions(+), 31 deletions(-) diff --git a/actions/debootstrap_action.go b/actions/debootstrap_action.go index 77ee63d..506270c 100644 --- a/actions/debootstrap_action.go +++ b/actions/debootstrap_action.go @@ -15,6 +15,8 @@ Yaml syntax: variant: "name" keyring-package: keyring-file: + certificate: + private-key: Mandatory properties: @@ -40,6 +42,10 @@ Example: - keyring-file -- keyring file for repository validation. - merged-usr -- use merged '/usr' filesystem, true by default. + +- certificate -- client certificate stored in file to be used for downloading packages from the server. + +- private-key -- provide the client's private key in a file separate from the certificate. */ package actions @@ -51,6 +57,7 @@ import ( "strings" "github.com/go-debos/debos" + "github.com/go-debos/fakemachine" ) type DebootstrapAction struct { @@ -60,6 +67,8 @@ type DebootstrapAction struct { Variant string KeyringPackage string `yaml:"keyring-package"` KeyringFile string `yaml:"keyring-file"` + Certificate string + PrivateKey string `yaml:"private-key"` Components []string MergedUsr bool `yaml:"merged-usr"` CheckGpg bool `yaml:"check-gpg"` @@ -72,13 +81,57 @@ func NewDebootstrapAction() *DebootstrapAction { // Be secure by default d.CheckGpg = true // Use main as default component - d.Components = []string {"main"} + d.Components = []string{"main"} // Set generic default mirror d.Mirror = "http://deb.debian.org/debian" return &d } +func (d *DebootstrapAction) listOptionFiles(context *debos.DebosContext) []string { + files := []string{} + if d.Certificate != "" { + d.Certificate = debos.CleanPathAt(d.Certificate, context.RecipeDir) + files = append(files, d.Certificate) + } + + if d.PrivateKey != "" { + d.PrivateKey = debos.CleanPathAt(d.PrivateKey, context.RecipeDir) + files = append(files, d.PrivateKey) + } + + if d.KeyringFile != "" { + d.KeyringFile = debos.CleanPathAt(d.KeyringFile, context.RecipeDir) + files = append(files, d.KeyringFile) + } + + return files +} + +func (d *DebootstrapAction) Verify(context *debos.DebosContext) error { + files := d.listOptionFiles(context) + + // Check if all needed files exists + for _, f := range files { + if _, err := os.Stat(f); os.IsNotExist(err) { + return err + } + } + return nil +} + +func (d *DebootstrapAction) PreMachine(context *debos.DebosContext, m *fakemachine.Machine, args *[]string) error { + + mounts := d.listOptionFiles(context) + + // Mount configuration files outside of recipes directory + for _, mount := range mounts { + m.AddVolume(path.Dir(mount)) + } + + return nil +} + func (d *DebootstrapAction) RunSecondStage(context debos.DebosContext) error { cmdline := []string{ "/debootstrap/debootstrap", @@ -96,7 +149,7 @@ func (d *DebootstrapAction) RunSecondStage(context debos.DebosContext) error { err := c.Run("Debootstrap (stage 2)", cmdline...) - if (err != nil) { + if err != nil { log := path.Join(context.Rootdir, "debootstrap/debootstrap.log") _ = debos.Command{}.Run("debootstrap.log", "cat", log) } @@ -117,14 +170,21 @@ func (d *DebootstrapAction) Run(context *debos.DebosContext) error { if !d.CheckGpg { cmdline = append(cmdline, fmt.Sprintf("--no-check-gpg")) } else if d.KeyringFile != "" { - path := debos.CleanPathAt(d.KeyringFile, context.RecipeDir) - cmdline = append(cmdline, fmt.Sprintf("--keyring=%s", path)) + cmdline = append(cmdline, fmt.Sprintf("--keyring=%s", d.KeyringFile)) } if d.KeyringPackage != "" { cmdline = append(cmdline, fmt.Sprintf("--include=%s", d.KeyringPackage)) } + if d.Certificate != "" { + cmdline = append(cmdline, fmt.Sprintf("--certificate=%s", d.Certificate)) + } + + if d.PrivateKey != "" { + cmdline = append(cmdline, fmt.Sprintf("--private-key=%s", d.PrivateKey)) + } + if d.Components != nil { s := strings.Join(d.Components, ",") cmdline = append(cmdline, fmt.Sprintf("--components=%s", s)) diff --git a/actions/filesystem_deploy_action.go b/actions/filesystem_deploy_action.go index af087f1..1519904 100644 --- a/actions/filesystem_deploy_action.go +++ b/actions/filesystem_deploy_action.go @@ -1,8 +1,13 @@ /* FilesystemDeploy Action -Deploy prepared root filesystem to output image. This action requires -'image-partition' action to be executed before it. +Deploy prepared root filesystem to output image by copying the files from the +temporary scratch directory to the mounted image and optionally creates various +configuration files for the image: '/etc/fstab' and '/etc/kernel/cmdline'. This +action requires 'image-partition' action to be executed before it. + +After this action has ran, subsequent actions are executed on the mounted output +image. Yaml syntax: - action: filesystem-deploy diff --git a/actions/image_partition_action.go b/actions/image_partition_action.go index 0850990..9b9cddd 100644 --- a/actions/image_partition_action.go +++ b/actions/image_partition_action.go @@ -2,6 +2,10 @@ ImagePartition Action This action creates an image file, partitions it and formats the filesystems. +Mountpoints can be defined so the created partitions can be mounted during the +build, and optionally (but by-default) mounted at boot in the final system. The +mountpoints are sorted on their position in the filesystem hierarchy so the +order in the recipe does not matter. Yaml syntax: - action: image-partition @@ -36,8 +40,8 @@ Properties for mount points are described below. Yaml syntax for partitions: partitions: - - name: label - name: partition name + - name: partition name + partlabel: partition label fs: filesystem start: offset end: offset @@ -48,7 +52,8 @@ Yaml syntax for partitions: Mandatory properties: - name -- is used for referencing named partition for mount points -configuration (below) and label the filesystem located on this partition. +configuration (below) and label the filesystem located on this partition. Must be +unique. - fs -- filesystem type used for formatting. @@ -63,6 +68,17 @@ form -- '32MB', '1GB' or as disk percentage -- '100%'. Optional properties: +- partlabel -- label for the partition in the GPT partition table. Defaults +to the `name` property of the partition. May only be used for GPT partitions. + +- parttype -- set the partition type in the partition table. The string should +be in a hexadecimal format (2-characters) for msdos partition tables and GUID format +(36-characters) for GPT partition tables. For instance, "82" for msdos sets the +partition type to Linux Swap. Whereas "0657fd6d-a4ab-43c4-84e5-0933c84b4f4f" for +GPT sets the partition type to Linux Swap. +For msdos partition types hex codes see: https://en.wikipedia.org/wiki/Partition_type +For gpt partition type GUIDs see: https://systemd.io/DISCOVERABLE_PARTITIONS/ + - features -- list of additional filesystem features which need to be enabled for partition. @@ -82,17 +98,18 @@ Yaml syntax for mount points: Mandatory properties: -- partition -- partition name for mounting. +- partition -- partition name for mounting. The partion must exist under `partitions`. - mountpoint -- path in the target root filesystem where the named partition -should be mounted. +should be mounted. Must be unique, only one partition can be mounted per +mountpoint. Optional properties: - options -- list of options to be added to appropriate entry in fstab file. - buildtime -- if set to true then the mountpoint only used during the debos run. -No entry in `/etc/fstab' will be created. +No entry in `/etc/fstab` will be created. The mountpoints directory will be removed from the image, so it is recommended to define a `mountpoint` path which is temporary and unique for the image, for example: `/mnt/temporary_mount`. @@ -134,6 +151,7 @@ import ( "os/exec" "path" "path/filepath" + "sort" "strings" "syscall" "time" @@ -142,15 +160,17 @@ import ( ) type Partition struct { - number int - Name string - Start string - End string - FS string - Flags []string - Features []string - Fsck bool "fsck" - FSUUID string + number int + Name string + PartLabel string + PartType string + Start string + End string + FS string + Flags []string + Features []string + Fsck bool "fsck" + FSUUID string } type Mountpoint struct { @@ -250,6 +270,16 @@ func (i ImagePartitionAction) getPartitionDevice(number int, context debos.Debos } } +func (i *ImagePartitionAction) triggerDeviceNodes(context *debos.DebosContext) error { + err := debos.Command{}.Run("udevadm", "udevadm", "trigger", "--settle", context.Image) + if err != nil { + log.Printf("Failed to trigger device nodes") + return err + } + + return nil +} + func (i ImagePartitionAction) PreMachine(context *debos.DebosContext, m *fakemachine.Machine, args *[]string) error { image, err := m.CreateImage(i.ImageName, i.size) @@ -276,6 +306,11 @@ func (i ImagePartitionAction) formatPartition(p *Partition, context debos.DebosC if len(p.Features) > 0 { cmdline = append(cmdline, "-O", strings.Join(p.Features, ",")) } + case "f2fs": + cmdline = append(cmdline, "mkfs.f2fs", "-l", p.Name) + if len(p.Features) > 0 { + cmdline = append(cmdline, "-O", strings.Join(p.Features, ",")) + } case "hfs": cmdline = append(cmdline, "mkfs.hfs", "-h", "-v", p.Name) case "hfsplus": @@ -348,7 +383,9 @@ func (i ImagePartitionAction) Run(context *debos.DebosContext) error { } /* Defer will keep the fd open until the function returns, at which points * the filesystems will have been mounted protecting from more udev funnyness - */ + * After the fd is closed the kernel needs to be informed of partition table + * changes (defer calls are executed in LIFO order) */ + defer i.triggerDeviceNodes(context) defer imageFD.Close() err = syscall.Flock(int(imageFD.Fd()), syscall.LOCK_EX) @@ -366,9 +403,14 @@ func (i ImagePartitionAction) Run(context *debos.DebosContext) error { } for idx, _ := range i.Partitions { p := &i.Partitions[idx] + + if p.PartLabel == "" { + p.PartLabel = p.Name + } + var name string if i.PartitionType == "gpt" { - name = p.Name + name = p.PartLabel } else { name = "primary" } @@ -379,6 +421,7 @@ func (i ImagePartitionAction) Run(context *debos.DebosContext) error { command = append(command, "fat32") case "hfsplus": command = append(command, "hfs+") + case "f2fs": case "none": default: command = append(command, p.FS) @@ -400,6 +443,14 @@ func (i ImagePartitionAction) Run(context *debos.DebosContext) error { } } + if p.PartType != "" { + err = debos.Command{}.Run("sfdisk", "sfdisk", "--part-type", context.Image, fmt.Sprintf("%d", p.number), p.PartType) + if err != nil { + return err + } + } + + devicePath := i.getPartitionDevice(p.number, *context) err = i.formatPartition(p, *context) @@ -413,6 +464,23 @@ func (i ImagePartitionAction) Run(context *debos.DebosContext) error { context.ImageMntDir = path.Join(context.Scratchdir, "mnt") os.MkdirAll(context.ImageMntDir, 0755) + + // sort mountpoints based on position in filesystem hierarchy + sort.SliceStable(i.Mountpoints, func(a, b int) bool { + mntA := i.Mountpoints[a].Mountpoint + mntB := i.Mountpoints[b].Mountpoint + + // root should always be mounted first + if (mntA == "/") { + return true + } + if (mntB == "/") { + return false + } + + return strings.Count(mntA, "/") < strings.Count(mntB, "/") + }) + for _, m := range i.Mountpoints { dev := i.getPartitionDevice(m.part.number, *context) mntpath := path.Join(context.ImageMntDir, m.Mountpoint) @@ -449,6 +517,11 @@ func (i ImagePartitionAction) Cleanup(context *debos.DebosContext) error { if m.Buildtime == true { if err = os.Remove(mntpath); err != nil { log.Printf("Failed to remove temporary mount point %s: %s", m.Mountpoint, err) + + if err.(*os.PathError).Err.Error() == "read-only file system" { + continue + } + return err } } @@ -513,6 +586,31 @@ func (i *ImagePartitionAction) Verify(context *debos.DebosContext) error { if p.Name == "" { return fmt.Errorf("Partition without a name") } + + // check for duplicate partition names + for j := idx + 1; j < len(i.Partitions); j++ { + if i.Partitions[j].Name == p.Name { + return fmt.Errorf("Partition %s already exists", p.Name) + } + } + + if i.PartitionType != "gpt" && p.PartLabel != "" { + return fmt.Errorf("Can only set partition partlabel on GPT filesystem") + } + + if p.PartType != "" { + var partTypeLen int + switch i.PartitionType { + case "gpt": + partTypeLen = 36 + case "msdos": + partTypeLen = 2 + } + if len(p.PartType) != partTypeLen { + return fmt.Errorf("incorrect partition type for %s, should be %d characters", p.Name, partTypeLen) + } + } + if p.Start == "" { return fmt.Errorf("Partition %s missing start", p.Name) } @@ -530,6 +628,14 @@ func (i *ImagePartitionAction) Verify(context *debos.DebosContext) error { for idx, _ := range i.Mountpoints { m := &i.Mountpoints[idx] + + // check for duplicate mountpoints + for j := idx + 1; j < len(i.Mountpoints); j++ { + if i.Mountpoints[j].Mountpoint == m.Mountpoint { + return fmt.Errorf("Mountpoint %s already exists", m.Mountpoint) + } + } + for pidx, _ := range i.Partitions { p := &i.Partitions[pidx] if m.Partition == p.Name { @@ -538,7 +644,7 @@ func (i *ImagePartitionAction) Verify(context *debos.DebosContext) error { } } if m.part == nil { - return fmt.Errorf("Couldn't fount partition for %s", m.Mountpoint) + return fmt.Errorf("Couldn't find partition for %s", m.Mountpoint) } } diff --git a/actions/run_action.go b/actions/run_action.go index 8d36228..13159d1 100644 --- a/actions/run_action.go +++ b/actions/run_action.go @@ -2,7 +2,8 @@ Run Action Allows to run any available command or script in the filesystem or -in host environment. +in build process host environment: specifically inside the fakemachine created +by Debos. Yaml syntax: - action: run @@ -24,8 +25,10 @@ Optional properties: - chroot -- run script or command in target filesystem if set to true. Otherwise the command or script is executed within the build process, with access to the filesystem ($ROOTDIR), the image if any ($IMAGE), the -recipe directory ($RECIPEDIR) and the artifact directory ($ARTIFACTDIR). -In both cases it is run with root privileges. +recipe directory ($RECIPEDIR), the artifact directory ($ARTIFACTDIR) and the +directory where the image is mounted ($IMAGEMNTDIR). +In both cases it is run with root privileges. If unset, chroot is set to false and +the command or script is run in the host environment. - label -- if non-empty, this string is used to label output. If empty, a label is derived from the command or script. @@ -60,6 +63,10 @@ func (run *RunAction) Verify(context *debos.DebosContext) error { if run.PostProcess && run.Chroot { return errors.New("Cannot run postprocessing in the chroot") } + + if run.Script == "" && run.Command == "" { + return errors.New("Script and Command both cannot be empty") + } return nil } @@ -120,6 +127,9 @@ func (run *RunAction) doRun(context debos.DebosContext) error { cmd.AddEnvKey("ROOTDIR", context.Rootdir) cmd.AddEnvKey("RECIPEDIR", context.RecipeDir) cmd.AddEnvKey("ARTIFACTDIR", context.Artifactdir) + if context.ImageMntDir != "" { + cmd.AddEnvKey("IMAGEMNTDIR", context.ImageMntDir) + } } if context.Image != "" { cmd.AddEnvKey("IMAGE", context.Image) diff --git a/commands.go b/commands.go index 52704c5..7bf45cc 100644 --- a/commands.go +++ b/commands.go @@ -223,7 +223,9 @@ func (cmd Command) Run(label string, cmdline ...string) error { options = append(options, cmdline...) case CHROOT_METHOD_NSPAWN: // We use own resolv.conf handling - options = append(options, "systemd-nspawn", "-q", "--resolv-conf=off", "-D", cmd.Chroot) + options = append(options, "systemd-nspawn", "-q") + options = append(options, "--resolv-conf=off") + options = append(options, "--timezone=off") for _, e := range cmd.extraEnv { options = append(options, "--setenv", e) @@ -232,6 +234,7 @@ func (cmd Command) Run(label string, cmdline ...string) error { options = append(options, "--bind", b) } + options = append(options, "-D", cmd.Chroot) options = append(options, cmdline...) } diff --git a/docker/Dockerfile b/docker/Dockerfile index 252c295..c393617 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,12 +20,16 @@ RUN apt-get update && \ libostree-dev && \ rm -rf /var/lib/apt/lists/* +# Build testify from source +# testify v1.3.0 is the last version to work with golang v1.11 +WORKDIR $GOPATH/src/github.com/stretchr/testify +RUN git clone --depth 1 --branch v1.3.0 https://github.com/stretchr/testify . && \ + GO111MODULE=on go get ./... + # Build debos COPY . $GOPATH/src/github.com/go-debos/debos WORKDIR $GOPATH/src/github.com/go-debos/debos/cmd/debos -RUN go get -d ./... && \ - go get -d github.com/stretchr/testify && \ - go install +RUN go get -t ./... ### second stage - runner ### FROM debian:buster-slim as runner @@ -63,6 +67,8 @@ RUN apt-get update && \ debootstrap \ dosfstools \ e2fsprogs \ + fdisk \ + f2fs-tools \ gzip \ pigz \ libostree-1-1 \ @@ -73,6 +79,7 @@ RUN apt-get update && \ qemu-user-static \ systemd \ systemd-container \ + u-boot-tools \ unzip \ xz-utils && \ rm -rf /var/lib/apt/lists/* -- cgit v1.2.3