summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAndrej Shadura <andrew.shadura@collabora.co.uk>2020-10-14 22:18:31 +0200
committerAndrej Shadura <andrew.shadura@collabora.co.uk>2020-10-14 22:18:31 +0200
commit4e972d6d6743bed5dd638e95dd1fdbadc2eaf89e (patch)
treed8d73b99967c44198bd25e4906c7d7b763d81cbe
parentdae893c4893e9c32c62ce7b477b62b49b864de2a (diff)
parent9c6de646795022f856ed7c22d28e2d63e4c0ca22 (diff)
Update upstream source from tag 'upstream/1.0.0+git20200909.d2ab9d3'
Update to upstream version '1.0.0+git20200909.d2ab9d3' with Debian dir db4083400488263e917de99d795542cd87fa9019
-rw-r--r--actions/debootstrap_action.go68
-rw-r--r--actions/filesystem_deploy_action.go9
-rw-r--r--actions/image_partition_action.go142
-rw-r--r--actions/run_action.go16
-rw-r--r--commands.go5
-rw-r--r--docker/Dockerfile13
6 files changed, 222 insertions, 31 deletions
diff --git a/actions/debootstrap_action.go b/actions/debootstrap_action.go
index 77ee63d..506270c 100644
--- a/actions/debootstrap_action.go
+++ b/actions/debootstrap_action.go
@@ -15,6 +15,8 @@ Yaml syntax:
variant: "name"
keyring-package:
keyring-file:
+ certificate:
+ private-key:
Mandatory properties:
@@ -40,6 +42,10 @@ Example:
- keyring-file -- keyring file for repository validation.
- merged-usr -- use merged '/usr' filesystem, true by default.
+
+- certificate -- client certificate stored in file to be used for downloading packages from the server.
+
+- private-key -- provide the client's private key in a file separate from the certificate.
*/
package actions
@@ -51,6 +57,7 @@ import (
"strings"
"github.com/go-debos/debos"
+ "github.com/go-debos/fakemachine"
)
type DebootstrapAction struct {
@@ -60,6 +67,8 @@ type DebootstrapAction struct {
Variant string
KeyringPackage string `yaml:"keyring-package"`
KeyringFile string `yaml:"keyring-file"`
+ Certificate string
+ PrivateKey string `yaml:"private-key"`
Components []string
MergedUsr bool `yaml:"merged-usr"`
CheckGpg bool `yaml:"check-gpg"`
@@ -72,13 +81,57 @@ func NewDebootstrapAction() *DebootstrapAction {
// Be secure by default
d.CheckGpg = true
// Use main as default component
- d.Components = []string {"main"}
+ d.Components = []string{"main"}
// Set generic default mirror
d.Mirror = "http://deb.debian.org/debian"
return &d
}
+func (d *DebootstrapAction) listOptionFiles(context *debos.DebosContext) []string {
+ files := []string{}
+ if d.Certificate != "" {
+ d.Certificate = debos.CleanPathAt(d.Certificate, context.RecipeDir)
+ files = append(files, d.Certificate)
+ }
+
+ if d.PrivateKey != "" {
+ d.PrivateKey = debos.CleanPathAt(d.PrivateKey, context.RecipeDir)
+ files = append(files, d.PrivateKey)
+ }
+
+ if d.KeyringFile != "" {
+ d.KeyringFile = debos.CleanPathAt(d.KeyringFile, context.RecipeDir)
+ files = append(files, d.KeyringFile)
+ }
+
+ return files
+}
+
+func (d *DebootstrapAction) Verify(context *debos.DebosContext) error {
+ files := d.listOptionFiles(context)
+
+ // Check if all needed files exists
+ for _, f := range files {
+ if _, err := os.Stat(f); os.IsNotExist(err) {
+ return err
+ }
+ }
+ return nil
+}
+
+func (d *DebootstrapAction) PreMachine(context *debos.DebosContext, m *fakemachine.Machine, args *[]string) error {
+
+ mounts := d.listOptionFiles(context)
+
+ // Mount configuration files outside of recipes directory
+ for _, mount := range mounts {
+ m.AddVolume(path.Dir(mount))
+ }
+
+ return nil
+}
+
func (d *DebootstrapAction) RunSecondStage(context debos.DebosContext) error {
cmdline := []string{
"/debootstrap/debootstrap",
@@ -96,7 +149,7 @@ func (d *DebootstrapAction) RunSecondStage(context debos.DebosContext) error {
err := c.Run("Debootstrap (stage 2)", cmdline...)
- if (err != nil) {
+ if err != nil {
log := path.Join(context.Rootdir, "debootstrap/debootstrap.log")
_ = debos.Command{}.Run("debootstrap.log", "cat", log)
}
@@ -117,14 +170,21 @@ func (d *DebootstrapAction) Run(context *debos.DebosContext) error {
if !d.CheckGpg {
cmdline = append(cmdline, fmt.Sprintf("--no-check-gpg"))
} else if d.KeyringFile != "" {
- path := debos.CleanPathAt(d.KeyringFile, context.RecipeDir)
- cmdline = append(cmdline, fmt.Sprintf("--keyring=%s", path))
+ cmdline = append(cmdline, fmt.Sprintf("--keyring=%s", d.KeyringFile))
}
if d.KeyringPackage != "" {
cmdline = append(cmdline, fmt.Sprintf("--include=%s", d.KeyringPackage))
}
+ if d.Certificate != "" {
+ cmdline = append(cmdline, fmt.Sprintf("--certificate=%s", d.Certificate))
+ }
+
+ if d.PrivateKey != "" {
+ cmdline = append(cmdline, fmt.Sprintf("--private-key=%s", d.PrivateKey))
+ }
+
if d.Components != nil {
s := strings.Join(d.Components, ",")
cmdline = append(cmdline, fmt.Sprintf("--components=%s", s))
diff --git a/actions/filesystem_deploy_action.go b/actions/filesystem_deploy_action.go
index af087f1..1519904 100644
--- a/actions/filesystem_deploy_action.go
+++ b/actions/filesystem_deploy_action.go
@@ -1,8 +1,13 @@
/*
FilesystemDeploy Action
-Deploy prepared root filesystem to output image. This action requires
-'image-partition' action to be executed before it.
+Deploy prepared root filesystem to output image by copying the files from the
+temporary scratch directory to the mounted image and optionally creates various
+configuration files for the image: '/etc/fstab' and '/etc/kernel/cmdline'. This
+action requires 'image-partition' action to be executed before it.
+
+After this action has ran, subsequent actions are executed on the mounted output
+image.
Yaml syntax:
- action: filesystem-deploy
diff --git a/actions/image_partition_action.go b/actions/image_partition_action.go
index 0850990..9b9cddd 100644
--- a/actions/image_partition_action.go
+++ b/actions/image_partition_action.go
@@ -2,6 +2,10 @@
ImagePartition Action
This action creates an image file, partitions it and formats the filesystems.
+Mountpoints can be defined so the created partitions can be mounted during the
+build, and optionally (but by-default) mounted at boot in the final system. The
+mountpoints are sorted on their position in the filesystem hierarchy so the
+order in the recipe does not matter.
Yaml syntax:
- action: image-partition
@@ -36,8 +40,8 @@ Properties for mount points are described below.
Yaml syntax for partitions:
partitions:
- - name: label
- name: partition name
+ - name: partition name
+ partlabel: partition label
fs: filesystem
start: offset
end: offset
@@ -48,7 +52,8 @@ Yaml syntax for partitions:
Mandatory properties:
- name -- is used for referencing named partition for mount points
-configuration (below) and label the filesystem located on this partition.
+configuration (below) and label the filesystem located on this partition. Must be
+unique.
- fs -- filesystem type used for formatting.
@@ -63,6 +68,17 @@ form -- '32MB', '1GB' or as disk percentage -- '100%'.
Optional properties:
+- partlabel -- label for the partition in the GPT partition table. Defaults
+to the `name` property of the partition. May only be used for GPT partitions.
+
+- parttype -- set the partition type in the partition table. The string should
+be in a hexadecimal format (2-characters) for msdos partition tables and GUID format
+(36-characters) for GPT partition tables. For instance, "82" for msdos sets the
+partition type to Linux Swap. Whereas "0657fd6d-a4ab-43c4-84e5-0933c84b4f4f" for
+GPT sets the partition type to Linux Swap.
+For msdos partition types hex codes see: https://en.wikipedia.org/wiki/Partition_type
+For gpt partition type GUIDs see: https://systemd.io/DISCOVERABLE_PARTITIONS/
+
- features -- list of additional filesystem features which need to be enabled
for partition.
@@ -82,17 +98,18 @@ Yaml syntax for mount points:
Mandatory properties:
-- partition -- partition name for mounting.
+- partition -- partition name for mounting. The partion must exist under `partitions`.
- mountpoint -- path in the target root filesystem where the named partition
-should be mounted.
+should be mounted. Must be unique, only one partition can be mounted per
+mountpoint.
Optional properties:
- options -- list of options to be added to appropriate entry in fstab file.
- buildtime -- if set to true then the mountpoint only used during the debos run.
-No entry in `/etc/fstab' will be created.
+No entry in `/etc/fstab` will be created.
The mountpoints directory will be removed from the image, so it is recommended
to define a `mountpoint` path which is temporary and unique for the image,
for example: `/mnt/temporary_mount`.
@@ -134,6 +151,7 @@ import (
"os/exec"
"path"
"path/filepath"
+ "sort"
"strings"
"syscall"
"time"
@@ -142,15 +160,17 @@ import (
)
type Partition struct {
- number int
- Name string
- Start string
- End string
- FS string
- Flags []string
- Features []string
- Fsck bool "fsck"
- FSUUID string
+ number int
+ Name string
+ PartLabel string
+ PartType string
+ Start string
+ End string
+ FS string
+ Flags []string
+ Features []string
+ Fsck bool "fsck"
+ FSUUID string
}
type Mountpoint struct {
@@ -250,6 +270,16 @@ func (i ImagePartitionAction) getPartitionDevice(number int, context debos.Debos
}
}
+func (i *ImagePartitionAction) triggerDeviceNodes(context *debos.DebosContext) error {
+ err := debos.Command{}.Run("udevadm", "udevadm", "trigger", "--settle", context.Image)
+ if err != nil {
+ log.Printf("Failed to trigger device nodes")
+ return err
+ }
+
+ return nil
+}
+
func (i ImagePartitionAction) PreMachine(context *debos.DebosContext, m *fakemachine.Machine,
args *[]string) error {
image, err := m.CreateImage(i.ImageName, i.size)
@@ -276,6 +306,11 @@ func (i ImagePartitionAction) formatPartition(p *Partition, context debos.DebosC
if len(p.Features) > 0 {
cmdline = append(cmdline, "-O", strings.Join(p.Features, ","))
}
+ case "f2fs":
+ cmdline = append(cmdline, "mkfs.f2fs", "-l", p.Name)
+ if len(p.Features) > 0 {
+ cmdline = append(cmdline, "-O", strings.Join(p.Features, ","))
+ }
case "hfs":
cmdline = append(cmdline, "mkfs.hfs", "-h", "-v", p.Name)
case "hfsplus":
@@ -348,7 +383,9 @@ func (i ImagePartitionAction) Run(context *debos.DebosContext) error {
}
/* Defer will keep the fd open until the function returns, at which points
* the filesystems will have been mounted protecting from more udev funnyness
- */
+ * After the fd is closed the kernel needs to be informed of partition table
+ * changes (defer calls are executed in LIFO order) */
+ defer i.triggerDeviceNodes(context)
defer imageFD.Close()
err = syscall.Flock(int(imageFD.Fd()), syscall.LOCK_EX)
@@ -366,9 +403,14 @@ func (i ImagePartitionAction) Run(context *debos.DebosContext) error {
}
for idx, _ := range i.Partitions {
p := &i.Partitions[idx]
+
+ if p.PartLabel == "" {
+ p.PartLabel = p.Name
+ }
+
var name string
if i.PartitionType == "gpt" {
- name = p.Name
+ name = p.PartLabel
} else {
name = "primary"
}
@@ -379,6 +421,7 @@ func (i ImagePartitionAction) Run(context *debos.DebosContext) error {
command = append(command, "fat32")
case "hfsplus":
command = append(command, "hfs+")
+ case "f2fs":
case "none":
default:
command = append(command, p.FS)
@@ -400,6 +443,14 @@ func (i ImagePartitionAction) Run(context *debos.DebosContext) error {
}
}
+ if p.PartType != "" {
+ err = debos.Command{}.Run("sfdisk", "sfdisk", "--part-type", context.Image, fmt.Sprintf("%d", p.number), p.PartType)
+ if err != nil {
+ return err
+ }
+ }
+
+
devicePath := i.getPartitionDevice(p.number, *context)
err = i.formatPartition(p, *context)
@@ -413,6 +464,23 @@ func (i ImagePartitionAction) Run(context *debos.DebosContext) error {
context.ImageMntDir = path.Join(context.Scratchdir, "mnt")
os.MkdirAll(context.ImageMntDir, 0755)
+
+ // sort mountpoints based on position in filesystem hierarchy
+ sort.SliceStable(i.Mountpoints, func(a, b int) bool {
+ mntA := i.Mountpoints[a].Mountpoint
+ mntB := i.Mountpoints[b].Mountpoint
+
+ // root should always be mounted first
+ if (mntA == "/") {
+ return true
+ }
+ if (mntB == "/") {
+ return false
+ }
+
+ return strings.Count(mntA, "/") < strings.Count(mntB, "/")
+ })
+
for _, m := range i.Mountpoints {
dev := i.getPartitionDevice(m.part.number, *context)
mntpath := path.Join(context.ImageMntDir, m.Mountpoint)
@@ -449,6 +517,11 @@ func (i ImagePartitionAction) Cleanup(context *debos.DebosContext) error {
if m.Buildtime == true {
if err = os.Remove(mntpath); err != nil {
log.Printf("Failed to remove temporary mount point %s: %s", m.Mountpoint, err)
+
+ if err.(*os.PathError).Err.Error() == "read-only file system" {
+ continue
+ }
+
return err
}
}
@@ -513,6 +586,31 @@ func (i *ImagePartitionAction) Verify(context *debos.DebosContext) error {
if p.Name == "" {
return fmt.Errorf("Partition without a name")
}
+
+ // check for duplicate partition names
+ for j := idx + 1; j < len(i.Partitions); j++ {
+ if i.Partitions[j].Name == p.Name {
+ return fmt.Errorf("Partition %s already exists", p.Name)
+ }
+ }
+
+ if i.PartitionType != "gpt" && p.PartLabel != "" {
+ return fmt.Errorf("Can only set partition partlabel on GPT filesystem")
+ }
+
+ if p.PartType != "" {
+ var partTypeLen int
+ switch i.PartitionType {
+ case "gpt":
+ partTypeLen = 36
+ case "msdos":
+ partTypeLen = 2
+ }
+ if len(p.PartType) != partTypeLen {
+ return fmt.Errorf("incorrect partition type for %s, should be %d characters", p.Name, partTypeLen)
+ }
+ }
+
if p.Start == "" {
return fmt.Errorf("Partition %s missing start", p.Name)
}
@@ -530,6 +628,14 @@ func (i *ImagePartitionAction) Verify(context *debos.DebosContext) error {
for idx, _ := range i.Mountpoints {
m := &i.Mountpoints[idx]
+
+ // check for duplicate mountpoints
+ for j := idx + 1; j < len(i.Mountpoints); j++ {
+ if i.Mountpoints[j].Mountpoint == m.Mountpoint {
+ return fmt.Errorf("Mountpoint %s already exists", m.Mountpoint)
+ }
+ }
+
for pidx, _ := range i.Partitions {
p := &i.Partitions[pidx]
if m.Partition == p.Name {
@@ -538,7 +644,7 @@ func (i *ImagePartitionAction) Verify(context *debos.DebosContext) error {
}
}
if m.part == nil {
- return fmt.Errorf("Couldn't fount partition for %s", m.Mountpoint)
+ return fmt.Errorf("Couldn't find partition for %s", m.Mountpoint)
}
}
diff --git a/actions/run_action.go b/actions/run_action.go
index 8d36228..13159d1 100644
--- a/actions/run_action.go
+++ b/actions/run_action.go
@@ -2,7 +2,8 @@
Run Action
Allows to run any available command or script in the filesystem or
-in host environment.
+in build process host environment: specifically inside the fakemachine created
+by Debos.
Yaml syntax:
- action: run
@@ -24,8 +25,10 @@ Optional properties:
- chroot -- run script or command in target filesystem if set to true.
Otherwise the command or script is executed within the build process, with
access to the filesystem ($ROOTDIR), the image if any ($IMAGE), the
-recipe directory ($RECIPEDIR) and the artifact directory ($ARTIFACTDIR).
-In both cases it is run with root privileges.
+recipe directory ($RECIPEDIR), the artifact directory ($ARTIFACTDIR) and the
+directory where the image is mounted ($IMAGEMNTDIR).
+In both cases it is run with root privileges. If unset, chroot is set to false and
+the command or script is run in the host environment.
- label -- if non-empty, this string is used to label output. If empty,
a label is derived from the command or script.
@@ -60,6 +63,10 @@ func (run *RunAction) Verify(context *debos.DebosContext) error {
if run.PostProcess && run.Chroot {
return errors.New("Cannot run postprocessing in the chroot")
}
+
+ if run.Script == "" && run.Command == "" {
+ return errors.New("Script and Command both cannot be empty")
+ }
return nil
}
@@ -120,6 +127,9 @@ func (run *RunAction) doRun(context debos.DebosContext) error {
cmd.AddEnvKey("ROOTDIR", context.Rootdir)
cmd.AddEnvKey("RECIPEDIR", context.RecipeDir)
cmd.AddEnvKey("ARTIFACTDIR", context.Artifactdir)
+ if context.ImageMntDir != "" {
+ cmd.AddEnvKey("IMAGEMNTDIR", context.ImageMntDir)
+ }
}
if context.Image != "" {
cmd.AddEnvKey("IMAGE", context.Image)
diff --git a/commands.go b/commands.go
index 52704c5..7bf45cc 100644
--- a/commands.go
+++ b/commands.go
@@ -223,7 +223,9 @@ func (cmd Command) Run(label string, cmdline ...string) error {
options = append(options, cmdline...)
case CHROOT_METHOD_NSPAWN:
// We use own resolv.conf handling
- options = append(options, "systemd-nspawn", "-q", "--resolv-conf=off", "-D", cmd.Chroot)
+ options = append(options, "systemd-nspawn", "-q")
+ options = append(options, "--resolv-conf=off")
+ options = append(options, "--timezone=off")
for _, e := range cmd.extraEnv {
options = append(options, "--setenv", e)
@@ -232,6 +234,7 @@ func (cmd Command) Run(label string, cmdline ...string) error {
options = append(options, "--bind", b)
}
+ options = append(options, "-D", cmd.Chroot)
options = append(options, cmdline...)
}
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 252c295..c393617 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -20,12 +20,16 @@ RUN apt-get update && \
libostree-dev && \
rm -rf /var/lib/apt/lists/*
+# Build testify from source
+# testify v1.3.0 is the last version to work with golang v1.11
+WORKDIR $GOPATH/src/github.com/stretchr/testify
+RUN git clone --depth 1 --branch v1.3.0 https://github.com/stretchr/testify . && \
+ GO111MODULE=on go get ./...
+
# Build debos
COPY . $GOPATH/src/github.com/go-debos/debos
WORKDIR $GOPATH/src/github.com/go-debos/debos/cmd/debos
-RUN go get -d ./... && \
- go get -d github.com/stretchr/testify && \
- go install
+RUN go get -t ./...
### second stage - runner ###
FROM debian:buster-slim as runner
@@ -63,6 +67,8 @@ RUN apt-get update && \
debootstrap \
dosfstools \
e2fsprogs \
+ fdisk \
+ f2fs-tools \
gzip \
pigz \
libostree-1-1 \
@@ -73,6 +79,7 @@ RUN apt-get update && \
qemu-user-static \
systemd \
systemd-container \
+ u-boot-tools \
unzip \
xz-utils && \
rm -rf /var/lib/apt/lists/*