diff options
author | Héctor Orón Martínez <zumbi@debian.org> | 2018-01-09 19:12:07 +0100 |
---|---|---|
committer | Héctor Orón Martínez <zumbi@debian.org> | 2018-01-09 19:12:07 +0100 |
commit | 4808cb7058c548bf76476ec2f9618d784d76bdda (patch) | |
tree | 1dc1e8cc24171783fc8d9da306b1e92798960a15 /actions |
New upstream version 1.0.0+git20171222.87b0d5e
Diffstat (limited to 'actions')
-rw-r--r-- | actions/actions_doc.go | 6 | ||||
-rw-r--r-- | actions/apt_action.go | 61 | ||||
-rw-r--r-- | actions/debootstrap_action.go | 146 | ||||
-rw-r--r-- | actions/download_action.go | 168 | ||||
-rw-r--r-- | actions/filesystem_deploy_action.go | 131 | ||||
-rw-r--r-- | actions/image_partition_action.go | 432 | ||||
-rw-r--r-- | actions/ostree_commit_action.go | 84 | ||||
-rw-r--r-- | actions/ostree_deploy_action.go | 183 | ||||
-rw-r--r-- | actions/overlay_action.go | 67 | ||||
-rw-r--r-- | actions/pack_action.go | 39 | ||||
-rw-r--r-- | actions/raw_action.go | 130 | ||||
-rw-r--r-- | actions/run_action.go | 132 | ||||
-rw-r--r-- | actions/unpack_action.go | 100 |
13 files changed, 1679 insertions, 0 deletions
diff --git a/actions/actions_doc.go b/actions/actions_doc.go new file mode 100644 index 0000000..7ebd532 --- /dev/null +++ b/actions/actions_doc.go @@ -0,0 +1,6 @@ +// Copyright 2017, Collabora Ltd. + +/* +Package 'actions' implements 'debos' modules used for OS creation. +*/ +package actions diff --git a/actions/apt_action.go b/actions/apt_action.go new file mode 100644 index 0000000..681c069 --- /dev/null +++ b/actions/apt_action.go @@ -0,0 +1,61 @@ +/* +Apt Action + +Install packages and their dependencies to the target rootfs with 'apt'. + +Yaml syntax: + - action: apt + recommends: bool + packages: + - package1 + - package2 + +Mandatory properties: + +- packages -- list of packages to install + +Optional properties: + +- recommends -- boolean indicating if suggested packages will be installed +*/ +package actions + +import ( + "github.com/go-debos/debos" +) + +type AptAction struct { + debos.BaseAction `yaml:",inline"` + Recommends bool + Packages []string +} + +func (apt *AptAction) Run(context *debos.DebosContext) error { + apt.LogStart() + aptOptions := []string{"apt-get", "-y"} + + if !apt.Recommends { + aptOptions = append(aptOptions, "--no-install-recommends") + } + + aptOptions = append(aptOptions, "install") + aptOptions = append(aptOptions, apt.Packages...) + + c := debos.NewChrootCommandForContext(*context) + c.AddEnv("DEBIAN_FRONTEND=noninteractive") + + err := c.Run("apt", "apt-get", "update") + if err != nil { + return err + } + err = c.Run("apt", aptOptions...) + if err != nil { + return err + } + err = c.Run("apt", "apt-get", "clean") + if err != nil { + return err + } + + return nil +} diff --git a/actions/debootstrap_action.go b/actions/debootstrap_action.go new file mode 100644 index 0000000..a53d153 --- /dev/null +++ b/actions/debootstrap_action.go @@ -0,0 +1,146 @@ +/* +Debootstrap Action + +Construct the target rootfs with debootstrap tool. + +Yaml syntax: + - action: debootstrap + mirror: URL + suite: "name" + components: <list of components> + variant: "name" + keyring-package: + +Mandatory properties: + +- suite -- release code name or symbolic name (e.g. "stable") + +Optional properties: + +- mirror -- URL with Debian-compatible repository + +- variant -- name of the bootstrap script variant to use + +- components -- list of components to use for packages selection. +Example: + components: [ main, contrib ] + +- keyring-package -- keyring for packages validation. Currently ignored. + +- merged-usr -- use merged '/usr' filesystem, true by default. +*/ +package actions + +import ( + "fmt" + "io" + "os" + "path" + "strings" + + "github.com/go-debos/debos" +) + +type DebootstrapAction struct { + debos.BaseAction `yaml:",inline"` + Suite string + Mirror string + Variant string + KeyringPackage string `yaml:"keyring-package"` + Components []string + MergedUsr bool `yaml:"merged-usr"` +} + +func NewDebootstrapAction() *DebootstrapAction { + d := DebootstrapAction{} + // Use filesystem with merged '/usr' by default + d.MergedUsr = true + return &d + +} + +func (d *DebootstrapAction) RunSecondStage(context debos.DebosContext) error { + cmdline := []string{ + "/debootstrap/debootstrap", + "--no-check-gpg", + "--second-stage"} + + if d.Components != nil { + s := strings.Join(d.Components, ",") + cmdline = append(cmdline, fmt.Sprintf("--components=%s", s)) + } + + c := debos.NewChrootCommandForContext(context) + // Can't use nspawn for debootstrap as it wants to create device nodes + c.ChrootMethod = debos.CHROOT_METHOD_CHROOT + + return c.Run("Debootstrap (stage 2)", cmdline...) +} + +func (d *DebootstrapAction) Run(context *debos.DebosContext) error { + d.LogStart() + cmdline := []string{"debootstrap", "--no-check-gpg"} + + if d.MergedUsr { + cmdline = append(cmdline, "--merged-usr") + } + + if d.KeyringPackage != "" { + cmdline = append(cmdline, fmt.Sprintf("--keyring=%s", d.KeyringPackage)) + } + + if d.Components != nil { + s := strings.Join(d.Components, ",") + cmdline = append(cmdline, fmt.Sprintf("--components=%s", s)) + } + + /* FIXME drop the hardcoded amd64 assumption" */ + foreign := context.Architecture != "amd64" + + if foreign { + cmdline = append(cmdline, "--foreign") + cmdline = append(cmdline, fmt.Sprintf("--arch=%s", context.Architecture)) + + } + + if d.Variant != "" { + cmdline = append(cmdline, fmt.Sprintf("--variant=%s", d.Variant)) + } + + cmdline = append(cmdline, d.Suite) + cmdline = append(cmdline, context.Rootdir) + cmdline = append(cmdline, d.Mirror) + cmdline = append(cmdline, "/usr/share/debootstrap/scripts/unstable") + + err := debos.Command{}.Run("Debootstrap", cmdline...) + + if err != nil { + return err + } + + if foreign { + err = d.RunSecondStage(*context) + if err != nil { + return err + } + } + + /* HACK */ + srclist, err := os.OpenFile(path.Join(context.Rootdir, "etc/apt/sources.list"), + os.O_RDWR|os.O_CREATE, 0755) + if err != nil { + return err + } + _, err = io.WriteString(srclist, fmt.Sprintf("deb %s %s %s\n", + d.Mirror, + d.Suite, + strings.Join(d.Components, " "))) + if err != nil { + return err + } + srclist.Close() + + c := debos.NewChrootCommandForContext(*context) + + return c.Run("apt clean", "/usr/bin/apt-get", "clean") +} diff --git a/actions/download_action.go b/actions/download_action.go new file mode 100644 index 0000000..a4099e7 --- /dev/null +++ b/actions/download_action.go @@ -0,0 +1,168 @@ +/* +Download Action + +Download a single file from Internet and unpack it in place if needed. + +Yaml syntax: + - action: download + url: http://example.domain/path/filename.ext + name: firmware + filename: output_name + unpack: bool + compression: gz + +Mandatory properties: + +- url -- URL to an object for download + +- name -- string which allow to use downloaded object in other actions +via 'origin' property. If 'unpack' property is set to 'true' name will +refer to temporary directory with extracted content. + +Optional properties: + +- filename -- use this property as the name for saved file. Useful if URL does not +contain file name in path, for example it is possible to download files from URLs without path part. + +- unpack -- hint for action to extract all files from downloaded archive. +See the 'Unpack' action for more information. + +- compression -- optional hint for unpack allowing to use proper compression method. +See the 'Unpack' action for more information. +*/ +package actions + +import ( + "fmt" + "github.com/go-debos/debos" + "net/url" + "path" +) + +type DownloadAction struct { + debos.BaseAction `yaml:",inline"` + Url string // URL for downloading + Filename string // File name, overrides the name from URL. + Unpack bool // Unpack downloaded file to directory dedicated for download + Compression string // compression type + Name string // exporting path to file or directory(in case of unpack) +} + +// validateUrl checks if supported URL is passed from recipe +// Return: +// - parsed URL +// - nil in case of success +func (d *DownloadAction) validateUrl() (*url.URL, error) { + + url, err := url.Parse(d.Url) + if err != nil { + return url, err + } + + switch url.Scheme { + case "http", "https": + // Supported scheme + default: + return url, fmt.Errorf("Unsupported URL is provided: '%s'", url.String()) + } + + return url, nil +} + +func (d *DownloadAction) validateFilename(context *debos.DebosContext, url *url.URL) (filename string, err error) { + if len(d.Filename) == 0 { + // Trying to guess the name from URL Path + filename = path.Base(url.Path) + } else { + filename = path.Base(d.Filename) + } + if len(filename) == 0 { + return "", fmt.Errorf("Incorrect filename is provided for '%s'", d.Url) + } + filename = path.Join(context.Scratchdir, filename) + return filename, nil +} + +func (d *DownloadAction) archive(filename string) (debos.Archive, error) { + archive, err := debos.NewArchive(filename) + if err != nil { + return archive, err + } + switch archive.Type() { + case debos.Tar: + if len(d.Compression) > 0 { + if err := archive.AddOption("tarcompression", d.Compression); err != nil { + return archive, err + } + } + default: + } + return archive, nil +} + +func (d *DownloadAction) Verify(context *debos.DebosContext) error { + var filename string + + if len(d.Name) == 0 { + return fmt.Errorf("Property 'name' is mandatory for download action\n") + } + + url, err := d.validateUrl() + if err != nil { + return err + } + filename, err = d.validateFilename(context, url) + if err != nil { + return err + } + if d.Unpack == true { + if _, err := d.archive(filename); err != nil { + return err + } + } + return nil +} + +func (d *DownloadAction) Run(context *debos.DebosContext) error { + var filename string + d.LogStart() + + url, err := d.validateUrl() + if err != nil { + return err + } + + filename, err = d.validateFilename(context, url) + if err != nil { + return err + } + originPath := filename + + switch url.Scheme { + case "http", "https": + err := debos.DownloadHttpUrl(url.String(), filename) + if err != nil { + return err + } + default: + return fmt.Errorf("Unsupported URL is provided: '%s'", url.String()) + } + + if d.Unpack == true { + archive, err := d.archive(filename) + if err != nil { + return err + } + + targetdir := filename + ".d" + err = archive.RelaxedUnpack(targetdir) + if err != nil { + return err + } + originPath = targetdir + } + + context.Origins[d.Name] = originPath + + return nil +} diff --git a/actions/filesystem_deploy_action.go b/actions/filesystem_deploy_action.go new file mode 100644 index 0000000..bb89a83 --- /dev/null +++ b/actions/filesystem_deploy_action.go @@ -0,0 +1,131 @@ +/* +FilesystemDeploy Action + +Deploy prepared root filesystem to output image. This action requires +'image-partition' action to be executed before it. + +Yaml syntax: + - action: filesystem-deploy + setup-fstab: bool + setup-kernel-cmdline: bool + +Optional properties: + +- setup-fstab -- generate '/etc/fstab' file according to information provided +by 'image-partition' action. By default is 'true'. + +- setup-kernel-cmdline -- add location of root partition to '/etc/kernel/cmdline' +file on target image. By default is 'true'. +*/ +package actions + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path" + "strings" + + "github.com/go-debos/debos" +) + +type FilesystemDeployAction struct { + debos.BaseAction `yaml:",inline"` + SetupFSTab bool `yaml:"setup-fstab"` + SetupKernelCmdline bool `yaml:"setup-kernel-cmdline"` +} + +func NewFilesystemDeployAction() *FilesystemDeployAction { + fd := &FilesystemDeployAction{SetupFSTab: true, SetupKernelCmdline: true} + fd.Description = "Deploying filesystem" + + return fd +} + +func (fd *FilesystemDeployAction) setupFSTab(context *debos.DebosContext) error { + if context.ImageFSTab.Len() == 0 { + return errors.New("Fstab not generated, missing image-partition action?") + } + + log.Print("Setting up fstab") + + err := os.MkdirAll(path.Join(context.Rootdir, "etc"), 0755) + if err != nil { + return fmt.Errorf("Couldn't create etc in image: %v", err) + } + + fstab := path.Join(context.Rootdir, "etc/fstab") + f, err := os.OpenFile(fstab, os.O_RDWR|os.O_CREATE, 0755) + + if err != nil { + return fmt.Errorf("Couldn't open fstab: %v", err) + } + + _, err = io.Copy(f, &context.ImageFSTab) + + if err != nil { + return fmt.Errorf("Couldn't write fstab: %v", err) + } + f.Close() + + return nil +} + +func (fd *FilesystemDeployAction) setupKernelCmdline(context *debos.DebosContext) error { + log.Print("Setting up /etc/kernel/cmdline") + + err := os.MkdirAll(path.Join(context.Rootdir, "etc", "kernel"), 0755) + if err != nil { + return fmt.Errorf("Couldn't create etc/kernel in image: %v", err) + } + path := path.Join(context.Rootdir, "etc/kernel/cmdline") + current, _ := ioutil.ReadFile(path) + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0755) + + if err != nil { + log.Fatalf("Couldn't open kernel cmdline: %v", err) + } + + cmdline := fmt.Sprintf("%s %s\n", + strings.TrimSpace(string(current)), + context.ImageKernelRoot) + + _, err = f.WriteString(cmdline) + if err != nil { + return fmt.Errorf("Couldn't write kernel/cmdline: %v", err) + } + + f.Close() + return nil +} + +func (fd *FilesystemDeployAction) Run(context *debos.DebosContext) error { + fd.LogStart() + /* Copying files is actually silly hafd, one has to keep permissions, ACL's + * extended attribute, misc, other. Leave it to cp... + */ + err := debos.Command{}.Run("Deploy to image", "cp", "-a", context.Rootdir+"/.", context.ImageMntDir) + if err != nil { + return fmt.Errorf("rootfs deploy failed: %v", err) + } + context.Rootdir = context.ImageMntDir + context.Origins["filesystem"] = context.ImageMntDir + + if fd.SetupFSTab { + err = fd.setupFSTab(context) + if err != nil { + return err + } + } + if fd.SetupKernelCmdline { + err = fd.setupKernelCmdline(context) + if err != nil { + return err + } + } + + return nil +} diff --git a/actions/image_partition_action.go b/actions/image_partition_action.go new file mode 100644 index 0000000..5054e6d --- /dev/null +++ b/actions/image_partition_action.go @@ -0,0 +1,432 @@ +/* +ImagePartition Action + +This action creates an image file, partitions it and formats the filesystems. + +Yaml syntax: + - action: image-partition + imagename: image_name + imagesize: size + partitiontype: gpt + gpt_gap: offset + partitions: + <list of partitions> + mountpoints: + <list of mount points> + +Mandatory properties: + +- imagename -- the name of the image file. + +- imagesize -- generated image size in human-readable form, examples: 100MB, 1GB, etc. + +- partitiontype -- partition table type. Currently only 'gpt' and 'msdos' +partition tables are supported. + +- gpt_gap -- shifting GPT allow to use this gap for bootloaders, for example if +U-Boot intersects with original GPT placement. +Only works if parted supports an extra argument to mklabel to specify the gpt offset. + +- partitions -- list of partitions, at least one partition is needed. +Partition properties are described below. + +- mountpoints -- list of mount points for partitions. +Properties for mount points are described below. + +Yaml syntax for partitions: + + partitions: + - name: label + name: partition name + fs: filesystem + start: offset + end: offset + flags: list of flags + +Mandatory properties: + +- name -- is used for referencing named partition for mount points +configuration (below) and label the filesystem located on this partition. + +- fs -- filesystem type used for formatting. + +'none' fs type should be used for partition without filesystem. + +- start -- offset from beginning of the disk there the partition starts. + +- end -- offset from beginning of the disk there the partition ends. + +For 'start' and 'end' properties offset can be written in human readable +form -- '32MB', '1GB' or as disk percentage -- '100%'. + +Optional properties: + +- flags -- list of additional flags for partition compatible with parted(8) +'set' command. + +Yaml syntax for mount points: + + mountpoints: + - mountpoint: path + partition: partition label + options: list of options + +Mandatory properties: + +- partition -- partition name for mounting. + +- mountpoint -- path in the target root filesystem where the named partition +should be mounted. + +Optional properties: + +- options -- list of options to be added to appropriate entry in fstab file. + +Layout example for Raspberry PI 3: + + - action: image-partition + imagename: "debian-rpi3.img" + imagesize: 1GB + partitiontype: msdos + mountpoints: + - mountpoint: / + partition: root + - mountpoint: /boot/firmware + partition: firmware + options: [ x-systemd.automount ] + partitions: + - name: firmware + fs: vfat + start: 0% + end: 64MB + - name: root + fs: ext4 + start: 64MB + end: 100% + flags: [ boot ] +*/ +package actions + +import ( + "errors" + "fmt" + "github.com/docker/go-units" + "github.com/go-debos/fakemachine" + "log" + "os" + "os/exec" + "path" + "strings" + "syscall" + + "github.com/go-debos/debos" +) + +type Partition struct { + number int + Name string + Start string + End string + FS string + Flags []string + FSUUID string +} + +type Mountpoint struct { + Mountpoint string + Partition string + Options []string + part *Partition +} + +type ImagePartitionAction struct { + debos.BaseAction `yaml:",inline"` + ImageName string + ImageSize string + PartitionType string + GptGap string "gpt_gap" + Partitions []Partition + Mountpoints []Mountpoint + size int64 + usingLoop bool +} + +func (i *ImagePartitionAction) generateFSTab(context *debos.DebosContext) error { + context.ImageFSTab.Reset() + + for _, m := range i.Mountpoints { + options := []string{"defaults"} + options = append(options, m.Options...) + if m.part.FSUUID == "" { + return fmt.Errorf("Missing fs UUID for partition %s!?!", m.part.Name) + } + context.ImageFSTab.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0\t0\n", + m.part.FSUUID, m.Mountpoint, m.part.FS, + strings.Join(options, ","))) + } + + return nil +} + +func (i *ImagePartitionAction) generateKernelRoot(context *debos.DebosContext) error { + for _, m := range i.Mountpoints { + if m.Mountpoint == "/" { + if m.part.FSUUID == "" { + return errors.New("No fs UUID for root partition !?!") + } + context.ImageKernelRoot = fmt.Sprintf("root=UUID=%s", m.part.FSUUID) + break + } + } + + return nil +} + +func (i ImagePartitionAction) getPartitionDevice(number int, context debos.DebosContext) string { + suffix := "p" + /* Check partition naming first: if used 'by-id'i naming convention */ + if strings.Contains(context.Image, "/disk/by-id/") { + suffix = "-part" + } + + /* If the iamge device has a digit as the last character, the partition + * suffix is p<number> else it's just <number> */ + last := context.Image[len(context.Image)-1] + if last >= '0' && last <= '9' { + return fmt.Sprintf("%s%s%d", context.Image, suffix, number) + } else { + return fmt.Sprintf("%s%d", context.Image, number) + } +} + +func (i ImagePartitionAction) PreMachine(context *debos.DebosContext, m *fakemachine.Machine, + args *[]string) error { + image, err := m.CreateImage(i.ImageName, i.size) + if err != nil { + return err + } + + context.Image = image + *args = append(*args, "--internal-image", image) + return nil +} + +func (i ImagePartitionAction) formatPartition(p *Partition, context debos.DebosContext) error { + label := fmt.Sprintf("Formatting partition %d", p.number) + path := i.getPartitionDevice(p.number, context) + + cmdline := []string{} + switch p.FS { + case "vfat": + cmdline = append(cmdline, "mkfs.vfat", "-n", p.Name) + case "btrfs": + // Force formatting to prevent failure in case if partition was formatted already + cmdline = append(cmdline, "mkfs.btrfs", "-L", p.Name, "-f") + case "none": + default: + cmdline = append(cmdline, fmt.Sprintf("mkfs.%s", p.FS), "-L", p.Name) + } + + if len(cmdline) != 0 { + cmdline = append(cmdline, path) + + cmd := debos.Command{} + if err := cmd.Run(label, cmdline...); err != nil { + return err + } + } + + if p.FS != "none" { + uuid, err := exec.Command("blkid", "-o", "value", "-s", "UUID", "-p", "-c", "none", path).Output() + if err != nil { + return fmt.Errorf("Failed to get uuid: %s", err) + } + p.FSUUID = strings.TrimSpace(string(uuid[:])) + } + + return nil +} + +func (i ImagePartitionAction) PreNoMachine(context *debos.DebosContext) error { + + img, err := os.OpenFile(i.ImageName, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + return fmt.Errorf("Couldn't open image file: %v", err) + } + + err = img.Truncate(i.size) + if err != nil { + return fmt.Errorf("Couldn't resize image file: %v", err) + } + + img.Close() + + loop, err := exec.Command("losetup", "-f", "--show", i.ImageName).Output() + if err != nil { + return fmt.Errorf("Failed to setup loop device") + } + context.Image = strings.TrimSpace(string(loop[:])) + i.usingLoop = true + + return nil +} + +func (i ImagePartitionAction) Run(context *debos.DebosContext) error { + i.LogStart() + + command := []string{"parted", "-s", context.Image, "mklabel", i.PartitionType} + if len(i.GptGap) > 0 { + command = append(command, i.GptGap) + } + err := debos.Command{}.Run("parted", command...) + if err != nil { + return err + } + for idx, _ := range i.Partitions { + p := &i.Partitions[idx] + var name string + if i.PartitionType == "gpt" { + name = p.Name + } else { + name = "primary" + } + + command := []string{"parted", "-a", "none", "-s", "--", context.Image, "mkpart", name} + switch p.FS { + case "vfat": + command = append(command, "fat32") + case "none": + default: + command = append(command, p.FS) + } + command = append(command, p.Start, p.End) + + err = debos.Command{}.Run("parted", command...) + if err != nil { + return err + } + + if p.Flags != nil { + for _, flag := range p.Flags { + err = debos.Command{}.Run("parted", "parted", "-s", context.Image, "set", + fmt.Sprintf("%d", p.number), flag, "on") + if err != nil { + return err + } + } + } + + devicePath := i.getPartitionDevice(p.number, *context) + // Give a chance for udevd to create proper symlinks + err = debos.Command{}.Run("udevadm", "udevadm", "settle", "-t", "5", + "-E", devicePath) + if err != nil { + return err + } + + err = i.formatPartition(p, *context) + if err != nil { + return err + } + + context.ImagePartitions = append(context.ImagePartitions, + debos.Partition{p.Name, devicePath}) + } + + context.ImageMntDir = path.Join(context.Scratchdir, "mnt") + os.MkdirAll(context.ImageMntDir, 0755) + for _, m := range i.Mountpoints { + dev := i.getPartitionDevice(m.part.number, *context) + mntpath := path.Join(context.ImageMntDir, m.Mountpoint) + os.MkdirAll(mntpath, 0755) + err := syscall.Mount(dev, mntpath, m.part.FS, 0, "") + if err != nil { + return fmt.Errorf("%s mount failed: %v", m.part.Name, err) + } + } + + err = i.generateFSTab(context) + if err != nil { + return err + } + + err = i.generateKernelRoot(context) + if err != nil { + return err + } + + return nil +} + +func (i ImagePartitionAction) Cleanup(context debos.DebosContext) error { + for idx := len(i.Mountpoints) - 1; idx >= 0; idx-- { + m := i.Mountpoints[idx] + mntpath := path.Join(context.ImageMntDir, m.Mountpoint) + syscall.Unmount(mntpath, 0) + } + + if i.usingLoop { + exec.Command("losetup", "-d", context.Image).Run() + } + + return nil +} + +func (i *ImagePartitionAction) Verify(context *debos.DebosContext) error { + if len(i.GptGap) > 0 { + log.Println("WARNING: special version of parted is needed for 'gpt_gap' option") + if i.PartitionType != "gpt" { + return fmt.Errorf("gpt_gap property could be used only with 'gpt' label") + } + // Just check if it contains correct value + _, err := units.FromHumanSize(i.GptGap) + if err != nil { + return fmt.Errorf("Failed to parse GPT offset: %s", i.GptGap) + } + } + + num := 1 + for idx, _ := range i.Partitions { + p := &i.Partitions[idx] + p.number = num + num++ + if p.Name == "" { + return fmt.Errorf("Partition without a name") + } + if p.Start == "" { + return fmt.Errorf("Partition %s missing start", p.Name) + } + if p.End == "" { + return fmt.Errorf("Partition %s missing end", p.Name) + } + + switch p.FS { + case "fat32": + p.FS = "vfat" + case "": + return fmt.Errorf("Partition %s missing fs type", p.Name) + } + } + + for idx, _ := range i.Mountpoints { + m := &i.Mountpoints[idx] + for pidx, _ := range i.Partitions { + p := &i.Partitions[pidx] + if m.Partition == p.Name { + m.part = p + break + } + } + if m.part == nil { + return fmt.Errorf("Couldn't fount partition for %s", m.Mountpoint) + } + } + + size, err := units.FromHumanSize(i.ImageSize) + if err != nil { + return fmt.Errorf("Failed to parse image size: %s", i.ImageSize) + } + + i.size = size + return nil +} diff --git a/actions/ostree_commit_action.go b/actions/ostree_commit_action.go new file mode 100644 index 0000000..6d41b89 --- /dev/null +++ b/actions/ostree_commit_action.go @@ -0,0 +1,84 @@ +/* +OstreeCommit Action + +Create OSTree commit from rootfs. + +Yaml syntax: + - action: ostree-commit + repository: repository name + branch: branch name + subject: commit message + +Mandatory properties: + +- repository -- path to repository with OSTree structure; the same path is +used by 'ostree' tool with '--repo' argument. +This path is relative to 'artifact' directory. +Please keep in mind -- you will need a root privileges for 'bare' repository +type (https://ostree.readthedocs.io/en/latest/manual/repo/#repository-types-and-locations). + +- branch -- OSTree branch name that should be used for the commit. + +Optional properties: + +- subject -- one line message with commit description. +*/ +package actions + +import ( + "log" + "os" + "path" + + "github.com/go-debos/debos" + "github.com/sjoerdsimons/ostree-go/pkg/otbuiltin" +) + +type OstreeCommitAction struct { + debos.BaseAction `yaml:",inline"` + Repository string + Branch string + Subject string + Command string +} + +func emptyDir(dir string) { + d, _ := os.Open(dir) + defer d.Close() + files, _ := d.Readdirnames(-1) + for _, f := range files { + os.RemoveAll(f) + } +} + +func (ot *OstreeCommitAction) Run(context *debos.DebosContext) error { + ot.LogStart() + repoPath := path.Join(context.Artifactdir, ot.Repository) + + emptyDir(path.Join(context.Rootdir, "dev")) + + repo, err := otbuiltin.OpenRepo(repoPath) + if err != nil { + return err + } + + _, err = repo.PrepareTransaction() + if err != nil { + return err + } + + opts := otbuiltin.NewCommitOptions() + opts.Subject = ot.Subject + ret, err := repo.Commit(context.Rootdir, ot.Branch, opts) + if err != nil { + return err + } else { + log.Printf("Commit: %s\n", ret) + } + _, err = repo.CommitTransaction() + if err != nil { + return err + } + + return nil +} diff --git a/actions/ostree_deploy_action.go b/actions/ostree_deploy_action.go new file mode 100644 index 0000000..b2d373d --- /dev/null +++ b/actions/ostree_deploy_action.go @@ -0,0 +1,183 @@ +/* +OstreeDeploy Action + +Deploy the OSTree branch to the image. +If any preparation has been done for rootfs, it can be overwritten +during this step. + +Action 'image-partition' must be called prior to OSTree deploy. + +Yaml syntax: + - action: ostree-deploy + repository: repository name + remote_repository: URL + branch: branch name + os: os name + setup-fstab: bool + setup-kernel-cmdline: bool + appendkernelcmdline: arguments + +Mandatory properties: + +- remote_repository -- URL to remote OSTree repository for pulling stateroot branch. +Currently not implemented, please prepare local repository instead. + +- repository -- path to repository with OSTree structure. +This path is relative to 'artifact' directory. + +- os -- os deployment name, as explained in: +https://ostree.readthedocs.io/en/latest/manual/deployment/ + +- branch -- branch of the repository to use for populating the image. + +Optional properties: + +- setup-fstab -- create '/etc/fstab' file for image + +- setup-kernel-cmdline -- add the information from the 'image-partition' +action to the configured commandline. + +- append-kernel-cmdline -- additional kernel command line arguments passed to kernel. +*/ +package actions + +import ( + "fmt" + "io" + "os" + "path" + "strings" + + "github.com/go-debos/debos" + ostree "github.com/sjoerdsimons/ostree-go/pkg/otbuiltin" +) + +type OstreeDeployAction struct { + debos.BaseAction `yaml:",inline"` + Repository string + RemoteRepository string "remote_repository" + Branch string + Os string + SetupFSTab bool `yaml:"setup-fstab"` + SetupKernelCmdline bool `yaml:"setup-kernel-cmdline"` + AppendKernelCmdline string `yaml:"append-kernel-cmdline"` +} + +func NewOstreeDeployAction() *OstreeDeployAction { + ot := &OstreeDeployAction{SetupFSTab: true, SetupKernelCmdline: true} + ot.Description = "Deploying from ostree" + return ot +} + +func (ot *OstreeDeployAction) setupFSTab(deployment *ostree.Deployment, context *debos.DebosContext) error { + deploymentDir := fmt.Sprintf("ostree/deploy/%s/deploy/%s.%d", + deployment.Osname(), deployment.Csum(), deployment.Deployserial()) + + etcDir := path.Join(context.Rootdir, deploymentDir, "etc") + + err := os.Mkdir(etcDir, 0755) + if err != nil && !os.IsExist(err) { + return err + } + + dst, err := os.OpenFile(path.Join(etcDir, "fstab"), os.O_WRONLY|os.O_CREATE, 0755) + defer dst.Close() + if err != nil { + return err + } + + _, err = io.Copy(dst, &context.ImageFSTab) + + return err +} + +func (ot *OstreeDeployAction) Run(context *debos.DebosContext) error { + ot.LogStart() + + // This is to handle cases there we didn't partition an image + if len(context.ImageMntDir) != 0 { + /* First deploy the current rootdir to the image so it can seed e.g. + * bootloader configuration */ + err := debos.Command{}.Run("Deploy to image", "cp", "-a", context.Rootdir+"/.", context.ImageMntDir) + if err != nil { + return fmt.Errorf("rootfs deploy failed: %v", err) + } + context.Rootdir = context.ImageMntDir + } + + repoPath := "file://" + path.Join(context.Artifactdir, ot.Repository) + + sysroot := ostree.NewSysroot(context.Rootdir) + err := sysroot.InitializeFS() + if err != nil { + return err + } + + err = sysroot.InitOsname(ot.Os, nil) + if err != nil { + return err + } + + /* HACK: Getting the repository form the sysroot gets ostree confused on + * whether it should configure /etc/ostree or the repo configuration, + so reopen by hand */ + /* dstRepo, err := sysroot.Repo(nil) */ + dstRepo, err := ostree.OpenRepo(path.Join(context.Rootdir, "ostree/repo")) + if err != nil { + return err + } + + /* FIXME: add support for gpg signing commits so this is no longer needed */ + opts := ostree.RemoteOptions{NoGpgVerify: true} + err = dstRepo.RemoteAdd("origin", ot.RemoteRepository, opts, nil) + if err != nil { + return err + } + + var options ostree.PullOptions + options.OverrideRemoteName = "origin" + options.Refs = []string{ot.Branch} + + err = dstRepo.PullWithOptions(repoPath, options, nil, nil) + if err != nil { + return err + } + + /* Required by ostree to make sure a bunch of information was pulled in */ + sysroot.Load(nil) + + revision, err := dstRepo.ResolveRev(ot.Branch, false) + if err != nil { + return err + } + + var kargs []string + if ot.SetupKernelCmdline { + kargs = append(kargs, context.ImageKernelRoot) + } + + if ot.AppendKernelCmdline != "" { + s := strings.Split(ot.AppendKernelCmdline, " ") + kargs = append(kargs, s...) + } + + origin := sysroot.OriginNewFromRefspec("origin:" + ot.Branch) + deployment, err := sysroot.DeployTree(ot.Os, revision, origin, nil, kargs, nil) + if err != nil { + return err + } + + if ot.SetupFSTab { + err = ot.setupFSTab(deployment, context) + if err != nil { + return err + } + } + + err = sysroot.SimpleWriteDeployment(ot.Os, deployment, nil, 0, nil) + if err != nil { + return err + } + + return nil +} diff --git a/actions/overlay_action.go b/actions/overlay_action.go new file mode 100644 index 0000000..f17ecb3 --- /dev/null +++ b/actions/overlay_action.go @@ -0,0 +1,67 @@ +/* +Overlay Action + +Recursive copy of directory or file to target filesystem. + +Yaml syntax: + - action: overlay + origin: name + source: directory + destination: directory + +Mandatory properties: + +- source -- relative path to the directory or file located in path referenced by `origin`. +In case if this property is absent then pure path referenced by 'origin' will be used. + +Optional properties: + +- origin -- reference to named file or directory. + +- destination -- absolute path in the target rootfs where 'source' will be copied. +All existing files will be overwritten. +If destination isn't set '/' of the rootfs will be usedi. +*/ +package actions + +import ( + "fmt" + "path" + + "github.com/go-debos/debos" +) + +type OverlayAction struct { + debos.BaseAction `yaml:",inline"` + Origin string // origin of overlay, here the export from other action may be used + Source string // external path there overlay is + Destination string // path inside of rootfs +} + +func (overlay *OverlayAction) Verify(context *debos.DebosContext) error { + if _, err := debos.RestrictedPath(context.Rootdir, overlay.Destination); err != nil { + return err + } + return nil +} + +func (overlay *OverlayAction) Run(context *debos.DebosContext) error { + overlay.LogStart() + origin := context.RecipeDir + + //Trying to get a filename from exports first + if len(overlay.Origin) > 0 { + var found bool + if origin, found = context.Origins[overlay.Origin]; !found { + return fmt.Errorf("Origin not found '%s'", overlay.Origin) + } + } + + sourcedir := path.Join(origin, overlay.Source) + destination, err := debos.RestrictedPath(context.Rootdir, overlay.Destination) + if err != nil { + return err + } + + return debos.CopyTree(sourcedir, destination) +} diff --git a/actions/pack_action.go b/actions/pack_action.go new file mode 100644 index 0000000..a90cb1d --- /dev/null +++ b/actions/pack_action.go @@ -0,0 +1,39 @@ +/* +Pack Action + +Create tarball with filesystem. + +Yaml syntax: + - action: pack + file: filename.ext + compression: gz + +Mandatory properties: + +- file -- name of the output tarball. + +- compression -- compression type to use. Only 'gz' is supported at the moment. + +*/ +package actions + +import ( + "log" + "path" + + "github.com/go-debos/debos" +) + +type PackAction struct { + debos.BaseAction `yaml:",inline"` + Compression string + File string +} + +func (pf *PackAction) Run(context *debos.DebosContext) error { + pf.LogStart() + outfile := path.Join(context.Artifactdir, pf.File) + + log.Printf("Compression to %s\n", outfile) + return debos.Command{}.Run("Packing", "tar", "czf", outfile, "-C", context.Rootdir, ".") +} diff --git a/actions/raw_action.go b/actions/raw_action.go new file mode 100644 index 0000000..d3e66fe --- /dev/null +++ b/actions/raw_action.go @@ -0,0 +1,130 @@ +/* +Raw Action + +Directly write a file to the output image at a given offset. +This is typically useful for bootloaders. + +Yaml syntax: + - action: raw + origin: name + source: filename + offset: bytes + +Mandatory properties: + +- origin -- reference to named file or directory. + +- source -- the name of file located in 'origin' to be written into the output image. + +Optional properties: + +- offset -- offset in bytes for output image file. +It is possible to use internal templating mechanism of debos to calculate offset +with sectors (512 bytes) instead of bytes, for instance: '{{ sector 256 }}'. +The default value is zero. + +- partition -- named partition to write to +*/ +package actions + +import ( + "errors" + "fmt" + "io/ioutil" + "log" + "os" + "path" + "strconv" + + "github.com/go-debos/debos" +) + +type RawAction struct { + debos.BaseAction `yaml:",inline"` + Origin string // there the source comes from + Offset string + Source string // relative path inside of origin + Path string // deprecated option (for backward compatibility) + Partition string // Partition to write otherwise full image +} + +func (raw *RawAction) checkDeprecatedSyntax() error { + + // New syntax is based on 'origin' and 'source' + // Check if we do not mix new and old syntax + // TODO: remove deprecated syntax verification + if len(raw.Path) > 0 { + // Deprecated syntax based on 'source' and 'path' + log.Printf("Usage of 'source' and 'path' properties is deprecated.") + log.Printf("Please use 'origin' and 'source' properties.") + if len(raw.Origin) > 0 { + return errors.New("Can't mix 'origin' and 'path'(deprecated option) properties") + } + if len(raw.Source) == 0 { + return errors.New("'source' and 'path' properties can't be empty") + } + // Switch to new syntax + raw.Origin = raw.Source + raw.Source = raw.Path + raw.Path = "" + } + return nil +} + +func (raw *RawAction) Verify(context *debos.DebosContext) error { + if err := raw.checkDeprecatedSyntax(); err != nil { + return err + } + + if len(raw.Origin) == 0 || len(raw.Source) == 0 { + return errors.New("'origin' and 'source' properties can't be empty") + } + + return nil +} + +func (raw *RawAction) Run(context *debos.DebosContext) error { + raw.LogStart() + origin, found := context.Origins[raw.Origin] + if !found { + return fmt.Errorf("Origin `%s` doesn't exist\n", raw.Origin) + } + s := path.Join(origin, raw.Source) + content, err := ioutil.ReadFile(s) + + if err != nil { + return fmt.Errorf("Failed to read %s", s) + } + + var devicePath string + if raw.Partition != "" { + for _, p := range context.ImagePartitions { + if p.Name == raw.Partition { + devicePath = p.DevicePath + break + } + } + + if devicePath == "" { + return fmt.Errorf("Failed to find partition named %s", raw.Partition) + } + } else { + devicePath = context.Image + } + + target, err := os.OpenFile(devicePath, os.O_WRONLY, 0) + if err != nil { + return fmt.Errorf("Failed to open %s: %v", devicePath, err) + } + + offset, err := strconv.ParseInt(raw.Offset, 0, 64) + if err != nil { + return fmt.Errorf("Couldn't parse offset %v", err) + } + bytes, err := target.WriteAt(content, offset) + if bytes != len(content) { + return errors.New("Couldn't write complete data") + } + + return nil +} diff --git a/actions/run_action.go b/actions/run_action.go new file mode 100644 index 0000000..ad374d8 --- /dev/null +++ b/actions/run_action.go @@ -0,0 +1,132 @@ +/* +Run Action + +Allows to run any available command or script in the filesystem or +in host environment. + +Yaml syntax: + - action: run + chroot: bool + postprocess: bool + script: script name + command: command line + +Properties 'command' and 'script' are mutually exclusive. + +- command -- command with arguments; the command expected to be accessible in +host's or chrooted environment -- depending on 'chroot' property. + +- script -- script with arguments; script must be located in recipe directory. + +Optional properties: + +- chroot -- run script or command in target filesystem if set to true. +In other case the command or script is executed within the build process, with +access to the filesystem and the image. In both cases it is run with root privileges. + +- postprocess -- if set script or command is executed after all other commands and +has access to the image file. + + +Properties 'chroot' and 'postprocess' are mutually exclusive. +*/ +package actions + +import ( + "errors" + "github.com/go-debos/fakemachine" + "path" + "strings" + + "github.com/go-debos/debos" +) + +type RunAction struct { + debos.BaseAction `yaml:",inline"` + Chroot bool + PostProcess bool + Script string + Command string +} + +func (run *RunAction) Verify(context *debos.DebosContext) error { + if run.PostProcess && run.Chroot { + return errors.New("Cannot run postprocessing in the chroot") + } + return nil +} + +func (run *RunAction) PreMachine(context *debos.DebosContext, m *fakemachine.Machine, + args *[]string) error { + + if run.Script == "" { + return nil + } + + run.Script = debos.CleanPathAt(run.Script, context.RecipeDir) + // Expect we have no blank spaces in path + scriptpath := strings.Split(run.Script, " ") + + if !run.PostProcess { + m.AddVolume(path.Dir(scriptpath[0])) + } + + return nil +} + +func (run *RunAction) doRun(context debos.DebosContext) error { + run.LogStart() + var cmdline []string + var label string + var cmd debos.Command + + if run.Chroot { + cmd = debos.NewChrootCommandForContext(context) + } else { + cmd = debos.Command{} + } + + if run.Script != "" { + script := strings.SplitN(run.Script, " ", 2) + script[0] = debos.CleanPathAt(script[0], context.RecipeDir) + if run.Chroot { + scriptpath := path.Dir(script[0]) + cmd.AddBindMount(scriptpath, "/script") + script[0] = strings.Replace(script[0], scriptpath, "/script", 1) + } + cmdline = []string{strings.Join(script, " ")} + label = path.Base(run.Script) + } else { + cmdline = []string{run.Command} + label = run.Command + } + + // Command/script with options passed as single string + cmdline = append([]string{"sh", "-c"}, cmdline...) + + if !run.PostProcess { + if !run.Chroot { + cmd.AddEnvKey("ROOTDIR", context.Rootdir) + } + if context.Image != "" { + cmd.AddEnvKey("IMAGE", context.Image) + } + } + + return cmd.Run(label, cmdline...) +} + +func (run *RunAction) Run(context *debos.DebosContext) error { + if run.PostProcess { + /* This runs in postprocessing instead */ + return nil + } + return run.doRun(*context) +} + +func (run *RunAction) PostMachine(context debos.DebosContext) error { + if !run.PostProcess { + return nil + } + return run.doRun(context) +} diff --git a/actions/unpack_action.go b/actions/unpack_action.go new file mode 100644 index 0000000..d4993b1 --- /dev/null +++ b/actions/unpack_action.go @@ -0,0 +1,100 @@ +/* +Unpack Action + +Unpack files from archive to the filesystem. +Useful for creating target rootfs from saved tarball with prepared file structure. + +Only (compressed) tar archives are supported currently. + +Yaml syntax: + - action: unpack + origin: name + file: file.ext + compression: gz + +Mandatory properties: + +- file -- archive's file name. It is possible to skip this property if 'origin' +referenced to downloaded file. + +One of the mandatory properties may be omitted with limitations mentioned above. +It is expected to find archive with name pointed in `file` property inside of `origin` in case if both properties are used. + +Optional properties: + +- origin -- reference to a named file or directory. +The default value is 'artifacts' directory in case if this property is omitted. + +- compression -- optional hint for unpack allowing to use proper compression method. + +Currently only 'gz', bzip2' and 'xz' compression types are supported. +If not provided an attempt to autodetect the compression type will be done. +*/ +package actions + +import ( + "fmt" + "github.com/go-debos/debos" +) + +type UnpackAction struct { + debos.BaseAction `yaml:",inline"` + Compression string + Origin string + File string +} + +func (pf *UnpackAction) Verify(context *debos.DebosContext) error { + + if len(pf.Origin) == 0 && len(pf.File) == 0 { + return fmt.Errorf("Filename can't be empty. Please add 'file' and/or 'origin' property.") + } + + archive, err := debos.NewArchive(pf.File) + if err != nil { + return err + } + if len(pf.Compression) > 0 { + if archive.Type() != debos.Tar { + return fmt.Errorf("Option 'compression' is supported for Tar archives only.") + } + if err := archive.AddOption("tarcompression", pf.Compression); err != nil { + return fmt.Errorf("'%s': %s", pf.File, err) + } + } + + return nil +} + +func (pf *UnpackAction) Run(context *debos.DebosContext) error { + pf.LogStart() + var origin string + + if len(pf.Origin) > 0 { + var found bool + //Trying to get a filename from origins first + origin, found = context.Origins[pf.Origin] + if !found { + return fmt.Errorf("Origin not found '%s'", pf.Origin) + } + } else { + origin = context.Artifactdir + } + + infile, err := debos.RestrictedPath(origin, pf.File) + if err != nil { + return err + } + + archive, err := debos.NewArchive(infile) + if err != nil { + return err + } + if len(pf.Compression) > 0 { + if err := archive.AddOption("tarcompression", pf.Compression); err != nil { + return err + } + } + + return archive.Unpack(context.Rootdir) +} |