diff options
-rw-r--r-- | .gitignore | 51 | ||||
-rw-r--r-- | LICENSE | 201 | ||||
-rw-r--r-- | README.md | 13 | ||||
-rw-r--r-- | TODO | 56 | ||||
-rw-r--r-- | action.go | 67 | ||||
-rw-r--r-- | actions/actions_doc.go | 6 | ||||
-rw-r--r-- | actions/apt_action.go | 61 | ||||
-rw-r--r-- | actions/debootstrap_action.go | 146 | ||||
-rw-r--r-- | actions/download_action.go | 168 | ||||
-rw-r--r-- | actions/filesystem_deploy_action.go | 131 | ||||
-rw-r--r-- | actions/image_partition_action.go | 432 | ||||
-rw-r--r-- | actions/ostree_commit_action.go | 84 | ||||
-rw-r--r-- | actions/ostree_deploy_action.go | 183 | ||||
-rw-r--r-- | actions/overlay_action.go | 67 | ||||
-rw-r--r-- | actions/pack_action.go | 39 | ||||
-rw-r--r-- | actions/raw_action.go | 130 | ||||
-rw-r--r-- | actions/run_action.go | 132 | ||||
-rw-r--r-- | actions/unpack_action.go | 100 | ||||
-rw-r--r-- | archiver.go | 217 | ||||
-rw-r--r-- | archiver_test.go | 157 | ||||
-rw-r--r-- | cmd/debos/debos.go | 219 | ||||
-rw-r--r-- | commands.go | 203 | ||||
-rw-r--r-- | commands_test.go | 9 | ||||
-rw-r--r-- | debug.go | 31 | ||||
-rw-r--r-- | filesystem.go | 105 | ||||
-rw-r--r-- | net.go | 45 | ||||
-rw-r--r-- | os.go | 74 | ||||
-rw-r--r-- | recipe/recipe.go | 175 | ||||
-rw-r--r-- | recipe/recipe_test.go | 182 |
29 files changed, 3484 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..32eae19 --- /dev/null +++ b/.gitignore @@ -0,0 +1,51 @@ + +# Created by https://www.gitignore.io/api/vim,linux,go + +### Go ### +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 +.glide/ + +# Golang project vendor packages which should be ignored +vendor/ + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### Vim ### +# swap +[._]*.s[a-v][a-z] +[._]*.sw[a-p] +[._]s[a-v][a-z] +[._]sw[a-p] +# session +Session.vim +# temporary +.netrwhist +# auto-generated tag files +tags + +# End of https://www.gitignore.io/api/vim,linux,go @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..3defeeb --- /dev/null +++ b/README.md @@ -0,0 +1,13 @@ +debos +===== + +Debian OS builder. debos is a tool to make creation of various debian based os +"images" simpler. While most other tools focus on specific use-case, debos is +more meant as a toolchain to make comon actions trivial while providing enough +rope to do whatever tweaking that might be required behind the scene. + +debos expects a yaml file as input, syntax description can be found at: + https://godoc.org/github.com/go-debos/debos/actions + +and examples are to be found at: + https://github.com/go-debos/debos-recipes @@ -0,0 +1,56 @@ +Before 1.0 +* Rename to debos + +Potentially After 1.0 + +* Make logging more consistent +* Have a mode to output the final yaml after templating and then exit (dry-run?) + + +* Documentation of all actions and overall concepts! +* Tests, individual actions are mostly testable.. + +* Support having devices as image files (fakemachine should handle it ok) + +* Create a standard docker image with debos to push to docker hub for easy + image creation + +* Template function to include other files +* Dependency system between recipes ? + +* Warn on unknown yaml fields? + +* New actions: +** Create a manifest from install debian packages +** Install all dbgsym packages for a given rootfs +** Install all dev package for installed libraries +** Generalize the ostree conversion into an action? +** Ostree checkout for incremental fast updates? +** Action to get remote content which can be copied in +*** Download tarball from http and unpack (same content) +*** Download git tree (e.g. to get rpi firmware) +*** Download .deb (e.g. u-boot to raw write rather then install) + +* Check what triggers ostree changes and try to minimize those + +* Control passwd & group contents as that can be problematic with ostree + +* Rewrite debootstrap in pure go to add a bunch of smarts (e.g. parallel + downloads, local caching etc) + +* Rewrite pack/unpack in pure go and support more formats + + +* Make actions using (host) commands check their existance early + + +* Ensure we copy xattrs? + +* Fix race in qemu-helper (if qemu-user-static gets installed in the system + chroot things will get confused) + +* Do shell compatible parsing of script: argument to run actions and + environment substitution + +* Support gpg signing ostree commits + diff --git a/action.go b/action.go new file mode 100644 index 0000000..f7c4062 --- /dev/null +++ b/action.go @@ -0,0 +1,67 @@ +package debos + +import ( + "bytes" + "github.com/go-debos/fakemachine" + "log" +) + +// Mapping from partition name as configured in the image-partition action to +// device path for usage by other actions +type Partition struct { + Name string + DevicePath string +} + +type DebosContext struct { + Scratchdir string + Rootdir string + Artifactdir string + Downloaddir string + Image string + ImagePartitions []Partition + ImageMntDir string + ImageFSTab bytes.Buffer // Fstab as per partitioning + ImageKernelRoot string // Kernel cmdline root= snippet for the / of the image + RecipeDir string + Architecture string + DebugShell string + Origins map[string]string +} + +type Action interface { + /* FIXME verify should probably be prepare or somesuch */ + Verify(context *DebosContext) error + PreMachine(context *DebosContext, m *fakemachine.Machine, args *[]string) error + PreNoMachine(context *DebosContext) error + Run(context *DebosContext) error + Cleanup(context DebosContext) error + PostMachine(context DebosContext) error + String() string +} + +type BaseAction struct { + Action string + Description string +} + +func (b *BaseAction) LogStart() { + log.Printf("==== %s ====\n", b) +} + +func (b *BaseAction) Verify(context *DebosContext) error { return nil } +func (b *BaseAction) PreMachine(context *DebosContext, + m *fakemachine.Machine, + args *[]string) error { + return nil +} +func (b *BaseAction) PreNoMachine(context *DebosContext) error { return nil } +func (b *BaseAction) Run(context *DebosContext) error { return nil } +func (b *BaseAction) Cleanup(context DebosContext) error { return nil } +func (b *BaseAction) PostMachine(context DebosContext) error { return nil } +func (b *BaseAction) String() string { + if b.Description == "" { + return b.Action + } + return b.Description +} diff --git a/actions/actions_doc.go b/actions/actions_doc.go new file mode 100644 index 0000000..7ebd532 --- /dev/null +++ b/actions/actions_doc.go @@ -0,0 +1,6 @@ +// Copyright 2017, Collabora Ltd. + +/* +Package 'actions' implements 'debos' modules used for OS creation. +*/ +package actions diff --git a/actions/apt_action.go b/actions/apt_action.go new file mode 100644 index 0000000..681c069 --- /dev/null +++ b/actions/apt_action.go @@ -0,0 +1,61 @@ +/* +Apt Action + +Install packages and their dependencies to the target rootfs with 'apt'. + +Yaml syntax: + - action: apt + recommends: bool + packages: + - package1 + - package2 + +Mandatory properties: + +- packages -- list of packages to install + +Optional properties: + +- recommends -- boolean indicating if suggested packages will be installed +*/ +package actions + +import ( + "github.com/go-debos/debos" +) + +type AptAction struct { + debos.BaseAction `yaml:",inline"` + Recommends bool + Packages []string +} + +func (apt *AptAction) Run(context *debos.DebosContext) error { + apt.LogStart() + aptOptions := []string{"apt-get", "-y"} + + if !apt.Recommends { + aptOptions = append(aptOptions, "--no-install-recommends") + } + + aptOptions = append(aptOptions, "install") + aptOptions = append(aptOptions, apt.Packages...) + + c := debos.NewChrootCommandForContext(*context) + c.AddEnv("DEBIAN_FRONTEND=noninteractive") + + err := c.Run("apt", "apt-get", "update") + if err != nil { + return err + } + err = c.Run("apt", aptOptions...) + if err != nil { + return err + } + err = c.Run("apt", "apt-get", "clean") + if err != nil { + return err + } + + return nil +} diff --git a/actions/debootstrap_action.go b/actions/debootstrap_action.go new file mode 100644 index 0000000..a53d153 --- /dev/null +++ b/actions/debootstrap_action.go @@ -0,0 +1,146 @@ +/* +Debootstrap Action + +Construct the target rootfs with debootstrap tool. + +Yaml syntax: + - action: debootstrap + mirror: URL + suite: "name" + components: <list of components> + variant: "name" + keyring-package: + +Mandatory properties: + +- suite -- release code name or symbolic name (e.g. "stable") + +Optional properties: + +- mirror -- URL with Debian-compatible repository + +- variant -- name of the bootstrap script variant to use + +- components -- list of components to use for packages selection. +Example: + components: [ main, contrib ] + +- keyring-package -- keyring for packages validation. Currently ignored. + +- merged-usr -- use merged '/usr' filesystem, true by default. +*/ +package actions + +import ( + "fmt" + "io" + "os" + "path" + "strings" + + "github.com/go-debos/debos" +) + +type DebootstrapAction struct { + debos.BaseAction `yaml:",inline"` + Suite string + Mirror string + Variant string + KeyringPackage string `yaml:"keyring-package"` + Components []string + MergedUsr bool `yaml:"merged-usr"` +} + +func NewDebootstrapAction() *DebootstrapAction { + d := DebootstrapAction{} + // Use filesystem with merged '/usr' by default + d.MergedUsr = true + return &d + +} + +func (d *DebootstrapAction) RunSecondStage(context debos.DebosContext) error { + cmdline := []string{ + "/debootstrap/debootstrap", + "--no-check-gpg", + "--second-stage"} + + if d.Components != nil { + s := strings.Join(d.Components, ",") + cmdline = append(cmdline, fmt.Sprintf("--components=%s", s)) + } + + c := debos.NewChrootCommandForContext(context) + // Can't use nspawn for debootstrap as it wants to create device nodes + c.ChrootMethod = debos.CHROOT_METHOD_CHROOT + + return c.Run("Debootstrap (stage 2)", cmdline...) +} + +func (d *DebootstrapAction) Run(context *debos.DebosContext) error { + d.LogStart() + cmdline := []string{"debootstrap", "--no-check-gpg"} + + if d.MergedUsr { + cmdline = append(cmdline, "--merged-usr") + } + + if d.KeyringPackage != "" { + cmdline = append(cmdline, fmt.Sprintf("--keyring=%s", d.KeyringPackage)) + } + + if d.Components != nil { + s := strings.Join(d.Components, ",") + cmdline = append(cmdline, fmt.Sprintf("--components=%s", s)) + } + + /* FIXME drop the hardcoded amd64 assumption" */ + foreign := context.Architecture != "amd64" + + if foreign { + cmdline = append(cmdline, "--foreign") + cmdline = append(cmdline, fmt.Sprintf("--arch=%s", context.Architecture)) + + } + + if d.Variant != "" { + cmdline = append(cmdline, fmt.Sprintf("--variant=%s", d.Variant)) + } + + cmdline = append(cmdline, d.Suite) + cmdline = append(cmdline, context.Rootdir) + cmdline = append(cmdline, d.Mirror) + cmdline = append(cmdline, "/usr/share/debootstrap/scripts/unstable") + + err := debos.Command{}.Run("Debootstrap", cmdline...) + + if err != nil { + return err + } + + if foreign { + err = d.RunSecondStage(*context) + if err != nil { + return err + } + } + + /* HACK */ + srclist, err := os.OpenFile(path.Join(context.Rootdir, "etc/apt/sources.list"), + os.O_RDWR|os.O_CREATE, 0755) + if err != nil { + return err + } + _, err = io.WriteString(srclist, fmt.Sprintf("deb %s %s %s\n", + d.Mirror, + d.Suite, + strings.Join(d.Components, " "))) + if err != nil { + return err + } + srclist.Close() + + c := debos.NewChrootCommandForContext(*context) + + return c.Run("apt clean", "/usr/bin/apt-get", "clean") +} diff --git a/actions/download_action.go b/actions/download_action.go new file mode 100644 index 0000000..a4099e7 --- /dev/null +++ b/actions/download_action.go @@ -0,0 +1,168 @@ +/* +Download Action + +Download a single file from Internet and unpack it in place if needed. + +Yaml syntax: + - action: download + url: http://example.domain/path/filename.ext + name: firmware + filename: output_name + unpack: bool + compression: gz + +Mandatory properties: + +- url -- URL to an object for download + +- name -- string which allow to use downloaded object in other actions +via 'origin' property. If 'unpack' property is set to 'true' name will +refer to temporary directory with extracted content. + +Optional properties: + +- filename -- use this property as the name for saved file. Useful if URL does not +contain file name in path, for example it is possible to download files from URLs without path part. + +- unpack -- hint for action to extract all files from downloaded archive. +See the 'Unpack' action for more information. + +- compression -- optional hint for unpack allowing to use proper compression method. +See the 'Unpack' action for more information. +*/ +package actions + +import ( + "fmt" + "github.com/go-debos/debos" + "net/url" + "path" +) + +type DownloadAction struct { + debos.BaseAction `yaml:",inline"` + Url string // URL for downloading + Filename string // File name, overrides the name from URL. + Unpack bool // Unpack downloaded file to directory dedicated for download + Compression string // compression type + Name string // exporting path to file or directory(in case of unpack) +} + +// validateUrl checks if supported URL is passed from recipe +// Return: +// - parsed URL +// - nil in case of success +func (d *DownloadAction) validateUrl() (*url.URL, error) { + + url, err := url.Parse(d.Url) + if err != nil { + return url, err + } + + switch url.Scheme { + case "http", "https": + // Supported scheme + default: + return url, fmt.Errorf("Unsupported URL is provided: '%s'", url.String()) + } + + return url, nil +} + +func (d *DownloadAction) validateFilename(context *debos.DebosContext, url *url.URL) (filename string, err error) { + if len(d.Filename) == 0 { + // Trying to guess the name from URL Path + filename = path.Base(url.Path) + } else { + filename = path.Base(d.Filename) + } + if len(filename) == 0 { + return "", fmt.Errorf("Incorrect filename is provided for '%s'", d.Url) + } + filename = path.Join(context.Scratchdir, filename) + return filename, nil +} + +func (d *DownloadAction) archive(filename string) (debos.Archive, error) { + archive, err := debos.NewArchive(filename) + if err != nil { + return archive, err + } + switch archive.Type() { + case debos.Tar: + if len(d.Compression) > 0 { + if err := archive.AddOption("tarcompression", d.Compression); err != nil { + return archive, err + } + } + default: + } + return archive, nil +} + +func (d *DownloadAction) Verify(context *debos.DebosContext) error { + var filename string + + if len(d.Name) == 0 { + return fmt.Errorf("Property 'name' is mandatory for download action\n") + } + + url, err := d.validateUrl() + if err != nil { + return err + } + filename, err = d.validateFilename(context, url) + if err != nil { + return err + } + if d.Unpack == true { + if _, err := d.archive(filename); err != nil { + return err + } + } + return nil +} + +func (d *DownloadAction) Run(context *debos.DebosContext) error { + var filename string + d.LogStart() + + url, err := d.validateUrl() + if err != nil { + return err + } + + filename, err = d.validateFilename(context, url) + if err != nil { + return err + } + originPath := filename + + switch url.Scheme { + case "http", "https": + err := debos.DownloadHttpUrl(url.String(), filename) + if err != nil { + return err + } + default: + return fmt.Errorf("Unsupported URL is provided: '%s'", url.String()) + } + + if d.Unpack == true { + archive, err := d.archive(filename) + if err != nil { + return err + } + + targetdir := filename + ".d" + err = archive.RelaxedUnpack(targetdir) + if err != nil { + return err + } + originPath = targetdir + } + + context.Origins[d.Name] = originPath + + return nil +} diff --git a/actions/filesystem_deploy_action.go b/actions/filesystem_deploy_action.go new file mode 100644 index 0000000..bb89a83 --- /dev/null +++ b/actions/filesystem_deploy_action.go @@ -0,0 +1,131 @@ +/* +FilesystemDeploy Action + +Deploy prepared root filesystem to output image. This action requires +'image-partition' action to be executed before it. + +Yaml syntax: + - action: filesystem-deploy + setup-fstab: bool + setup-kernel-cmdline: bool + +Optional properties: + +- setup-fstab -- generate '/etc/fstab' file according to information provided +by 'image-partition' action. By default is 'true'. + +- setup-kernel-cmdline -- add location of root partition to '/etc/kernel/cmdline' +file on target image. By default is 'true'. +*/ +package actions + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path" + "strings" + + "github.com/go-debos/debos" +) + +type FilesystemDeployAction struct { + debos.BaseAction `yaml:",inline"` + SetupFSTab bool `yaml:"setup-fstab"` + SetupKernelCmdline bool `yaml:"setup-kernel-cmdline"` +} + +func NewFilesystemDeployAction() *FilesystemDeployAction { + fd := &FilesystemDeployAction{SetupFSTab: true, SetupKernelCmdline: true} + fd.Description = "Deploying filesystem" + + return fd +} + +func (fd *FilesystemDeployAction) setupFSTab(context *debos.DebosContext) error { + if context.ImageFSTab.Len() == 0 { + return errors.New("Fstab not generated, missing image-partition action?") + } + + log.Print("Setting up fstab") + + err := os.MkdirAll(path.Join(context.Rootdir, "etc"), 0755) + if err != nil { + return fmt.Errorf("Couldn't create etc in image: %v", err) + } + + fstab := path.Join(context.Rootdir, "etc/fstab") + f, err := os.OpenFile(fstab, os.O_RDWR|os.O_CREATE, 0755) + + if err != nil { + return fmt.Errorf("Couldn't open fstab: %v", err) + } + + _, err = io.Copy(f, &context.ImageFSTab) + + if err != nil { + return fmt.Errorf("Couldn't write fstab: %v", err) + } + f.Close() + + return nil +} + +func (fd *FilesystemDeployAction) setupKernelCmdline(context *debos.DebosContext) error { + log.Print("Setting up /etc/kernel/cmdline") + + err := os.MkdirAll(path.Join(context.Rootdir, "etc", "kernel"), 0755) + if err != nil { + return fmt.Errorf("Couldn't create etc/kernel in image: %v", err) + } + path := path.Join(context.Rootdir, "etc/kernel/cmdline") + current, _ := ioutil.ReadFile(path) + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0755) + + if err != nil { + log.Fatalf("Couldn't open kernel cmdline: %v", err) + } + + cmdline := fmt.Sprintf("%s %s\n", + strings.TrimSpace(string(current)), + context.ImageKernelRoot) + + _, err = f.WriteString(cmdline) + if err != nil { + return fmt.Errorf("Couldn't write kernel/cmdline: %v", err) + } + + f.Close() + return nil +} + +func (fd *FilesystemDeployAction) Run(context *debos.DebosContext) error { + fd.LogStart() + /* Copying files is actually silly hafd, one has to keep permissions, ACL's + * extended attribute, misc, other. Leave it to cp... + */ + err := debos.Command{}.Run("Deploy to image", "cp", "-a", context.Rootdir+"/.", context.ImageMntDir) + if err != nil { + return fmt.Errorf("rootfs deploy failed: %v", err) + } + context.Rootdir = context.ImageMntDir + context.Origins["filesystem"] = context.ImageMntDir + + if fd.SetupFSTab { + err = fd.setupFSTab(context) + if err != nil { + return err + } + } + if fd.SetupKernelCmdline { + err = fd.setupKernelCmdline(context) + if err != nil { + return err + } + } + + return nil +} diff --git a/actions/image_partition_action.go b/actions/image_partition_action.go new file mode 100644 index 0000000..5054e6d --- /dev/null +++ b/actions/image_partition_action.go @@ -0,0 +1,432 @@ +/* +ImagePartition Action + +This action creates an image file, partitions it and formats the filesystems. + +Yaml syntax: + - action: image-partition + imagename: image_name + imagesize: size + partitiontype: gpt + gpt_gap: offset + partitions: + <list of partitions> + mountpoints: + <list of mount points> + +Mandatory properties: + +- imagename -- the name of the image file. + +- imagesize -- generated image size in human-readable form, examples: 100MB, 1GB, etc. + +- partitiontype -- partition table type. Currently only 'gpt' and 'msdos' +partition tables are supported. + +- gpt_gap -- shifting GPT allow to use this gap for bootloaders, for example if +U-Boot intersects with original GPT placement. +Only works if parted supports an extra argument to mklabel to specify the gpt offset. + +- partitions -- list of partitions, at least one partition is needed. +Partition properties are described below. + +- mountpoints -- list of mount points for partitions. +Properties for mount points are described below. + +Yaml syntax for partitions: + + partitions: + - name: label + name: partition name + fs: filesystem + start: offset + end: offset + flags: list of flags + +Mandatory properties: + +- name -- is used for referencing named partition for mount points +configuration (below) and label the filesystem located on this partition. + +- fs -- filesystem type used for formatting. + +'none' fs type should be used for partition without filesystem. + +- start -- offset from beginning of the disk there the partition starts. + +- end -- offset from beginning of the disk there the partition ends. + +For 'start' and 'end' properties offset can be written in human readable +form -- '32MB', '1GB' or as disk percentage -- '100%'. + +Optional properties: + +- flags -- list of additional flags for partition compatible with parted(8) +'set' command. + +Yaml syntax for mount points: + + mountpoints: + - mountpoint: path + partition: partition label + options: list of options + +Mandatory properties: + +- partition -- partition name for mounting. + +- mountpoint -- path in the target root filesystem where the named partition +should be mounted. + +Optional properties: + +- options -- list of options to be added to appropriate entry in fstab file. + +Layout example for Raspberry PI 3: + + - action: image-partition + imagename: "debian-rpi3.img" + imagesize: 1GB + partitiontype: msdos + mountpoints: + - mountpoint: / + partition: root + - mountpoint: /boot/firmware + partition: firmware + options: [ x-systemd.automount ] + partitions: + - name: firmware + fs: vfat + start: 0% + end: 64MB + - name: root + fs: ext4 + start: 64MB + end: 100% + flags: [ boot ] +*/ +package actions + +import ( + "errors" + "fmt" + "github.com/docker/go-units" + "github.com/go-debos/fakemachine" + "log" + "os" + "os/exec" + "path" + "strings" + "syscall" + + "github.com/go-debos/debos" +) + +type Partition struct { + number int + Name string + Start string + End string + FS string + Flags []string + FSUUID string +} + +type Mountpoint struct { + Mountpoint string + Partition string + Options []string + part *Partition +} + +type ImagePartitionAction struct { + debos.BaseAction `yaml:",inline"` + ImageName string + ImageSize string + PartitionType string + GptGap string "gpt_gap" + Partitions []Partition + Mountpoints []Mountpoint + size int64 + usingLoop bool +} + +func (i *ImagePartitionAction) generateFSTab(context *debos.DebosContext) error { + context.ImageFSTab.Reset() + + for _, m := range i.Mountpoints { + options := []string{"defaults"} + options = append(options, m.Options...) + if m.part.FSUUID == "" { + return fmt.Errorf("Missing fs UUID for partition %s!?!", m.part.Name) + } + context.ImageFSTab.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0\t0\n", + m.part.FSUUID, m.Mountpoint, m.part.FS, + strings.Join(options, ","))) + } + + return nil +} + +func (i *ImagePartitionAction) generateKernelRoot(context *debos.DebosContext) error { + for _, m := range i.Mountpoints { + if m.Mountpoint == "/" { + if m.part.FSUUID == "" { + return errors.New("No fs UUID for root partition !?!") + } + context.ImageKernelRoot = fmt.Sprintf("root=UUID=%s", m.part.FSUUID) + break + } + } + + return nil +} + +func (i ImagePartitionAction) getPartitionDevice(number int, context debos.DebosContext) string { + suffix := "p" + /* Check partition naming first: if used 'by-id'i naming convention */ + if strings.Contains(context.Image, "/disk/by-id/") { + suffix = "-part" + } + + /* If the iamge device has a digit as the last character, the partition + * suffix is p<number> else it's just <number> */ + last := context.Image[len(context.Image)-1] + if last >= '0' && last <= '9' { + return fmt.Sprintf("%s%s%d", context.Image, suffix, number) + } else { + return fmt.Sprintf("%s%d", context.Image, number) + } +} + +func (i ImagePartitionAction) PreMachine(context *debos.DebosContext, m *fakemachine.Machine, + args *[]string) error { + image, err := m.CreateImage(i.ImageName, i.size) + if err != nil { + return err + } + + context.Image = image + *args = append(*args, "--internal-image", image) + return nil +} + +func (i ImagePartitionAction) formatPartition(p *Partition, context debos.DebosContext) error { + label := fmt.Sprintf("Formatting partition %d", p.number) + path := i.getPartitionDevice(p.number, context) + + cmdline := []string{} + switch p.FS { + case "vfat": + cmdline = append(cmdline, "mkfs.vfat", "-n", p.Name) + case "btrfs": + // Force formatting to prevent failure in case if partition was formatted already + cmdline = append(cmdline, "mkfs.btrfs", "-L", p.Name, "-f") + case "none": + default: + cmdline = append(cmdline, fmt.Sprintf("mkfs.%s", p.FS), "-L", p.Name) + } + + if len(cmdline) != 0 { + cmdline = append(cmdline, path) + + cmd := debos.Command{} + if err := cmd.Run(label, cmdline...); err != nil { + return err + } + } + + if p.FS != "none" { + uuid, err := exec.Command("blkid", "-o", "value", "-s", "UUID", "-p", "-c", "none", path).Output() + if err != nil { + return fmt.Errorf("Failed to get uuid: %s", err) + } + p.FSUUID = strings.TrimSpace(string(uuid[:])) + } + + return nil +} + +func (i ImagePartitionAction) PreNoMachine(context *debos.DebosContext) error { + + img, err := os.OpenFile(i.ImageName, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + return fmt.Errorf("Couldn't open image file: %v", err) + } + + err = img.Truncate(i.size) + if err != nil { + return fmt.Errorf("Couldn't resize image file: %v", err) + } + + img.Close() + + loop, err := exec.Command("losetup", "-f", "--show", i.ImageName).Output() + if err != nil { + return fmt.Errorf("Failed to setup loop device") + } + context.Image = strings.TrimSpace(string(loop[:])) + i.usingLoop = true + + return nil +} + +func (i ImagePartitionAction) Run(context *debos.DebosContext) error { + i.LogStart() + + command := []string{"parted", "-s", context.Image, "mklabel", i.PartitionType} + if len(i.GptGap) > 0 { + command = append(command, i.GptGap) + } + err := debos.Command{}.Run("parted", command...) + if err != nil { + return err + } + for idx, _ := range i.Partitions { + p := &i.Partitions[idx] + var name string + if i.PartitionType == "gpt" { + name = p.Name + } else { + name = "primary" + } + + command := []string{"parted", "-a", "none", "-s", "--", context.Image, "mkpart", name} + switch p.FS { + case "vfat": + command = append(command, "fat32") + case "none": + default: + command = append(command, p.FS) + } + command = append(command, p.Start, p.End) + + err = debos.Command{}.Run("parted", command...) + if err != nil { + return err + } + + if p.Flags != nil { + for _, flag := range p.Flags { + err = debos.Command{}.Run("parted", "parted", "-s", context.Image, "set", + fmt.Sprintf("%d", p.number), flag, "on") + if err != nil { + return err + } + } + } + + devicePath := i.getPartitionDevice(p.number, *context) + // Give a chance for udevd to create proper symlinks + err = debos.Command{}.Run("udevadm", "udevadm", "settle", "-t", "5", + "-E", devicePath) + if err != nil { + return err + } + + err = i.formatPartition(p, *context) + if err != nil { + return err + } + + context.ImagePartitions = append(context.ImagePartitions, + debos.Partition{p.Name, devicePath}) + } + + context.ImageMntDir = path.Join(context.Scratchdir, "mnt") + os.MkdirAll(context.ImageMntDir, 0755) + for _, m := range i.Mountpoints { + dev := i.getPartitionDevice(m.part.number, *context) + mntpath := path.Join(context.ImageMntDir, m.Mountpoint) + os.MkdirAll(mntpath, 0755) + err := syscall.Mount(dev, mntpath, m.part.FS, 0, "") + if err != nil { + return fmt.Errorf("%s mount failed: %v", m.part.Name, err) + } + } + + err = i.generateFSTab(context) + if err != nil { + return err + } + + err = i.generateKernelRoot(context) + if err != nil { + return err + } + + return nil +} + +func (i ImagePartitionAction) Cleanup(context debos.DebosContext) error { + for idx := len(i.Mountpoints) - 1; idx >= 0; idx-- { + m := i.Mountpoints[idx] + mntpath := path.Join(context.ImageMntDir, m.Mountpoint) + syscall.Unmount(mntpath, 0) + } + + if i.usingLoop { + exec.Command("losetup", "-d", context.Image).Run() + } + + return nil +} + +func (i *ImagePartitionAction) Verify(context *debos.DebosContext) error { + if len(i.GptGap) > 0 { + log.Println("WARNING: special version of parted is needed for 'gpt_gap' option") + if i.PartitionType != "gpt" { + return fmt.Errorf("gpt_gap property could be used only with 'gpt' label") + } + // Just check if it contains correct value + _, err := units.FromHumanSize(i.GptGap) + if err != nil { + return fmt.Errorf("Failed to parse GPT offset: %s", i.GptGap) + } + } + + num := 1 + for idx, _ := range i.Partitions { + p := &i.Partitions[idx] + p.number = num + num++ + if p.Name == "" { + return fmt.Errorf("Partition without a name") + } + if p.Start == "" { + return fmt.Errorf("Partition %s missing start", p.Name) + } + if p.End == "" { + return fmt.Errorf("Partition %s missing end", p.Name) + } + + switch p.FS { + case "fat32": + p.FS = "vfat" + case "": + return fmt.Errorf("Partition %s missing fs type", p.Name) + } + } + + for idx, _ := range i.Mountpoints { + m := &i.Mountpoints[idx] + for pidx, _ := range i.Partitions { + p := &i.Partitions[pidx] + if m.Partition == p.Name { + m.part = p + break + } + } + if m.part == nil { + return fmt.Errorf("Couldn't fount partition for %s", m.Mountpoint) + } + } + + size, err := units.FromHumanSize(i.ImageSize) + if err != nil { + return fmt.Errorf("Failed to parse image size: %s", i.ImageSize) + } + + i.size = size + return nil +} diff --git a/actions/ostree_commit_action.go b/actions/ostree_commit_action.go new file mode 100644 index 0000000..6d41b89 --- /dev/null +++ b/actions/ostree_commit_action.go @@ -0,0 +1,84 @@ +/* +OstreeCommit Action + +Create OSTree commit from rootfs. + +Yaml syntax: + - action: ostree-commit + repository: repository name + branch: branch name + subject: commit message + +Mandatory properties: + +- repository -- path to repository with OSTree structure; the same path is +used by 'ostree' tool with '--repo' argument. +This path is relative to 'artifact' directory. +Please keep in mind -- you will need a root privileges for 'bare' repository +type (https://ostree.readthedocs.io/en/latest/manual/repo/#repository-types-and-locations). + +- branch -- OSTree branch name that should be used for the commit. + +Optional properties: + +- subject -- one line message with commit description. +*/ +package actions + +import ( + "log" + "os" + "path" + + "github.com/go-debos/debos" + "github.com/sjoerdsimons/ostree-go/pkg/otbuiltin" +) + +type OstreeCommitAction struct { + debos.BaseAction `yaml:",inline"` + Repository string + Branch string + Subject string + Command string +} + +func emptyDir(dir string) { + d, _ := os.Open(dir) + defer d.Close() + files, _ := d.Readdirnames(-1) + for _, f := range files { + os.RemoveAll(f) + } +} + +func (ot *OstreeCommitAction) Run(context *debos.DebosContext) error { + ot.LogStart() + repoPath := path.Join(context.Artifactdir, ot.Repository) + + emptyDir(path.Join(context.Rootdir, "dev")) + + repo, err := otbuiltin.OpenRepo(repoPath) + if err != nil { + return err + } + + _, err = repo.PrepareTransaction() + if err != nil { + return err + } + + opts := otbuiltin.NewCommitOptions() + opts.Subject = ot.Subject + ret, err := repo.Commit(context.Rootdir, ot.Branch, opts) + if err != nil { + return err + } else { + log.Printf("Commit: %s\n", ret) + } + _, err = repo.CommitTransaction() + if err != nil { + return err + } + + return nil +} diff --git a/actions/ostree_deploy_action.go b/actions/ostree_deploy_action.go new file mode 100644 index 0000000..b2d373d --- /dev/null +++ b/actions/ostree_deploy_action.go @@ -0,0 +1,183 @@ +/* +OstreeDeploy Action + +Deploy the OSTree branch to the image. +If any preparation has been done for rootfs, it can be overwritten +during this step. + +Action 'image-partition' must be called prior to OSTree deploy. + +Yaml syntax: + - action: ostree-deploy + repository: repository name + remote_repository: URL + branch: branch name + os: os name + setup-fstab: bool + setup-kernel-cmdline: bool + appendkernelcmdline: arguments + +Mandatory properties: + +- remote_repository -- URL to remote OSTree repository for pulling stateroot branch. +Currently not implemented, please prepare local repository instead. + +- repository -- path to repository with OSTree structure. +This path is relative to 'artifact' directory. + +- os -- os deployment name, as explained in: +https://ostree.readthedocs.io/en/latest/manual/deployment/ + +- branch -- branch of the repository to use for populating the image. + +Optional properties: + +- setup-fstab -- create '/etc/fstab' file for image + +- setup-kernel-cmdline -- add the information from the 'image-partition' +action to the configured commandline. + +- append-kernel-cmdline -- additional kernel command line arguments passed to kernel. +*/ +package actions + +import ( + "fmt" + "io" + "os" + "path" + "strings" + + "github.com/go-debos/debos" + ostree "github.com/sjoerdsimons/ostree-go/pkg/otbuiltin" +) + +type OstreeDeployAction struct { + debos.BaseAction `yaml:",inline"` + Repository string + RemoteRepository string "remote_repository" + Branch string + Os string + SetupFSTab bool `yaml:"setup-fstab"` + SetupKernelCmdline bool `yaml:"setup-kernel-cmdline"` + AppendKernelCmdline string `yaml:"append-kernel-cmdline"` +} + +func NewOstreeDeployAction() *OstreeDeployAction { + ot := &OstreeDeployAction{SetupFSTab: true, SetupKernelCmdline: true} + ot.Description = "Deploying from ostree" + return ot +} + +func (ot *OstreeDeployAction) setupFSTab(deployment *ostree.Deployment, context *debos.DebosContext) error { + deploymentDir := fmt.Sprintf("ostree/deploy/%s/deploy/%s.%d", + deployment.Osname(), deployment.Csum(), deployment.Deployserial()) + + etcDir := path.Join(context.Rootdir, deploymentDir, "etc") + + err := os.Mkdir(etcDir, 0755) + if err != nil && !os.IsExist(err) { + return err + } + + dst, err := os.OpenFile(path.Join(etcDir, "fstab"), os.O_WRONLY|os.O_CREATE, 0755) + defer dst.Close() + if err != nil { + return err + } + + _, err = io.Copy(dst, &context.ImageFSTab) + + return err +} + +func (ot *OstreeDeployAction) Run(context *debos.DebosContext) error { + ot.LogStart() + + // This is to handle cases there we didn't partition an image + if len(context.ImageMntDir) != 0 { + /* First deploy the current rootdir to the image so it can seed e.g. + * bootloader configuration */ + err := debos.Command{}.Run("Deploy to image", "cp", "-a", context.Rootdir+"/.", context.ImageMntDir) + if err != nil { + return fmt.Errorf("rootfs deploy failed: %v", err) + } + context.Rootdir = context.ImageMntDir + } + + repoPath := "file://" + path.Join(context.Artifactdir, ot.Repository) + + sysroot := ostree.NewSysroot(context.Rootdir) + err := sysroot.InitializeFS() + if err != nil { + return err + } + + err = sysroot.InitOsname(ot.Os, nil) + if err != nil { + return err + } + + /* HACK: Getting the repository form the sysroot gets ostree confused on + * whether it should configure /etc/ostree or the repo configuration, + so reopen by hand */ + /* dstRepo, err := sysroot.Repo(nil) */ + dstRepo, err := ostree.OpenRepo(path.Join(context.Rootdir, "ostree/repo")) + if err != nil { + return err + } + + /* FIXME: add support for gpg signing commits so this is no longer needed */ + opts := ostree.RemoteOptions{NoGpgVerify: true} + err = dstRepo.RemoteAdd("origin", ot.RemoteRepository, opts, nil) + if err != nil { + return err + } + + var options ostree.PullOptions + options.OverrideRemoteName = "origin" + options.Refs = []string{ot.Branch} + + err = dstRepo.PullWithOptions(repoPath, options, nil, nil) + if err != nil { + return err + } + + /* Required by ostree to make sure a bunch of information was pulled in */ + sysroot.Load(nil) + + revision, err := dstRepo.ResolveRev(ot.Branch, false) + if err != nil { + return err + } + + var kargs []string + if ot.SetupKernelCmdline { + kargs = append(kargs, context.ImageKernelRoot) + } + + if ot.AppendKernelCmdline != "" { + s := strings.Split(ot.AppendKernelCmdline, " ") + kargs = append(kargs, s...) + } + + origin := sysroot.OriginNewFromRefspec("origin:" + ot.Branch) + deployment, err := sysroot.DeployTree(ot.Os, revision, origin, nil, kargs, nil) + if err != nil { + return err + } + + if ot.SetupFSTab { + err = ot.setupFSTab(deployment, context) + if err != nil { + return err + } + } + + err = sysroot.SimpleWriteDeployment(ot.Os, deployment, nil, 0, nil) + if err != nil { + return err + } + + return nil +} diff --git a/actions/overlay_action.go b/actions/overlay_action.go new file mode 100644 index 0000000..f17ecb3 --- /dev/null +++ b/actions/overlay_action.go @@ -0,0 +1,67 @@ +/* +Overlay Action + +Recursive copy of directory or file to target filesystem. + +Yaml syntax: + - action: overlay + origin: name + source: directory + destination: directory + +Mandatory properties: + +- source -- relative path to the directory or file located in path referenced by `origin`. +In case if this property is absent then pure path referenced by 'origin' will be used. + +Optional properties: + +- origin -- reference to named file or directory. + +- destination -- absolute path in the target rootfs where 'source' will be copied. +All existing files will be overwritten. +If destination isn't set '/' of the rootfs will be usedi. +*/ +package actions + +import ( + "fmt" + "path" + + "github.com/go-debos/debos" +) + +type OverlayAction struct { + debos.BaseAction `yaml:",inline"` + Origin string // origin of overlay, here the export from other action may be used + Source string // external path there overlay is + Destination string // path inside of rootfs +} + +func (overlay *OverlayAction) Verify(context *debos.DebosContext) error { + if _, err := debos.RestrictedPath(context.Rootdir, overlay.Destination); err != nil { + return err + } + return nil +} + +func (overlay *OverlayAction) Run(context *debos.DebosContext) error { + overlay.LogStart() + origin := context.RecipeDir + + //Trying to get a filename from exports first + if len(overlay.Origin) > 0 { + var found bool + if origin, found = context.Origins[overlay.Origin]; !found { + return fmt.Errorf("Origin not found '%s'", overlay.Origin) + } + } + + sourcedir := path.Join(origin, overlay.Source) + destination, err := debos.RestrictedPath(context.Rootdir, overlay.Destination) + if err != nil { + return err + } + + return debos.CopyTree(sourcedir, destination) +} diff --git a/actions/pack_action.go b/actions/pack_action.go new file mode 100644 index 0000000..a90cb1d --- /dev/null +++ b/actions/pack_action.go @@ -0,0 +1,39 @@ +/* +Pack Action + +Create tarball with filesystem. + +Yaml syntax: + - action: pack + file: filename.ext + compression: gz + +Mandatory properties: + +- file -- name of the output tarball. + +- compression -- compression type to use. Only 'gz' is supported at the moment. + +*/ +package actions + +import ( + "log" + "path" + + "github.com/go-debos/debos" +) + +type PackAction struct { + debos.BaseAction `yaml:",inline"` + Compression string + File string +} + +func (pf *PackAction) Run(context *debos.DebosContext) error { + pf.LogStart() + outfile := path.Join(context.Artifactdir, pf.File) + + log.Printf("Compression to %s\n", outfile) + return debos.Command{}.Run("Packing", "tar", "czf", outfile, "-C", context.Rootdir, ".") +} diff --git a/actions/raw_action.go b/actions/raw_action.go new file mode 100644 index 0000000..d3e66fe --- /dev/null +++ b/actions/raw_action.go @@ -0,0 +1,130 @@ +/* +Raw Action + +Directly write a file to the output image at a given offset. +This is typically useful for bootloaders. + +Yaml syntax: + - action: raw + origin: name + source: filename + offset: bytes + +Mandatory properties: + +- origin -- reference to named file or directory. + +- source -- the name of file located in 'origin' to be written into the output image. + +Optional properties: + +- offset -- offset in bytes for output image file. +It is possible to use internal templating mechanism of debos to calculate offset +with sectors (512 bytes) instead of bytes, for instance: '{{ sector 256 }}'. +The default value is zero. + +- partition -- named partition to write to +*/ +package actions + +import ( + "errors" + "fmt" + "io/ioutil" + "log" + "os" + "path" + "strconv" + + "github.com/go-debos/debos" +) + +type RawAction struct { + debos.BaseAction `yaml:",inline"` + Origin string // there the source comes from + Offset string + Source string // relative path inside of origin + Path string // deprecated option (for backward compatibility) + Partition string // Partition to write otherwise full image +} + +func (raw *RawAction) checkDeprecatedSyntax() error { + + // New syntax is based on 'origin' and 'source' + // Check if we do not mix new and old syntax + // TODO: remove deprecated syntax verification + if len(raw.Path) > 0 { + // Deprecated syntax based on 'source' and 'path' + log.Printf("Usage of 'source' and 'path' properties is deprecated.") + log.Printf("Please use 'origin' and 'source' properties.") + if len(raw.Origin) > 0 { + return errors.New("Can't mix 'origin' and 'path'(deprecated option) properties") + } + if len(raw.Source) == 0 { + return errors.New("'source' and 'path' properties can't be empty") + } + // Switch to new syntax + raw.Origin = raw.Source + raw.Source = raw.Path + raw.Path = "" + } + return nil +} + +func (raw *RawAction) Verify(context *debos.DebosContext) error { + if err := raw.checkDeprecatedSyntax(); err != nil { + return err + } + + if len(raw.Origin) == 0 || len(raw.Source) == 0 { + return errors.New("'origin' and 'source' properties can't be empty") + } + + return nil +} + +func (raw *RawAction) Run(context *debos.DebosContext) error { + raw.LogStart() + origin, found := context.Origins[raw.Origin] + if !found { + return fmt.Errorf("Origin `%s` doesn't exist\n", raw.Origin) + } + s := path.Join(origin, raw.Source) + content, err := ioutil.ReadFile(s) + + if err != nil { + return fmt.Errorf("Failed to read %s", s) + } + + var devicePath string + if raw.Partition != "" { + for _, p := range context.ImagePartitions { + if p.Name == raw.Partition { + devicePath = p.DevicePath + break + } + } + + if devicePath == "" { + return fmt.Errorf("Failed to find partition named %s", raw.Partition) + } + } else { + devicePath = context.Image + } + + target, err := os.OpenFile(devicePath, os.O_WRONLY, 0) + if err != nil { + return fmt.Errorf("Failed to open %s: %v", devicePath, err) + } + + offset, err := strconv.ParseInt(raw.Offset, 0, 64) + if err != nil { + return fmt.Errorf("Couldn't parse offset %v", err) + } + bytes, err := target.WriteAt(content, offset) + if bytes != len(content) { + return errors.New("Couldn't write complete data") + } + + return nil +} diff --git a/actions/run_action.go b/actions/run_action.go new file mode 100644 index 0000000..ad374d8 --- /dev/null +++ b/actions/run_action.go @@ -0,0 +1,132 @@ +/* +Run Action + +Allows to run any available command or script in the filesystem or +in host environment. + +Yaml syntax: + - action: run + chroot: bool + postprocess: bool + script: script name + command: command line + +Properties 'command' and 'script' are mutually exclusive. + +- command -- command with arguments; the command expected to be accessible in +host's or chrooted environment -- depending on 'chroot' property. + +- script -- script with arguments; script must be located in recipe directory. + +Optional properties: + +- chroot -- run script or command in target filesystem if set to true. +In other case the command or script is executed within the build process, with +access to the filesystem and the image. In both cases it is run with root privileges. + +- postprocess -- if set script or command is executed after all other commands and +has access to the image file. + + +Properties 'chroot' and 'postprocess' are mutually exclusive. +*/ +package actions + +import ( + "errors" + "github.com/go-debos/fakemachine" + "path" + "strings" + + "github.com/go-debos/debos" +) + +type RunAction struct { + debos.BaseAction `yaml:",inline"` + Chroot bool + PostProcess bool + Script string + Command string +} + +func (run *RunAction) Verify(context *debos.DebosContext) error { + if run.PostProcess && run.Chroot { + return errors.New("Cannot run postprocessing in the chroot") + } + return nil +} + +func (run *RunAction) PreMachine(context *debos.DebosContext, m *fakemachine.Machine, + args *[]string) error { + + if run.Script == "" { + return nil + } + + run.Script = debos.CleanPathAt(run.Script, context.RecipeDir) + // Expect we have no blank spaces in path + scriptpath := strings.Split(run.Script, " ") + + if !run.PostProcess { + m.AddVolume(path.Dir(scriptpath[0])) + } + + return nil +} + +func (run *RunAction) doRun(context debos.DebosContext) error { + run.LogStart() + var cmdline []string + var label string + var cmd debos.Command + + if run.Chroot { + cmd = debos.NewChrootCommandForContext(context) + } else { + cmd = debos.Command{} + } + + if run.Script != "" { + script := strings.SplitN(run.Script, " ", 2) + script[0] = debos.CleanPathAt(script[0], context.RecipeDir) + if run.Chroot { + scriptpath := path.Dir(script[0]) + cmd.AddBindMount(scriptpath, "/script") + script[0] = strings.Replace(script[0], scriptpath, "/script", 1) + } + cmdline = []string{strings.Join(script, " ")} + label = path.Base(run.Script) + } else { + cmdline = []string{run.Command} + label = run.Command + } + + // Command/script with options passed as single string + cmdline = append([]string{"sh", "-c"}, cmdline...) + + if !run.PostProcess { + if !run.Chroot { + cmd.AddEnvKey("ROOTDIR", context.Rootdir) + } + if context.Image != "" { + cmd.AddEnvKey("IMAGE", context.Image) + } + } + + return cmd.Run(label, cmdline...) +} + +func (run *RunAction) Run(context *debos.DebosContext) error { + if run.PostProcess { + /* This runs in postprocessing instead */ + return nil + } + return run.doRun(*context) +} + +func (run *RunAction) PostMachine(context debos.DebosContext) error { + if !run.PostProcess { + return nil + } + return run.doRun(context) +} diff --git a/actions/unpack_action.go b/actions/unpack_action.go new file mode 100644 index 0000000..d4993b1 --- /dev/null +++ b/actions/unpack_action.go @@ -0,0 +1,100 @@ +/* +Unpack Action + +Unpack files from archive to the filesystem. +Useful for creating target rootfs from saved tarball with prepared file structure. + +Only (compressed) tar archives are supported currently. + +Yaml syntax: + - action: unpack + origin: name + file: file.ext + compression: gz + +Mandatory properties: + +- file -- archive's file name. It is possible to skip this property if 'origin' +referenced to downloaded file. + +One of the mandatory properties may be omitted with limitations mentioned above. +It is expected to find archive with name pointed in `file` property inside of `origin` in case if both properties are used. + +Optional properties: + +- origin -- reference to a named file or directory. +The default value is 'artifacts' directory in case if this property is omitted. + +- compression -- optional hint for unpack allowing to use proper compression method. + +Currently only 'gz', bzip2' and 'xz' compression types are supported. +If not provided an attempt to autodetect the compression type will be done. +*/ +package actions + +import ( + "fmt" + "github.com/go-debos/debos" +) + +type UnpackAction struct { + debos.BaseAction `yaml:",inline"` + Compression string + Origin string + File string +} + +func (pf *UnpackAction) Verify(context *debos.DebosContext) error { + + if len(pf.Origin) == 0 && len(pf.File) == 0 { + return fmt.Errorf("Filename can't be empty. Please add 'file' and/or 'origin' property.") + } + + archive, err := debos.NewArchive(pf.File) + if err != nil { + return err + } + if len(pf.Compression) > 0 { + if archive.Type() != debos.Tar { + return fmt.Errorf("Option 'compression' is supported for Tar archives only.") + } + if err := archive.AddOption("tarcompression", pf.Compression); err != nil { + return fmt.Errorf("'%s': %s", pf.File, err) + } + } + + return nil +} + +func (pf *UnpackAction) Run(context *debos.DebosContext) error { + pf.LogStart() + var origin string + + if len(pf.Origin) > 0 { + var found bool + //Trying to get a filename from origins first + origin, found = context.Origins[pf.Origin] + if !found { + return fmt.Errorf("Origin not found '%s'", pf.Origin) + } + } else { + origin = context.Artifactdir + } + + infile, err := debos.RestrictedPath(origin, pf.File) + if err != nil { + return err + } + + archive, err := debos.NewArchive(infile) + if err != nil { + return err + } + if len(pf.Compression) > 0 { + if err := archive.AddOption("tarcompression", pf.Compression); err != nil { + return err + } + } + + return archive.Unpack(context.Rootdir) +} diff --git a/archiver.go b/archiver.go new file mode 100644 index 0000000..e67fdf6 --- /dev/null +++ b/archiver.go @@ -0,0 +1,217 @@ +package debos + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +type ArchiveType int + +// Supported types +const ( + _ ArchiveType = iota // Guess archive type from file extension + Tar + Zip + Deb +) + +type ArchiveBase struct { + file string // Path to archive file + atype ArchiveType + options map[interface{}]interface{} // Archiver-depending map with additional hints +} +type ArchiveTar struct { + ArchiveBase +} +type ArchiveZip struct { + ArchiveBase +} +type ArchiveDeb struct { + ArchiveBase +} + +type Unpacker interface { + Unpack(destination string) error + RelaxedUnpack(destination string) error +} + +type Archiver interface { + Type() ArchiveType + AddOption(key, value interface{}) error + Unpacker +} + +type Archive struct { + Archiver +} + +// Unpack archive as is +func (arc *ArchiveBase) Unpack(destination string) error { + return fmt.Errorf("Unpack is not supported for '%s'", arc.file) +} + +/* +RelaxedUnpack unpack archive in relaxed mode allowing to ignore or +avoid minor issues with unpacker tool or framework. +*/ +func (arc *ArchiveBase) RelaxedUnpack(destination string) error { + return arc.Unpack(destination) +} + +func (arc *ArchiveBase) AddOption(key, value interface{}) error { + if arc.options == nil { + arc.options = make(map[interface{}]interface{}) + } + arc.options[key] = value + return nil +} + +func (arc *ArchiveBase) Type() ArchiveType { return arc.atype } + +// Helper function for unpacking with external tool +func unpack(command []string, destination string) error { + if err := os.MkdirAll(destination, 0755); err != nil { + return err + } + return Command{}.Run("unpack", command...) +} + +// Helper function for checking allowed compression types +// Returns empty string for unknown +func tarOptions(compression string) string { + unpackTarOpts := map[string]string{ + "gz": "-z", + "bzip2": "-j", + "xz": "-J", + } // Trying to guess all other supported compression types + + return unpackTarOpts[compression] +} + +func (tar *ArchiveTar) Unpack(destination string) error { + command := []string{"tar"} + if options, ok := tar.options["taroptions"].([]string); ok { + for _, option := range options { + command = append(command, option) + } + } + command = append(command, "-C", destination) + command = append(command, "-x") + + if compression, ok := tar.options["tarcompression"]; ok { + if unpackTarOpt := tarOptions(compression.(string)); len(unpackTarOpt) > 0 { + command = append(command, unpackTarOpt) + } + } + command = append(command, "-f", tar.file) + + return unpack(command, destination) +} + +func (tar *ArchiveTar) RelaxedUnpack(destination string) error { + + taroptions := []string{"--no-same-owner", "--no-same-permissions"} + options, ok := tar.options["taroptions"].([]string) + defer func() { tar.options["taroptions"] = options }() + + if ok { + for _, option := range options { + taroptions = append(taroptions, option) + } + } + tar.options["taroptions"] = taroptions + + return tar.Unpack(destination) +} + +func (tar *ArchiveTar) AddOption(key, value interface{}) error { + + switch key { + case "taroptions": + // expect a slice + options, ok := value.([]string) + if !ok { + return fmt.Errorf("Wrong type for value") + } + tar.options["taroptions"] = options + + case "tarcompression": + compression, ok := value.(string) + if !ok { + return fmt.Errorf("Wrong type for value") + } + option := tarOptions(compression) + if len(option) == 0 { + return fmt.Errorf("Compression '%s' is not supported", compression) + } + tar.options["tarcompression"] = compression + + default: + return fmt.Errorf("Option '%v' is not supported for tar archive type", key) + } + return nil +} + +func (zip *ArchiveZip) Unpack(destination string) error { + command := []string{"unzip", zip.file, "-d", destination} + return unpack(command, destination) +} + +func (zip *ArchiveZip) RelaxedUnpack(destination string) error { + return zip.Unpack(destination) +} + +func (deb *ArchiveDeb) Unpack(destination string) error { + command := []string{"dpkg", "-x", deb.file, destination} + return unpack(command, destination) +} + +func (deb *ArchiveDeb) RelaxedUnpack(destination string) error { + return deb.Unpack(destination) +} + +/* +NewArchive associate correct structure and methods according to +archive type. If ArchiveType is omitted -- trying to guess the type. +Return ArchiveType or nil in case of error. +*/ +func NewArchive(file string, arcType ...ArchiveType) (Archive, error) { + var archive Archive + var atype ArchiveType + + if len(arcType) == 0 { + ext := filepath.Ext(file) + ext = strings.ToLower(ext) + + switch ext { + case ".deb": + atype = Deb + case ".zip": + atype = Zip + default: + //FIXME: guess Tar maybe? + atype = Tar + } + } else { + atype = arcType[0] + } + + common := ArchiveBase{} + common.file = file + common.atype = atype + common.options = make(map[interface{}]interface{}) + + switch atype { + case Tar: + archive = Archive{&ArchiveTar{ArchiveBase: common}} + case Zip: + archive = Archive{&ArchiveZip{ArchiveBase: common}} + case Deb: + archive = Archive{&ArchiveDeb{ArchiveBase: common}} + default: + return archive, fmt.Errorf("Unsupported archive '%s'", file) + } + return archive, nil +} diff --git a/archiver_test.go b/archiver_test.go new file mode 100644 index 0000000..3023c12 --- /dev/null +++ b/archiver_test.go @@ -0,0 +1,157 @@ +package debos_test + +import ( + _ "fmt" + "github.com/go-debos/debos" + "github.com/stretchr/testify/assert" + _ "reflect" + _ "strings" + "testing" +) + +func TestBase(t *testing.T) { + + // New archive + // Expect Tar by default + _, err := debos.NewArchive("test.base", 0) + assert.EqualError(t, err, "Unsupported archive 'test.base'") + + // Test base + archive := debos.ArchiveBase{} + arcType := archive.Type() + assert.Equal(t, 0, int(arcType)) + + // Add option + err = archive.AddOption("someoption", "somevalue") + assert.Empty(t, err) + + err = archive.Unpack("/tmp/test") + assert.EqualError(t, err, "Unpack is not supported for ''") + err = archive.RelaxedUnpack("/tmp/test") + assert.EqualError(t, err, "Unpack is not supported for ''") +} + +func TestTar_default(t *testing.T) { + + // New archive + // Expect Tar by default + archive, err := debos.NewArchive("test.tar.gz") + assert.NotEmpty(t, archive) + assert.Empty(t, err) + + // Type must be Tar by default + arcType := archive.Type() + assert.Equal(t, debos.Tar, arcType) + + // Test unpack + err = archive.Unpack("/tmp/test") + // Expect unpack failure + assert.EqualError(t, err, "exit status 2") + + // Expect failure for RelaxedUnpack + err = archive.RelaxedUnpack("/tmp/test") + assert.EqualError(t, err, "exit status 2") + + // Check options + err = archive.AddOption("taroptions", []string{"--option1"}) + assert.Empty(t, err) + err = archive.Unpack("/tmp/test") + assert.EqualError(t, err, "exit status 64") + err = archive.Unpack("/proc/debostest") + assert.EqualError(t, err, "mkdir /proc/debostest: no such file or directory") + err = archive.RelaxedUnpack("/tmp/test") + assert.EqualError(t, err, "exit status 64") + + // Add wrong option + err = archive.AddOption("someoption", "somevalue") + assert.EqualError(t, err, "Option 'someoption' is not supported for tar archive type") +} + +// Check supported compression types +func TestTar_compression(t *testing.T) { + compressions := map[string]string{ + "gz": "tar -C test -x -z -f test.tar.gz", + "bzip2": "tar -C test -x -j -f test.tar.gz", + "xz": "tar -C test -x -J -f test.tar.gz", + } + + // Force type + archive, err := debos.NewArchive("test.tar.gz", debos.Tar) + assert.NotEmpty(t, archive) + assert.Empty(t, err) + // Type must be Tar + arcType := archive.Type() + assert.Equal(t, debos.Tar, arcType) + + for compression, _ := range compressions { + err = archive.AddOption("tarcompression", compression) + assert.Empty(t, err) + err := archive.Unpack("test") + assert.EqualError(t, err, "exit status 2") + } + // Check of unsupported compression type + err = archive.AddOption("tarcompression", "fake") + assert.EqualError(t, err, "Compression 'fake' is not supported") + + // Pass incorrect type + err = archive.AddOption("taroptions", nil) + assert.EqualError(t, err, "Wrong type for value") + err = archive.AddOption("tarcompression", nil) + assert.EqualError(t, err, "Wrong type for value") +} + +func TestDeb(t *testing.T) { + + // Guess Deb + archive, err := debos.NewArchive("test.deb") + assert.NotEmpty(t, archive) + assert.Empty(t, err) + + // Type must be guessed as Deb + arcType := archive.Type() + assert.Equal(t, debos.Deb, arcType) + + // Force Deb type + archive, err = debos.NewArchive("test.deb", debos.Deb) + assert.NotEmpty(t, archive) + assert.Empty(t, err) + + // Type must be Deb + arcType = archive.Type() + assert.Equal(t, debos.Deb, arcType) + + // Expect unpack failure + err = archive.Unpack("/tmp/test") + assert.EqualError(t, err, "exit status 2") + err = archive.Unpack("/proc/debostest") + assert.EqualError(t, err, "mkdir /proc/debostest: no such file or directory") + err = archive.RelaxedUnpack("/tmp/test") + assert.EqualError(t, err, "exit status 2") +} + +func TestZip(t *testing.T) { + // Guess zip + archive, err := debos.NewArchive("test.ZiP") + assert.NotEmpty(t, archive) + assert.Empty(t, err) + // Type must be guessed as Zip + arcType := archive.Type() + assert.Equal(t, debos.Zip, arcType) + + // Force Zip type + archive, err = debos.NewArchive("test.zip", debos.Zip) + assert.NotEmpty(t, archive) + assert.Empty(t, err) + + // Type must be Zip + arcType = archive.Type() + assert.Equal(t, debos.Zip, arcType) + + // Expect unpack failure + err = archive.Unpack("/tmp/test") + assert.EqualError(t, err, "exit status 9") + err = archive.Unpack("/proc/debostest") + assert.EqualError(t, err, "mkdir /proc/debostest: no such file or directory") + err = archive.RelaxedUnpack("/tmp/test") + assert.EqualError(t, err, "exit status 9") +} diff --git a/cmd/debos/debos.go b/cmd/debos/debos.go new file mode 100644 index 0000000..caff24b --- /dev/null +++ b/cmd/debos/debos.go @@ -0,0 +1,219 @@ +package main + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "path" + + "github.com/docker/go-units" + "github.com/go-debos/debos" + "github.com/go-debos/debos/recipe" + "github.com/go-debos/fakemachine" + "github.com/jessevdk/go-flags" +) + +func checkError(context debos.DebosContext, err error, a debos.Action, stage string) int { + if err == nil { + return 0 + } + + log.Printf("Action `%s` failed at stage %s, error: %s", a, stage, err) + debos.DebugShell(context) + return 1 +} + +func main() { + var context debos.DebosContext + var options struct { + ArtifactDir string `long:"artifactdir"` + InternalImage string `long:"internal-image" hidden:"true"` + TemplateVars map[string]string `short:"t" long:"template-var" description:"Template variables"` + DebugShell bool `long:"debug-shell" description:"Fall into interactive shell on error"` + Shell string `short:"s" long:"shell" description:"Redefine interactive shell binary (default: bash)" optionsl:"" default:"/bin/bash"` + ScratchSize string `long:"scratchsize" description:"Size of disk backed scratch space"` + } + + var exitcode int = 0 + // Allow to run all deferred calls prior to os.Exit() + defer func() { + os.Exit(exitcode) + }() + + parser := flags.NewParser(&options, flags.Default) + args, err := parser.Parse() + + if err != nil { + flagsErr, ok := err.(*flags.Error) + if ok && flagsErr.Type == flags.ErrHelp { + return + } else { + fmt.Printf("%v\n", flagsErr) + exitcode = 1 + return + } + } + + if len(args) != 1 { + log.Println("No recipe given!") + exitcode = 1 + return + } + + // Set interactive shell binary only if '--debug-shell' options passed + if options.DebugShell { + context.DebugShell = options.Shell + } + + file := args[0] + file = debos.CleanPath(file) + + r := recipe.Recipe{} + if _, err := os.Stat(file); os.IsNotExist(err) { + log.Println(err) + exitcode = 1 + return + } + if err := r.Parse(file, options.TemplateVars); err != nil { + log.Println(err) + exitcode = 1 + return + } + + /* If fakemachine is supported the outer fake machine will never use the + * scratchdir, so just set it to /scrach as a dummy to prevent the outer + * debos createing a temporary direction */ + if fakemachine.InMachine() || fakemachine.Supported() { + context.Scratchdir = "/scratch" + } else { + log.Printf("fakemachine not supported, running on the host!") + cwd, _ := os.Getwd() + context.Scratchdir, err = ioutil.TempDir(cwd, ".debos-") + defer os.RemoveAll(context.Scratchdir) + } + + context.Rootdir = path.Join(context.Scratchdir, "root") + context.Image = options.InternalImage + context.RecipeDir = path.Dir(file) + + context.Artifactdir = options.ArtifactDir + if context.Artifactdir == "" { + context.Artifactdir, _ = os.Getwd() + } + context.Artifactdir = debos.CleanPath(context.Artifactdir) + + // Initialise origins map + context.Origins = make(map[string]string) + context.Origins["artifacts"] = context.Artifactdir + context.Origins["filesystem"] = context.Rootdir + context.Origins["recipe"] = context.RecipeDir + + context.Architecture = r.Architecture + + for _, a := range r.Actions { + err = a.Verify(&context) + if exitcode = checkError(context, err, a, "Verify"); exitcode != 0 { + return + } + } + + if !fakemachine.InMachine() && fakemachine.Supported() { + m := fakemachine.NewMachine() + var args []string + + if options.ScratchSize != "" { + size, err := units.FromHumanSize(options.ScratchSize) + if err != nil { + fmt.Printf("Couldn't parse scratch size: %v\n", err) + exitcode = 1 + return + } + m.SetScratch(size, "") + } + + m.AddVolume(context.Artifactdir) + args = append(args, "--artifactdir", context.Artifactdir) + + for k, v := range options.TemplateVars { + args = append(args, "--template-var", fmt.Sprintf("%s:\"%s\"", k, v)) + } + + m.AddVolume(context.RecipeDir) + args = append(args, file) + + if options.DebugShell { + args = append(args, "--debug-shell") + args = append(args, "--shell", fmt.Sprintf("%s", options.Shell)) + } + + for _, a := range r.Actions { + err = a.PreMachine(&context, m, &args) + if exitcode = checkError(context, err, a, "PreMachine"); exitcode != 0 { + return + } + } + + exitcode, err = m.RunInMachineWithArgs(args) + if err != nil { + fmt.Println(err) + return + } + + if exitcode != 0 { + return + } + + for _, a := range r.Actions { + err = a.PostMachine(context) + if exitcode = checkError(context, err, a, "Postmachine"); exitcode != 0 { + return + } + } + + log.Printf("==== Recipe done ====") + return + } + + if !fakemachine.InMachine() { + for _, a := range r.Actions { + err = a.PreNoMachine(&context) + if exitcode = checkError(context, err, a, "PreNoMachine"); exitcode != 0 { + return + } + } + } + + // Create Rootdir + if _, err = os.Stat(context.Rootdir); os.IsNotExist(err) { + err = os.Mkdir(context.Rootdir, 0755) + if err != nil && os.IsNotExist(err) { + exitcode = 1 + return + } + } + + for _, a := range r.Actions { + err = a.Run(&context) + if exitcode = checkError(context, err, a, "Run"); exitcode != 0 { + return + } + } + + for _, a := range r.Actions { + err = a.Cleanup(context) + if exitcode = checkError(context, err, a, "Cleanup"); exitcode != 0 { + return + } + } + + if !fakemachine.InMachine() { + for _, a := range r.Actions { + err = a.PostMachine(context) + if exitcode = checkError(context, err, a, "PostMachine"); exitcode != 0 { + return + } + } + log.Printf("==== Recipe done ====") + } +} diff --git a/commands.go b/commands.go new file mode 100644 index 0000000..557b636 --- /dev/null +++ b/commands.go @@ -0,0 +1,203 @@ +package debos + +import ( + "bytes" + "fmt" + "io" + "log" + "os" + "os/exec" + "path" +) + +type ChrootEnterMethod int + +const ( + CHROOT_METHOD_NONE = iota // use nspawn to create the chroot environment + CHROOT_METHOD_NSPAWN // No chroot in use + CHROOT_METHOD_CHROOT // use chroot to create the chroot environment +) + +type Command struct { + Architecture string // Architecture of the chroot, nil if same as host + Dir string // Working dir to run command in + Chroot string // Run in the chroot at path + ChrootMethod ChrootEnterMethod // Method to enter the chroot + + bindMounts []string /// Items to bind mount + extraEnv []string // Extra environment variables to set +} + +type commandWrapper struct { + label string + buffer *bytes.Buffer +} + +func newCommandWrapper(label string) *commandWrapper { + b := bytes.Buffer{} + return &commandWrapper{label, &b} +} + +func (w commandWrapper) out(atEOF bool) { + for { + s, err := w.buffer.ReadString('\n') + if err == nil { + log.Printf("%s | %v", w.label, s) + } else { + if len(s) > 0 { + if atEOF && err == io.EOF { + log.Printf("%s | %v\n", w.label, s) + } else { + w.buffer.WriteString(s) + } + } + break + } + } +} + +func (w commandWrapper) Write(p []byte) (n int, err error) { + n, err = w.buffer.Write(p) + w.out(false) + return +} + +func (w *commandWrapper) flush() { + w.out(true) +} + +func NewChrootCommandForContext(context DebosContext) Command { + c := Command{Architecture: context.Architecture, Chroot: context.Rootdir, ChrootMethod: CHROOT_METHOD_NSPAWN} + + if context.Image != "" { + path, err := RealPath(context.Image) + if err == nil { + c.AddBindMount(path, "") + } else { + log.Printf("Failed to get realpath for %s, %v", context.Image, err) + } + for _, p := range context.ImagePartitions { + path, err := RealPath(p.DevicePath) + if err != nil { + log.Printf("Failed to get realpath for %s, %v", p.DevicePath, err) + continue + } + c.AddBindMount(path, "") + } + c.AddBindMount("/dev/disk", "") + } + + return c +} + +func (cmd *Command) AddEnv(env string) { + cmd.extraEnv = append(cmd.extraEnv, env) +} + +func (cmd *Command) AddEnvKey(key, value string) { + cmd.extraEnv = append(cmd.extraEnv, fmt.Sprintf("%s=%s", key, value)) +} + +func (cmd *Command) AddBindMount(source, target string) { + var mount string + if target != "" { + mount = fmt.Sprintf("%s:%s", source, target) + } else { + mount = source + } + + cmd.bindMounts = append(cmd.bindMounts, mount) +} + +func (cmd Command) Run(label string, cmdline ...string) error { + q := newQemuHelper(cmd) + q.Setup() + + var options []string + switch cmd.ChrootMethod { + case CHROOT_METHOD_NONE: + options = cmdline + case CHROOT_METHOD_CHROOT: + options = append(options, "chroot") + options = append(options, cmd.Chroot) + options = append(options, cmdline...) + case CHROOT_METHOD_NSPAWN: + options = append(options, "systemd-nspawn", "-q", "-D", cmd.Chroot) + for _, e := range cmd.extraEnv { + options = append(options, "--setenv", e) + + } + for _, b := range cmd.bindMounts { + options = append(options, "--bind", b) + + } + options = append(options, cmdline...) + } + + exe := exec.Command(options[0], options[1:]...) + w := newCommandWrapper(label) + + exe.Stdin = nil + exe.Stdout = w + exe.Stderr = w + + if len(cmd.extraEnv) > 0 && cmd.ChrootMethod != CHROOT_METHOD_NSPAWN { + exe.Env = append(os.Environ(), cmd.extraEnv...) + } + + // Disable services start/stop for commands running in chroot + if cmd.ChrootMethod != CHROOT_METHOD_NONE { + services := ServiceHelper{cmd.Chroot} + services.Deny() + defer services.Allow() + } + + err := exe.Run() + w.flush() + q.Cleanup() + + return err +} + +type qemuHelper struct { + qemusrc string + qemutarget string +} + +func newQemuHelper(c Command) qemuHelper { + q := qemuHelper{} + + if c.Chroot == "" || c.Architecture == "" { + return q + } + + switch c.Architecture { + case "armhf", "armel", "arm": + q.qemusrc = "/usr/bin/qemu-arm-static" + case "arm64": + q.qemusrc = "/usr/bin/qemu-aarch64-static" + case "amd64", "i386": + /* Dummy, no qemu */ + default: + log.Panicf("Don't know qemu for Architecture %s", c.Architecture) + } + + if q.qemusrc != "" { + q.qemutarget = path.Join(c.Chroot, q.qemusrc) + } + + return q +} + +func (q qemuHelper) Setup() error { + if q.qemusrc == "" { + return nil + } + return CopyFile(q.qemusrc, q.qemutarget, 0755) +} + +func (q qemuHelper) Cleanup() { + if q.qemusrc != "" { + os.Remove(q.qemutarget) + } +} diff --git a/commands_test.go b/commands_test.go new file mode 100644 index 0000000..1fe4d46 --- /dev/null +++ b/commands_test.go @@ -0,0 +1,9 @@ +package debos + +import ( + "testing" +) + +func TestBasicCommand(t *testing.T) { + Command{}.Run("out", "ls", "-l") +} diff --git a/debug.go b/debug.go new file mode 100644 index 0000000..1fd1356 --- /dev/null +++ b/debug.go @@ -0,0 +1,31 @@ +package debos + +import ( + "fmt" + "log" + "os" +) + +/* +DebugShell function launches an interactive shell for +debug and problems investigation. +*/ +func DebugShell(context DebosContext) { + + if len(context.DebugShell) == 0 { + return + } + + pa := os.ProcAttr{ + Files: []*os.File{os.Stdin, os.Stdout, os.Stderr}, + Dir: context.Scratchdir, + } + + // Start an interactive shell for debug. + log.Printf(">>> Starting a debug shell") + if proc, err := os.StartProcess(context.DebugShell, []string{}, &pa); err != nil { + fmt.Printf("Failed: %s\n", err) + } else { + proc.Wait() + } +} diff --git a/filesystem.go b/filesystem.go new file mode 100644 index 0000000..a6e5239 --- /dev/null +++ b/filesystem.go @@ -0,0 +1,105 @@ +package debos + +import ( + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path" + "path/filepath" + "strings" +) + +func CleanPathAt(path, at string) string { + if filepath.IsAbs(path) { + return filepath.Clean(path) + } + + return filepath.Join(at, path) +} + +func CleanPath(path string) string { + cwd, _ := os.Getwd() + return CleanPathAt(path, cwd) +} + +func CopyFile(src, dst string, mode os.FileMode) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + tmp, err := ioutil.TempFile(filepath.Dir(dst), "") + if err != nil { + return err + } + _, err = io.Copy(tmp, in) + if err != nil { + tmp.Close() + os.Remove(tmp.Name()) + return err + } + if err = tmp.Close(); err != nil { + os.Remove(tmp.Name()) + return err + } + if err = os.Chmod(tmp.Name(), mode); err != nil { + os.Remove(tmp.Name()) + return err + } + return os.Rename(tmp.Name(), dst) +} + +func CopyTree(sourcetree, desttree string) error { + fmt.Printf("Overlaying %s on %s\n", sourcetree, desttree) + walker := func(p string, info os.FileInfo, err error) error { + + if err != nil { + return err + } + + suffix, _ := filepath.Rel(sourcetree, p) + target := path.Join(desttree, suffix) + switch info.Mode() & os.ModeType { + case 0: + CopyFile(p, target, info.Mode()) + case os.ModeDir: + os.Mkdir(target, info.Mode()) + case os.ModeSymlink: + link, err := os.Readlink(p) + if err != nil { + log.Panicf("Failed to read symlink %s: %v", suffix, err) + } + os.Symlink(link, target) + default: + log.Panicf("Not handled /%s %v", suffix, info.Mode()) + } + + return nil + } + + return filepath.Walk(sourcetree, walker) +} + +func RealPath(path string) (string, error) { + p, err := filepath.EvalSymlinks(path) + if err != nil { + return "", err + } + + return filepath.Abs(p) +} + +func RestrictedPath(prefix, dest string) (string, error) { + var err error + destination := path.Join(prefix, dest) + destination, err = filepath.Abs(destination) + if err != nil { + return "", err + } + if !strings.HasPrefix(destination, prefix) { + return "", fmt.Errorf("The resulting path points outside of prefix '%s': '%s'\n", prefix, destination) + } + return destination, nil +} @@ -0,0 +1,45 @@ +package debos + +import ( + "fmt" + "io" + "log" + "net/http" + "os" +) + +// Function for downloading single file object with http(s) protocol +func DownloadHttpUrl(url, filename string) error { + log.Printf("Download started: '%s' -> '%s'\n", url, filename) + + // TODO: Proxy support? + + // Check if file object already exists. + fi, err := os.Stat(filename) + if !os.IsNotExist(err) && !fi.Mode().IsRegular() { + return fmt.Errorf("Failed to download '%s': '%s' exists and it is not a regular file\n", url, filename) + } + + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("Url '%s' returned status code %d (%s)\n", url, resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + // Output file + output, err := os.Create(filename) + if err != nil { + return err + } + defer output.Close() + + if _, err := io.Copy(output, resp.Body); err != nil { + return err + } + + return nil +} @@ -0,0 +1,74 @@ +package debos + +import ( + "fmt" + "os" + "path" +) + +const debianPolicyHelper = "/usr/sbin/policy-rc.d" + +/* +ServiceHelper is used to manage services. +Currently supports only debian-based family. +*/ + +type ServiceHelper struct { + Rootdir string +} + +type ServicesManager interface { + Allow() error + Deny() error +} + +/* +Allow() allows to start/stop services on OS level. +*/ +func (s *ServiceHelper) Allow() error { + + helperFile := path.Join(s.Rootdir, debianPolicyHelper) + + if _, err := os.Stat(helperFile); os.IsNotExist(err) { + return nil + } + if err := os.Remove(helperFile); err != nil { + return err + } + return nil +} + +/* +Deny() prohibits to start/stop services on OS level. +*/ +func (s *ServiceHelper) Deny() error { + + helperFile := path.Join(s.Rootdir, debianPolicyHelper) + var helper = []byte(`#!/bin/sh + +exit 101 +`) + + if _, err := os.Stat(helperFile); os.IsExist(err) { + return fmt.Errorf("Policy helper file '%s' exists already", debianPolicyHelper) + } + if _, err := os.Stat(path.Dir(helperFile)); os.IsNotExist(err) { + // do not try to do something if ".../usr/sbin" is not exists + return nil + } + pf, err := os.Create(helperFile) + if err != nil { + return err + } + defer pf.Close() + + if _, err := pf.Write(helper); err != nil { + return err + } + + if err := pf.Chmod(0755); err != nil { + return err + } + + return nil +} diff --git a/recipe/recipe.go b/recipe/recipe.go new file mode 100644 index 0000000..49ff5a8 --- /dev/null +++ b/recipe/recipe.go @@ -0,0 +1,175 @@ +/* +Package 'recipe' implements actions mapping to YAML recipe. + +Recipe syntax + +Recipe is a YAML file which is pre-processed though Golang +text templating engine (https://golang.org/pkg/text/template) + +Recipe is composed of 2 parts: + +- header + +- actions + +Comments are allowed and should be prefixed with '#' symbol. + + # Declare variable 'Var' + {{- $Var := "Value" -}} + + # Header + architecture: arm64 + + # Actions are executed in listed order + actions: + - action: ActionName1 + property1: true + + - action: ActionName2 + # Use value of variable 'Var' defined above + property2: {{$Var}} + +Mandatory properties for receipt: + +- architecture -- target architecture + +- actions -- at least one action should be listed + +Supported actions + +- apt -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Apt_Action + +- debootstrap -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Debootstrap_Action + +- download -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Download_Action + +- filesystem-deploy -- https://godoc.org/github.com/go-debos/debos/actions#hdr-FilesystemDeploy_Action + +- image-partition -- https://godoc.org/github.com/go-debos/debos/actions#hdr-ImagePartition_Action + +- ostree-commit -- https://godoc.org/github.com/go-debos/debos/actions#hdr-OstreeCommit_Action + +- ostree-deploy -- https://godoc.org/github.com/go-debos/debos/actions#hdr-OstreeDeploy_Action + +- overlay -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Overlay_Action + +- pack -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Pack_Action + +- raw -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Raw_Action + +- run -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Run_Action + +- unpack -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Unpack_Action +*/ +package recipe + +import ( + "bytes" + "fmt" + "github.com/go-debos/debos" + "github.com/go-debos/debos/actions" + "gopkg.in/yaml.v2" + "path" + "text/template" +) + +/* the YamlAction just embed the Action interface and implements the + * UnmarshalYAML function so it can select the concrete implementer of a + * specific action at unmarshaling time */ +type YamlAction struct { + debos.Action +} + +type Recipe struct { + Architecture string + Actions []YamlAction +} + +func (y *YamlAction) UnmarshalYAML(unmarshal func(interface{}) error) error { + var aux debos.BaseAction + + err := unmarshal(&aux) + if err != nil { + return err + } + + switch aux.Action { + case "debootstrap": + y.Action = actions.NewDebootstrapAction() + case "pack": + y.Action = &actions.PackAction{} + case "unpack": + y.Action = &actions.UnpackAction{} + case "run": + y.Action = &actions.RunAction{} + case "apt": + y.Action = &actions.AptAction{} + case "ostree-commit": + y.Action = &actions.OstreeCommitAction{} + case "ostree-deploy": + y.Action = actions.NewOstreeDeployAction() + case "overlay": + y.Action = &actions.OverlayAction{} + case "image-partition": + y.Action = &actions.ImagePartitionAction{} + case "filesystem-deploy": + y.Action = actions.NewFilesystemDeployAction() + case "raw": + y.Action = &actions.RawAction{} + case "download": + y.Action = &actions.DownloadAction{} + default: + return fmt.Errorf("Unknown action: %v", aux.Action) + } + + unmarshal(y.Action) + + return nil +} + +func sector(s int) int { + return s * 512 +} + +/* +Parse method reads YAML recipe file and map all steps to appropriate actions. + +- file -- is the path to configuration file + +- templateVars -- optional argument allowing to use custom map for templating +engine. Multiple template maps have no effect; only first map will be used. +*/ +func (r *Recipe) Parse(file string, templateVars ...map[string]string) error { + t := template.New(path.Base(file)) + funcs := template.FuncMap{ + "sector": sector, + } + t.Funcs(funcs) + + if _, err := t.ParseFiles(file); err != nil { + return err + } + + if len(templateVars) == 0 { + templateVars = append(templateVars, make(map[string]string)) + } + + data := new(bytes.Buffer) + if err := t.Execute(data, templateVars[0]); err != nil { + return err + } + + if err := yaml.Unmarshal(data.Bytes(), &r); err != nil { + return err + } + + if len(r.Architecture) == 0 { + return fmt.Errorf("Recipe file must have 'architecture' property") + } + + if len(r.Actions) == 0 { + return fmt.Errorf("Recipe file must have at least one action") + } + + return nil +} diff --git a/recipe/recipe_test.go b/recipe/recipe_test.go new file mode 100644 index 0000000..01457e9 --- /dev/null +++ b/recipe/recipe_test.go @@ -0,0 +1,182 @@ +package recipe_test + +import ( + "github.com/go-debos/debos/recipe" + "github.com/stretchr/testify/assert" + "io/ioutil" + "os" + "testing" +) + +type testRecipe struct { + recipe string + err string +} + +// Test if incorrect file has been passed +func TestParse_incorrect_file(t *testing.T) { + var err error + + var tests = []struct { + filename string + err string + }{ + { + "non-existing.yaml", + "open non-existing.yaml: no such file or directory", + }, + { + "/proc", + "read /proc: is a directory", + }, + } + + for _, test := range tests { + r := recipe.Recipe{} + err = r.Parse(test.filename) + assert.EqualError(t, err, test.err) + } +} + +// Check common recipe syntax +func TestParse_syntax(t *testing.T) { + + var tests = []testRecipe{ + // Test if all actions are supported + {` +architecture: arm64 + +actions: + - action: apt + - action: debootstrap + - action: download + - action: filesystem-deploy + - action: image-partition + - action: ostree-commit + - action: ostree-deploy + - action: overlay + - action: pack + - action: raw + - action: run + - action: unpack +`, + "", // Do not expect failure + }, + // Test of unknown action in list + {` +architecture: arm64 + +actions: + - action: test_unknown_action +`, + "Unknown action: test_unknown_action", + }, + // Test if 'architecture' property absence + {` +actions: + - action: raw +`, + "Recipe file must have 'architecture' property", + }, + // Test if no actions listed + {` +architecture: arm64 +`, + "Recipe file must have at least one action", + }, + // Test of wrong syntax in Yaml + {`wrong`, + "yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `wrong` into recipe.Recipe", + }, + // Test if no actions listed + {` +architecture: arm64 +`, + "Recipe file must have at least one action", + }, + } + + for _, test := range tests { + runTest(t, test) + } +} + +// Check template engine +func TestParse_template(t *testing.T) { + + var test = testRecipe{ + // Test template variables + ` +{{ $action:= or .action "download" }} +architecture: arm64 +actions: + - action: {{ $action }} +`, + "", // Do not expect failure + } + + { // Test of embedded template + r := runTest(t, test) + assert.Equalf(t, r.Actions[0].String(), "download", + "Fail to use embedded variable definition from recipe:%s\n", + test.recipe) + } + + { // Test of user-defined template variable + var templateVars = map[string]string{ + "action": "pack", + } + + r := runTest(t, test, templateVars) + assert.Equalf(t, r.Actions[0].String(), "pack", + "Fail to redefine variable with user-defined map:%s\n", + test.recipe) + } +} + +// Test of 'sector' function embedded to recipe package +func TestParse_sector(t *testing.T) { + var testSector = testRecipe{ + // Fail with unknown action + ` +architecture: arm64 + +actions: + - action: {{ sector 42 }} +`, + "Unknown action: 21504", + } + runTest(t, testSector) +} + +func runTest(t *testing.T, test testRecipe, templateVars ...map[string]string) recipe.Recipe { + file, err := ioutil.TempFile(os.TempDir(), "recipe") + assert.Empty(t, err) + defer os.Remove(file.Name()) + + file.WriteString(test.recipe) + file.Close() + + r := recipe.Recipe{} + if len(templateVars) == 0 { + err = r.Parse(file.Name()) + } else { + err = r.Parse(file.Name(), templateVars[0]) + } + + failed := false + + if len(test.err) > 0 { + // Expected error? + failed = !assert.EqualError(t, err, test.err) + } else { + // Unexpected error + failed = !assert.Empty(t, err) + } + + if failed { + t.Logf("Failed recipe:%s\n", test.recipe) + } + + return r +} |