From 100044871faa0e1de2f62728c3d5b83e1ce41176 Mon Sep 17 00:00:00 2001 From: Andrej Shadura Date: Mon, 25 Mar 2019 10:34:02 +0100 Subject: New upstream version 1.0.0+git20190319.cf3fc48 --- README.md | 4 + action.go | 2 + actions/apt_action.go | 7 + actions/debootstrap_action.go | 5 + actions/ostree_commit_action.go | 36 +++- actions/ostree_deploy_action.go | 8 +- actions/recipe.go | 267 +++++++++++++++++++++++++++++ actions/recipe_action.go | 143 ++++++++++++++++ actions/recipe_test.go | 370 ++++++++++++++++++++++++++++++++++++++++ cmd/debos/debos.go | 25 ++- recipe/recipe.go | 175 ------------------- recipe/recipe_test.go | 182 -------------------- 12 files changed, 861 insertions(+), 363 deletions(-) create mode 100644 actions/recipe.go create mode 100644 actions/recipe_action.go create mode 100644 actions/recipe_test.go delete mode 100644 recipe/recipe.go delete mode 100644 recipe/recipe_test.go diff --git a/README.md b/README.md index bf49524..03b6d54 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,9 @@ Application Options: -s, --shell= Redefine interactive shell binary (default: bash) --scratchsize= Size of disk backed scratch space -e, --environ-var= Environment variables + -v, --verbose Verbose output + --print-recipe Print final recipe + --dry-run Compose final recipe to build but without any real work started ## Description @@ -38,6 +41,7 @@ Some of the actions provided by debos to customize and produce images are: * overlay: do a recursive copy of directories or files to the target filesystem * pack: create a tarball with the target filesystem * raw: directly write a file to the output image at a given offset +* recipe: includes the recipe actions at the given path * run: allows to run a command or script in the filesystem or in the host * unpack: unpack files from archive in the filesystem diff --git a/action.go b/action.go index b934e9b..85a4d12 100644 --- a/action.go +++ b/action.go @@ -37,6 +37,8 @@ type DebosContext struct { Origins map[string]string State DebosState EnvironVars map[string]string + PrintRecipe bool + Verbose bool } type Action interface { diff --git a/actions/apt_action.go b/actions/apt_action.go index 681c069..6f3988f 100644 --- a/actions/apt_action.go +++ b/actions/apt_action.go @@ -6,6 +6,7 @@ Install packages and their dependencies to the target rootfs with 'apt'. Yaml syntax: - action: apt recommends: bool + unauthenticated: bool packages: - package1 - package2 @@ -17,6 +18,7 @@ Mandatory properties: Optional properties: - recommends -- boolean indicating if suggested packages will be installed +- unauthenticated -- boolean indicating if unauthenticated packages can be installed */ package actions @@ -27,6 +29,7 @@ import ( type AptAction struct { debos.BaseAction `yaml:",inline"` Recommends bool + Unauthenticated bool Packages []string } @@ -38,6 +41,10 @@ func (apt *AptAction) Run(context *debos.DebosContext) error { aptOptions = append(aptOptions, "--no-install-recommends") } + if apt.Unauthenticated { + aptOptions = append(aptOptions, "--allow-unauthenticated") + } + aptOptions = append(aptOptions, "install") aptOptions = append(aptOptions, apt.Packages...) diff --git a/actions/debootstrap_action.go b/actions/debootstrap_action.go index 02cbb15..b4d6730 100644 --- a/actions/debootstrap_action.go +++ b/actions/debootstrap_action.go @@ -81,6 +81,11 @@ func (d *DebootstrapAction) RunSecondStage(context debos.DebosContext) error { "--no-check-gpg", "--second-stage"} + if d.Components != nil { + s := strings.Join(d.Components, ",") + cmdline = append(cmdline, fmt.Sprintf("--components=%s", s)) + } + c := debos.NewChrootCommandForContext(context) // Can't use nspawn for debootstrap as it wants to create device nodes c.ChrootMethod = debos.CHROOT_METHOD_CHROOT diff --git a/actions/ostree_commit_action.go b/actions/ostree_commit_action.go index a0d8333..85b3fda 100644 --- a/actions/ostree_commit_action.go +++ b/actions/ostree_commit_action.go @@ -8,6 +8,13 @@ Yaml syntax: repository: repository name branch: branch name subject: commit message + collection-id: org.apertis.example + ref-binding: + - branch1 + - branch2 + metadata: + key: value + vendor.key: somevalue Mandatory properties: @@ -22,10 +29,18 @@ type (https://ostree.readthedocs.io/en/latest/manual/repo/#repository-types-and- Optional properties: - subject -- one line message with commit description. + +- collection-id -- Collection ID ref binding (requires libostree 2018.6). + +- ref-binding -- enforce that the commit was retrieved from one of the branch names in this array. + If 'collection-id' is set and 'ref-binding' is empty, will default to the branch name. + +- metadata -- key-value pairs of meta information to be added into commit. */ package actions import ( + "fmt" "log" "os" "path" @@ -40,6 +55,9 @@ type OstreeCommitAction struct { Branch string Subject string Command string + CollectionID string `yaml:"collection-id"` + RefBinding []string `yaml:"ref-binding"` + Metadata map[string]string } func emptyDir(dir string) { @@ -54,7 +72,7 @@ func emptyDir(dir string) { for _, f := range files { err := os.RemoveAll(path.Join(dir, f)) if err != nil { - log.Fatalf("Failed to remove file: %v", err) + log.Fatalf("Failed to remove file: %v", err) } } } @@ -77,6 +95,22 @@ func (ot *OstreeCommitAction) Run(context *debos.DebosContext) error { opts := otbuiltin.NewCommitOptions() opts.Subject = ot.Subject + for k, v := range ot.Metadata { + str := fmt.Sprintf("%s=%s", k, v) + opts.AddMetadataString = append(opts.AddMetadataString, str) + } + + if ot.CollectionID != "" { + opts.CollectionID = ot.CollectionID + if len(ot.RefBinding) == 0 { + // Add current branch if not explitely set via 'ref-binding' + opts.RefBinding = append(opts.RefBinding, ot.Branch) + } + } + + // Add values from 'ref-binding' if any + opts.RefBinding = append(opts.RefBinding, ot.RefBinding...) + ret, err := repo.Commit(context.Rootdir, ot.Branch, opts) if err != nil { return err diff --git a/actions/ostree_deploy_action.go b/actions/ostree_deploy_action.go index dab2265..7d6e6bd 100644 --- a/actions/ostree_deploy_action.go +++ b/actions/ostree_deploy_action.go @@ -18,6 +18,7 @@ Yaml syntax: setup-fstab: bool setup-kernel-cmdline: bool appendkernelcmdline: arguments + collection-id: org.apertis.example Mandatory properties: @@ -44,6 +45,8 @@ action to the configured commandline. - tls-client-cert-path -- path to client certificate to use for the remote repository - tls-client-key-path -- path to client certificate key to use for the remote repository + +- collection-id -- Collection ID ref binding (require libostree 2018.6). */ package actions @@ -70,6 +73,7 @@ type OstreeDeployAction struct { AppendKernelCmdline string `yaml:"append-kernel-cmdline"` TlsClientCertPath string `yaml:"tls-client-cert-path"` TlsClientKeyPath string `yaml:"tls-client-key-path"` + CollectionID string `yaml:"collection-id"` } func NewOstreeDeployAction() *OstreeDeployAction { @@ -140,7 +144,9 @@ func (ot *OstreeDeployAction) Run(context *debos.DebosContext) error { /* FIXME: add support for gpg signing commits so this is no longer needed */ opts := ostree.RemoteOptions{NoGpgVerify: true, TlsClientCertPath: ot.TlsClientCertPath, - TlsClientKeyPath: ot.TlsClientKeyPath} + TlsClientKeyPath: ot.TlsClientKeyPath, + CollectionId: ot.CollectionID, + } err = dstRepo.RemoteAdd("origin", ot.RemoteRepository, opts, nil) if err != nil { diff --git a/actions/recipe.go b/actions/recipe.go new file mode 100644 index 0000000..1da6a6b --- /dev/null +++ b/actions/recipe.go @@ -0,0 +1,267 @@ +/* +Package 'recipe' implements actions mapping to YAML recipe. + +Recipe syntax + +Recipe is a YAML file which is pre-processed though Golang +text templating engine (https://golang.org/pkg/text/template) + +Recipe is composed of 2 parts: + +- header + +- actions + +Comments are allowed and should be prefixed with '#' symbol. + + # Declare variable 'Var' + {{- $Var := "Value" -}} + + # Header + architecture: arm64 + + # Actions are executed in listed order + actions: + - action: ActionName1 + property1: true + + - action: ActionName2 + # Use value of variable 'Var' defined above + property2: {{$Var}} + +Mandatory properties for receipt: + +- architecture -- target architecture + +- actions -- at least one action should be listed + +Supported actions + +- apt -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Apt_Action + +- debootstrap -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Debootstrap_Action + +- download -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Download_Action + +- filesystem-deploy -- https://godoc.org/github.com/go-debos/debos/actions#hdr-FilesystemDeploy_Action + +- image-partition -- https://godoc.org/github.com/go-debos/debos/actions#hdr-ImagePartition_Action + +- ostree-commit -- https://godoc.org/github.com/go-debos/debos/actions#hdr-OstreeCommit_Action + +- ostree-deploy -- https://godoc.org/github.com/go-debos/debos/actions#hdr-OstreeDeploy_Action + +- overlay -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Overlay_Action + +- pack -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Pack_Action + +- raw -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Raw_Action + +- recipe -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Recipe_Action + +- run -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Run_Action + +- unpack -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Unpack_Action +*/ +package actions + +import ( + "bytes" + "fmt" + "github.com/go-debos/debos" + "gopkg.in/yaml.v2" + "path" + "text/template" + "log" + "strings" + "reflect" +) + +/* the YamlAction just embed the Action interface and implements the + * UnmarshalYAML function so it can select the concrete implementer of a + * specific action at unmarshaling time */ +type YamlAction struct { + debos.Action +} + +type Recipe struct { + Architecture string + Actions []YamlAction +} + +func (y *YamlAction) UnmarshalYAML(unmarshal func(interface{}) error) error { + var aux debos.BaseAction + + err := unmarshal(&aux) + if err != nil { + return err + } + + switch aux.Action { + case "debootstrap": + y.Action = NewDebootstrapAction() + case "pack": + y.Action = &PackAction{} + case "unpack": + y.Action = &UnpackAction{} + case "run": + y.Action = &RunAction{} + case "apt": + y.Action = &AptAction{} + case "ostree-commit": + y.Action = &OstreeCommitAction{} + case "ostree-deploy": + y.Action = NewOstreeDeployAction() + case "overlay": + y.Action = &OverlayAction{} + case "image-partition": + y.Action = &ImagePartitionAction{} + case "filesystem-deploy": + y.Action = NewFilesystemDeployAction() + case "raw": + y.Action = &RawAction{} + case "download": + y.Action = &DownloadAction{} + case "recipe": + y.Action = &RecipeAction{} + default: + return fmt.Errorf("Unknown action: %v", aux.Action) + } + + unmarshal(y.Action) + + return nil +} + +func sector(s int) int { + return s * 512 +} + +func DumpActionStruct(iface interface{}) string { + var a []string + + s := reflect.ValueOf(iface) + t := reflect.TypeOf(iface) + + for i := 0; i < t.NumField(); i++ { + f := s.Field(i) + // Dump only exported entries + if f.CanInterface() { + str := fmt.Sprintf("%s: %v", s.Type().Field(i).Name, f.Interface()) + a = append(a, str) + } + } + + return strings.Join(a, ", ") +} + +const tabs = 2 + +func DumpActions(iface interface{}, depth int) { + tab := strings.Repeat(" ", depth * tabs) + entries := reflect.ValueOf(iface) + + for i := 0; i < entries.NumField(); i++ { + if entries.Type().Field(i).Name == "Actions" { + log.Printf("%s %s:\n", tab, entries.Type().Field(i).Name) + actions := reflect.ValueOf(entries.Field(i).Interface()) + for j := 0; j < actions.Len(); j++ { + yaml := reflect.ValueOf(actions.Index(j).Interface()) + DumpActionFields(yaml.Field(0).Interface(), depth + 1) + } + } else { + log.Printf("%s %s: %v\n", tab, entries.Type().Field(i).Name, entries.Field(i).Interface()) + } + } +} + +func DumpActionFields(iface interface{}, depth int) { + tab := strings.Repeat(" ", depth * tabs) + entries := reflect.ValueOf(iface).Elem() + + for i := 0; i < entries.NumField(); i++ { + f := entries.Field(i) + // Dump only exported entries + if f.CanInterface() { + switch f.Kind() { + case reflect.Struct: + if entries.Type().Field(i).Type.String() == "debos.BaseAction" { + // BaseAction is the only struct embbed in Action ActionFields + // dump it at the same level + log.Printf("%s- %s", tab, DumpActionStruct(f.Interface())) + } + + case reflect.Slice: + s := reflect.ValueOf(f.Interface()) + if s.Len() > 0 && s.Index(0).Kind() == reflect.Struct { + log.Printf("%s %s:\n", tab, entries.Type().Field(i).Name) + for j := 0; j < s.Len(); j++ { + if s.Index(j).Kind() == reflect.Struct { + log.Printf("%s { %s }", tab, DumpActionStruct(s.Index(j).Interface())) + } + } + } else { + log.Printf("%s %s: %s\n", tab, entries.Type().Field(i).Name, f) + } + + default: + log.Printf("%s %s: %v\n", tab, entries.Type().Field(i).Name, f.Interface()) + } + } + } +} + +/* +Parse method reads YAML recipe file and map all steps to appropriate actions. + +- file -- is the path to configuration file + +- templateVars -- optional argument allowing to use custom map for templating +engine. Multiple template maps have no effect; only first map will be used. +*/ +func (r *Recipe) Parse(file string, printRecipe bool, dump bool, templateVars ...map[string]string) error { + t := template.New(path.Base(file)) + funcs := template.FuncMap{ + "sector": sector, + } + t.Funcs(funcs) + + if _, err := t.ParseFiles(file); err != nil { + return err + } + + if len(templateVars) == 0 { + templateVars = append(templateVars, make(map[string]string)) + } + + data := new(bytes.Buffer) + if err := t.Execute(data, templateVars[0]); err != nil { + return err + } + + if printRecipe || dump { + log.Printf("Recipe '%s':", file) + } + + if printRecipe { + log.Printf("%s", data) + } + + if err := yaml.Unmarshal(data.Bytes(), &r); err != nil { + return err + } + + if dump { + DumpActions(reflect.ValueOf(*r).Interface(), 0) + } + + if len(r.Architecture) == 0 { + return fmt.Errorf("Recipe file must have 'architecture' property") + } + + if len(r.Actions) == 0 { + return fmt.Errorf("Recipe file must have at least one action") + } + + return nil +} diff --git a/actions/recipe_action.go b/actions/recipe_action.go new file mode 100644 index 0000000..e48f650 --- /dev/null +++ b/actions/recipe_action.go @@ -0,0 +1,143 @@ +/* +Recipe Action + +Include a recipe. + +Yaml syntax: + - action: recipe + recipe: path to recipe + variables: + key: value + +Mandatory properties: + +- recipe -- includes the recipe actions at the given path. + +Optional properties: + +- variables -- overrides or adds new template variables. + +*/ +package actions + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "github.com/go-debos/debos" + "github.com/go-debos/fakemachine" +) + +type RecipeAction struct { + debos.BaseAction `yaml:",inline"` + Recipe string + Variables map[string]string + Actions Recipe `yaml:"-"` + templateVars map[string]string +} + +func (recipe *RecipeAction) Verify(context *debos.DebosContext) error { + if len(recipe.Recipe) == 0 { + return errors.New("'recipe' property can't be empty") + } + + file := recipe.Recipe + if !filepath.IsAbs(file) { + file = filepath.Clean(context.RecipeDir + "/" + recipe.Recipe) + } + + if _, err := os.Stat(file); os.IsNotExist(err) { + return err + } + + // Initialise template vars + recipe.templateVars = make(map[string]string) + recipe.templateVars["included_recipe"] = "true" + recipe.templateVars["architecture"] = context.Architecture + + // Add Variables to template vars + for k, v := range recipe.Variables { + recipe.templateVars[k] = v + } + + if err := recipe.Actions.Parse(file, context.PrintRecipe, context.Verbose, recipe.templateVars); err != nil { + return err + } + + if context.Architecture != recipe.Actions.Architecture { + return fmt.Errorf("Expect architecture '%s' but got '%s'", context.Architecture, recipe.Actions.Architecture) + } + + for _, a := range recipe.Actions.Actions { + if err := a.Verify(context); err != nil { + return err + } + } + + return nil +} + +func (recipe *RecipeAction) PreMachine(context *debos.DebosContext, m *fakemachine.Machine, args *[]string) error { + // TODO: check args? + + for _, a := range recipe.Actions.Actions { + if err := a.PreMachine(context, m, args); err != nil { + return err + } + } + + return nil +} + +func (recipe *RecipeAction) PreNoMachine(context *debos.DebosContext) error { + for _, a := range recipe.Actions.Actions { + if err := a.PreNoMachine(context); err != nil { + return err + } + } + + return nil +} + +func (recipe *RecipeAction) Run(context *debos.DebosContext) error { + recipe.LogStart() + + for _, a := range recipe.Actions.Actions { + if err := a.Run(context); err != nil { + return err + } + } + + return nil +} + +func (recipe *RecipeAction) Cleanup(context *debos.DebosContext) error { + for _, a := range recipe.Actions.Actions { + if err := a.Cleanup(context); err != nil { + return err + } + } + + return nil +} + +func (recipe *RecipeAction) PostMachine(context *debos.DebosContext) error { + for _, a := range recipe.Actions.Actions { + if err := a.PostMachine(context); err != nil { + return err + } + } + + return nil +} + +func (recipe *RecipeAction) PostMachineCleanup(context *debos.DebosContext) error { + for _, a := range recipe.Actions.Actions { + if err := a.PostMachineCleanup(context); err != nil { + return err + } + } + + return nil +} diff --git a/actions/recipe_test.go b/actions/recipe_test.go new file mode 100644 index 0000000..972bf61 --- /dev/null +++ b/actions/recipe_test.go @@ -0,0 +1,370 @@ +package actions_test + +import ( + "github.com/go-debos/debos" + "github.com/go-debos/debos/actions" + "github.com/stretchr/testify/assert" + "io/ioutil" + "os" + "testing" + "strings" +) + +type testRecipe struct { + recipe string + err string +} + +// Test if incorrect file has been passed +func TestParse_incorrect_file(t *testing.T) { + var err error + + var tests = []struct { + filename string + err string + }{ + { + "non-existing.yaml", + "open non-existing.yaml: no such file or directory", + }, + { + "/proc", + "read /proc: is a directory", + }, + } + + for _, test := range tests { + r := actions.Recipe{} + err = r.Parse(test.filename, false, false) + assert.EqualError(t, err, test.err) + } +} + +// Check common recipe syntax +func TestParse_syntax(t *testing.T) { + + var tests = []testRecipe{ + // Test if all actions are supported + {` +architecture: arm64 + +actions: + - action: apt + - action: debootstrap + - action: download + - action: filesystem-deploy + - action: image-partition + - action: ostree-commit + - action: ostree-deploy + - action: overlay + - action: pack + - action: raw + - action: run + - action: unpack + - action: recipe +`, + "", // Do not expect failure + }, + // Test of unknown action in list + {` +architecture: arm64 + +actions: + - action: test_unknown_action +`, + "Unknown action: test_unknown_action", + }, + // Test if 'architecture' property absence + {` +actions: + - action: raw +`, + "Recipe file must have 'architecture' property", + }, + // Test if no actions listed + {` +architecture: arm64 +`, + "Recipe file must have at least one action", + }, + // Test of wrong syntax in Yaml + {`wrong`, + "yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `wrong` into actions.Recipe", + }, + // Test if no actions listed + {` +architecture: arm64 +`, + "Recipe file must have at least one action", + }, + } + + for _, test := range tests { + runTest(t, test) + } +} + +// Check template engine +func TestParse_template(t *testing.T) { + + var test = testRecipe{ + // Test template variables + ` +{{ $action:= or .action "download" }} +architecture: arm64 +actions: + - action: {{ $action }} +`, + "", // Do not expect failure + } + + { // Test of embedded template + r := runTest(t, test) + assert.Equalf(t, r.Actions[0].String(), "download", + "Fail to use embedded variable definition from recipe:%s\n", + test.recipe) + } + + { // Test of user-defined template variable + var templateVars = map[string]string{ + "action": "pack", + } + + r := runTest(t, test, templateVars) + assert.Equalf(t, r.Actions[0].String(), "pack", + "Fail to redefine variable with user-defined map:%s\n", + test.recipe) + } +} + +// Test of 'sector' function embedded to recipe package +func TestParse_sector(t *testing.T) { + var testSector = testRecipe{ + // Fail with unknown action + ` +architecture: arm64 + +actions: + - action: {{ sector 42 }} +`, + "Unknown action: 21504", + } + runTest(t, testSector) +} + +func runTest(t *testing.T, test testRecipe, templateVars ...map[string]string) actions.Recipe { + file, err := ioutil.TempFile(os.TempDir(), "recipe") + assert.Empty(t, err) + defer os.Remove(file.Name()) + + file.WriteString(test.recipe) + file.Close() + + r := actions.Recipe{} + if len(templateVars) == 0 { + err = r.Parse(file.Name(), false, false) + } else { + err = r.Parse(file.Name(), false, false, templateVars[0]) + } + + failed := false + + if len(test.err) > 0 { + // Expected error? + failed = !assert.EqualError(t, err, test.err) + } else { + // Unexpected error + failed = !assert.Empty(t, err) + } + + if failed { + t.Logf("Failed recipe:%s\n", test.recipe) + } + + return r +} + +type subRecipe struct { + name string + recipe string +} + +type testSubRecipe struct { + recipe string + subrecipe subRecipe + err string +} + +func TestSubRecipe(t *testing.T) { + // Embedded recipes + var recipeAmd64 = subRecipe { + "amd64.yaml", + ` +architecture: amd64 + +actions: + - action: run + command: ok.sh +`, + } + var recipeInheritedArch = subRecipe { + "inherited.yaml", + ` +{{- $architecture := or .architecture "armhf" }} +architecture: {{ $architecture }} + +actions: + - action: run + command: ok.sh +`, + } + var recipeArmhf = subRecipe { + "armhf.yaml", + ` +architecture: armhf + +actions: + - action: run + command: ok.sh +`, + } + var recipeIncluded = subRecipe { + "included.yaml", + ` +{{- $included_recipe := or .included_recipe "false"}} +architecture: amd64 + +actions: + - action: run + command: ok.sh + {{- if ne $included_recipe "true" }} + - action: recipe + recipe: armhf.yaml + {{- end }} +`, + } + + // test recipes + var tests = []testSubRecipe { + { + // Test recipe same architecture OK + ` +architecture: amd64 + +actions: + - action: recipe + recipe: amd64.yaml +`, + recipeAmd64, + "", // Do not expect failure + }, + { + // Test recipe with inherited architecture OK + ` +architecture: amd64 + +actions: + - action: recipe + recipe: inherited.yaml +`, + recipeInheritedArch, + "", // Do not expect failure + }, + { + // Fail with unknown recipe + ` +architecture: amd64 + +actions: + - action: recipe + recipe: unknown_recipe.yaml +`, + recipeAmd64, + "stat /tmp/unknown_recipe.yaml: no such file or directory", + }, + { + // Fail with different architecture recipe + ` +architecture: amd64 + +actions: + - action: recipe + recipe: armhf.yaml +`, + recipeArmhf, + "Expect architecture 'amd64' but got 'armhf'", + }, + { + // Test included_recipe prevents parsing OK + ` +architecture: amd64 + +actions: + - action: recipe + recipe: included.yaml +`, + recipeIncluded, + "", // Do not expect failure + }, + } + + for _, test := range tests { + runTestWithSubRecipes(t, test) + } +} + +func runTestWithSubRecipes(t *testing.T, test testSubRecipe, templateVars ...map[string]string) actions.Recipe { + var context debos.DebosContext + dir, err := ioutil.TempDir("", "go-debos") + assert.Empty(t, err) + defer os.RemoveAll(dir) + + file, err := ioutil.TempFile(dir, "recipe") + assert.Empty(t, err) + defer os.Remove(file.Name()) + + file.WriteString(test.recipe) + file.Close() + + file_subrecipe, err := os.Create(dir + "/" + test.subrecipe.name) + assert.Empty(t, err) + defer os.Remove(file_subrecipe.Name()) + + file_subrecipe.WriteString(test.subrecipe.recipe) + file_subrecipe.Close() + + r := actions.Recipe{} + if len(templateVars) == 0 { + err = r.Parse(file.Name(), false, false) + } else { + err = r.Parse(file.Name(), false, false, templateVars[0]) + } + + // Should not expect error during parse + failed := !assert.Empty(t, err) + + if !failed { + context.Architecture = r.Architecture + context.RecipeDir = dir + + for _, a := range r.Actions { + if err = a.Verify(&context); err != nil { + break + } + } + + if len(test.err) > 0 { + // Expected error? + failed = !assert.EqualError(t, err, strings.Replace(test.err, "/tmp", dir, 1)) + } else { + // Unexpected error + failed = !assert.Empty(t, err) + } + } + + if failed { + t.Logf("Failed recipe:%s\n", test.recipe) + } + + return r +} diff --git a/cmd/debos/debos.go b/cmd/debos/debos.go index 20f5253..063dd0d 100644 --- a/cmd/debos/debos.go +++ b/cmd/debos/debos.go @@ -10,7 +10,7 @@ import ( "github.com/docker/go-units" "github.com/go-debos/debos" - "github.com/go-debos/debos/recipe" + "github.com/go-debos/debos/actions" "github.com/go-debos/fakemachine" "github.com/jessevdk/go-flags" ) @@ -26,7 +26,7 @@ func checkError(context *debos.DebosContext, err error, a debos.Action, stage st return 1 } -func do_run(r recipe.Recipe, context *debos.DebosContext) int { +func do_run(r actions.Recipe, context *debos.DebosContext) int { for _, a := range r.Actions { err := a.Run(context) @@ -55,6 +55,7 @@ func warnLocalhost(variable string, value string) { } } + func main() { var context debos.DebosContext var options struct { @@ -68,6 +69,9 @@ func main() { Memory string `short:"m" long:"memory" description:"Amount of memory for build VM (default: 2048MB)"` ShowBoot bool `long:"show-boot" description:"Show boot/console messages from the fake machine"` EnvironVars map[string]string `short:"e" long:"environ-var" description:"Environment variables (use -e VARIABLE:VALUE syntax)"` + Verbose bool `short:"v" long:"verbose" description:"Verbose output"` + PrintRecipe bool `long:"print-recipe" description:"Print final recipe"` + DryRun bool `long:"dry-run" description:"Compose final recipe to build but without any real work started"` } // These are the environment variables that will be detected on the @@ -113,16 +117,24 @@ func main() { context.DebugShell = options.Shell } + if options.PrintRecipe { + context.PrintRecipe = options.PrintRecipe + } + + if options.Verbose { + context.Verbose = options.Verbose + } + file := args[0] file = debos.CleanPath(file) - r := recipe.Recipe{} + r := actions.Recipe{} if _, err := os.Stat(file); os.IsNotExist(err) { log.Println(err) exitcode = 1 return } - if err := r.Parse(file, options.TemplateVars); err != nil { + if err := r.Parse(file, options.PrintRecipe, options.Verbose, options.TemplateVars); err != nil { log.Println(err) exitcode = 1 return @@ -195,6 +207,11 @@ func main() { } } + if options.DryRun { + log.Printf("==== Recipe done (Dry run) ====") + return + } + if !fakemachine.InMachine() && fakemachine.Supported() { m := fakemachine.NewMachine() var args []string diff --git a/recipe/recipe.go b/recipe/recipe.go deleted file mode 100644 index 49ff5a8..0000000 --- a/recipe/recipe.go +++ /dev/null @@ -1,175 +0,0 @@ -/* -Package 'recipe' implements actions mapping to YAML recipe. - -Recipe syntax - -Recipe is a YAML file which is pre-processed though Golang -text templating engine (https://golang.org/pkg/text/template) - -Recipe is composed of 2 parts: - -- header - -- actions - -Comments are allowed and should be prefixed with '#' symbol. - - # Declare variable 'Var' - {{- $Var := "Value" -}} - - # Header - architecture: arm64 - - # Actions are executed in listed order - actions: - - action: ActionName1 - property1: true - - - action: ActionName2 - # Use value of variable 'Var' defined above - property2: {{$Var}} - -Mandatory properties for receipt: - -- architecture -- target architecture - -- actions -- at least one action should be listed - -Supported actions - -- apt -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Apt_Action - -- debootstrap -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Debootstrap_Action - -- download -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Download_Action - -- filesystem-deploy -- https://godoc.org/github.com/go-debos/debos/actions#hdr-FilesystemDeploy_Action - -- image-partition -- https://godoc.org/github.com/go-debos/debos/actions#hdr-ImagePartition_Action - -- ostree-commit -- https://godoc.org/github.com/go-debos/debos/actions#hdr-OstreeCommit_Action - -- ostree-deploy -- https://godoc.org/github.com/go-debos/debos/actions#hdr-OstreeDeploy_Action - -- overlay -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Overlay_Action - -- pack -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Pack_Action - -- raw -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Raw_Action - -- run -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Run_Action - -- unpack -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Unpack_Action -*/ -package recipe - -import ( - "bytes" - "fmt" - "github.com/go-debos/debos" - "github.com/go-debos/debos/actions" - "gopkg.in/yaml.v2" - "path" - "text/template" -) - -/* the YamlAction just embed the Action interface and implements the - * UnmarshalYAML function so it can select the concrete implementer of a - * specific action at unmarshaling time */ -type YamlAction struct { - debos.Action -} - -type Recipe struct { - Architecture string - Actions []YamlAction -} - -func (y *YamlAction) UnmarshalYAML(unmarshal func(interface{}) error) error { - var aux debos.BaseAction - - err := unmarshal(&aux) - if err != nil { - return err - } - - switch aux.Action { - case "debootstrap": - y.Action = actions.NewDebootstrapAction() - case "pack": - y.Action = &actions.PackAction{} - case "unpack": - y.Action = &actions.UnpackAction{} - case "run": - y.Action = &actions.RunAction{} - case "apt": - y.Action = &actions.AptAction{} - case "ostree-commit": - y.Action = &actions.OstreeCommitAction{} - case "ostree-deploy": - y.Action = actions.NewOstreeDeployAction() - case "overlay": - y.Action = &actions.OverlayAction{} - case "image-partition": - y.Action = &actions.ImagePartitionAction{} - case "filesystem-deploy": - y.Action = actions.NewFilesystemDeployAction() - case "raw": - y.Action = &actions.RawAction{} - case "download": - y.Action = &actions.DownloadAction{} - default: - return fmt.Errorf("Unknown action: %v", aux.Action) - } - - unmarshal(y.Action) - - return nil -} - -func sector(s int) int { - return s * 512 -} - -/* -Parse method reads YAML recipe file and map all steps to appropriate actions. - -- file -- is the path to configuration file - -- templateVars -- optional argument allowing to use custom map for templating -engine. Multiple template maps have no effect; only first map will be used. -*/ -func (r *Recipe) Parse(file string, templateVars ...map[string]string) error { - t := template.New(path.Base(file)) - funcs := template.FuncMap{ - "sector": sector, - } - t.Funcs(funcs) - - if _, err := t.ParseFiles(file); err != nil { - return err - } - - if len(templateVars) == 0 { - templateVars = append(templateVars, make(map[string]string)) - } - - data := new(bytes.Buffer) - if err := t.Execute(data, templateVars[0]); err != nil { - return err - } - - if err := yaml.Unmarshal(data.Bytes(), &r); err != nil { - return err - } - - if len(r.Architecture) == 0 { - return fmt.Errorf("Recipe file must have 'architecture' property") - } - - if len(r.Actions) == 0 { - return fmt.Errorf("Recipe file must have at least one action") - } - - return nil -} diff --git a/recipe/recipe_test.go b/recipe/recipe_test.go deleted file mode 100644 index 01457e9..0000000 --- a/recipe/recipe_test.go +++ /dev/null @@ -1,182 +0,0 @@ -package recipe_test - -import ( - "github.com/go-debos/debos/recipe" - "github.com/stretchr/testify/assert" - "io/ioutil" - "os" - "testing" -) - -type testRecipe struct { - recipe string - err string -} - -// Test if incorrect file has been passed -func TestParse_incorrect_file(t *testing.T) { - var err error - - var tests = []struct { - filename string - err string - }{ - { - "non-existing.yaml", - "open non-existing.yaml: no such file or directory", - }, - { - "/proc", - "read /proc: is a directory", - }, - } - - for _, test := range tests { - r := recipe.Recipe{} - err = r.Parse(test.filename) - assert.EqualError(t, err, test.err) - } -} - -// Check common recipe syntax -func TestParse_syntax(t *testing.T) { - - var tests = []testRecipe{ - // Test if all actions are supported - {` -architecture: arm64 - -actions: - - action: apt - - action: debootstrap - - action: download - - action: filesystem-deploy - - action: image-partition - - action: ostree-commit - - action: ostree-deploy - - action: overlay - - action: pack - - action: raw - - action: run - - action: unpack -`, - "", // Do not expect failure - }, - // Test of unknown action in list - {` -architecture: arm64 - -actions: - - action: test_unknown_action -`, - "Unknown action: test_unknown_action", - }, - // Test if 'architecture' property absence - {` -actions: - - action: raw -`, - "Recipe file must have 'architecture' property", - }, - // Test if no actions listed - {` -architecture: arm64 -`, - "Recipe file must have at least one action", - }, - // Test of wrong syntax in Yaml - {`wrong`, - "yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `wrong` into recipe.Recipe", - }, - // Test if no actions listed - {` -architecture: arm64 -`, - "Recipe file must have at least one action", - }, - } - - for _, test := range tests { - runTest(t, test) - } -} - -// Check template engine -func TestParse_template(t *testing.T) { - - var test = testRecipe{ - // Test template variables - ` -{{ $action:= or .action "download" }} -architecture: arm64 -actions: - - action: {{ $action }} -`, - "", // Do not expect failure - } - - { // Test of embedded template - r := runTest(t, test) - assert.Equalf(t, r.Actions[0].String(), "download", - "Fail to use embedded variable definition from recipe:%s\n", - test.recipe) - } - - { // Test of user-defined template variable - var templateVars = map[string]string{ - "action": "pack", - } - - r := runTest(t, test, templateVars) - assert.Equalf(t, r.Actions[0].String(), "pack", - "Fail to redefine variable with user-defined map:%s\n", - test.recipe) - } -} - -// Test of 'sector' function embedded to recipe package -func TestParse_sector(t *testing.T) { - var testSector = testRecipe{ - // Fail with unknown action - ` -architecture: arm64 - -actions: - - action: {{ sector 42 }} -`, - "Unknown action: 21504", - } - runTest(t, testSector) -} - -func runTest(t *testing.T, test testRecipe, templateVars ...map[string]string) recipe.Recipe { - file, err := ioutil.TempFile(os.TempDir(), "recipe") - assert.Empty(t, err) - defer os.Remove(file.Name()) - - file.WriteString(test.recipe) - file.Close() - - r := recipe.Recipe{} - if len(templateVars) == 0 { - err = r.Parse(file.Name()) - } else { - err = r.Parse(file.Name(), templateVars[0]) - } - - failed := false - - if len(test.err) > 0 { - // Expected error? - failed = !assert.EqualError(t, err, test.err) - } else { - // Unexpected error - failed = !assert.Empty(t, err) - } - - if failed { - t.Logf("Failed recipe:%s\n", test.recipe) - } - - return r -} -- cgit v1.2.3