diff options
Diffstat (limited to 'actions')
| -rw-r--r-- | actions/apt_action.go | 7 | ||||
| -rw-r--r-- | actions/debootstrap_action.go | 5 | ||||
| -rw-r--r-- | actions/ostree_commit_action.go | 36 | ||||
| -rw-r--r-- | actions/ostree_deploy_action.go | 8 | ||||
| -rw-r--r-- | actions/recipe.go | 267 | ||||
| -rw-r--r-- | actions/recipe_action.go | 143 | ||||
| -rw-r--r-- | actions/recipe_test.go | 370 |
7 files changed, 834 insertions, 2 deletions
diff --git a/actions/apt_action.go b/actions/apt_action.go index 681c069..6f3988f 100644 --- a/actions/apt_action.go +++ b/actions/apt_action.go @@ -6,6 +6,7 @@ Install packages and their dependencies to the target rootfs with 'apt'. Yaml syntax: - action: apt recommends: bool + unauthenticated: bool packages: - package1 - package2 @@ -17,6 +18,7 @@ Mandatory properties: Optional properties: - recommends -- boolean indicating if suggested packages will be installed +- unauthenticated -- boolean indicating if unauthenticated packages can be installed */ package actions @@ -27,6 +29,7 @@ import ( type AptAction struct { debos.BaseAction `yaml:",inline"` Recommends bool + Unauthenticated bool Packages []string } @@ -38,6 +41,10 @@ func (apt *AptAction) Run(context *debos.DebosContext) error { aptOptions = append(aptOptions, "--no-install-recommends") } + if apt.Unauthenticated { + aptOptions = append(aptOptions, "--allow-unauthenticated") + } + aptOptions = append(aptOptions, "install") aptOptions = append(aptOptions, apt.Packages...) diff --git a/actions/debootstrap_action.go b/actions/debootstrap_action.go index 02cbb15..b4d6730 100644 --- a/actions/debootstrap_action.go +++ b/actions/debootstrap_action.go @@ -81,6 +81,11 @@ func (d *DebootstrapAction) RunSecondStage(context debos.DebosContext) error { "--no-check-gpg", "--second-stage"} + if d.Components != nil { + s := strings.Join(d.Components, ",") + cmdline = append(cmdline, fmt.Sprintf("--components=%s", s)) + } + c := debos.NewChrootCommandForContext(context) // Can't use nspawn for debootstrap as it wants to create device nodes c.ChrootMethod = debos.CHROOT_METHOD_CHROOT diff --git a/actions/ostree_commit_action.go b/actions/ostree_commit_action.go index a0d8333..85b3fda 100644 --- a/actions/ostree_commit_action.go +++ b/actions/ostree_commit_action.go @@ -8,6 +8,13 @@ Yaml syntax: repository: repository name branch: branch name subject: commit message + collection-id: org.apertis.example + ref-binding: + - branch1 + - branch2 + metadata: + key: value + vendor.key: somevalue Mandatory properties: @@ -22,10 +29,18 @@ type (https://ostree.readthedocs.io/en/latest/manual/repo/#repository-types-and- Optional properties: - subject -- one line message with commit description. + +- collection-id -- Collection ID ref binding (requires libostree 2018.6). + +- ref-binding -- enforce that the commit was retrieved from one of the branch names in this array. + If 'collection-id' is set and 'ref-binding' is empty, will default to the branch name. + +- metadata -- key-value pairs of meta information to be added into commit. */ package actions import ( + "fmt" "log" "os" "path" @@ -40,6 +55,9 @@ type OstreeCommitAction struct { Branch string Subject string Command string + CollectionID string `yaml:"collection-id"` + RefBinding []string `yaml:"ref-binding"` + Metadata map[string]string } func emptyDir(dir string) { @@ -54,7 +72,7 @@ func emptyDir(dir string) { for _, f := range files { err := os.RemoveAll(path.Join(dir, f)) if err != nil { - log.Fatalf("Failed to remove file: %v", err) + log.Fatalf("Failed to remove file: %v", err) } } } @@ -77,6 +95,22 @@ func (ot *OstreeCommitAction) Run(context *debos.DebosContext) error { opts := otbuiltin.NewCommitOptions() opts.Subject = ot.Subject + for k, v := range ot.Metadata { + str := fmt.Sprintf("%s=%s", k, v) + opts.AddMetadataString = append(opts.AddMetadataString, str) + } + + if ot.CollectionID != "" { + opts.CollectionID = ot.CollectionID + if len(ot.RefBinding) == 0 { + // Add current branch if not explitely set via 'ref-binding' + opts.RefBinding = append(opts.RefBinding, ot.Branch) + } + } + + // Add values from 'ref-binding' if any + opts.RefBinding = append(opts.RefBinding, ot.RefBinding...) + ret, err := repo.Commit(context.Rootdir, ot.Branch, opts) if err != nil { return err diff --git a/actions/ostree_deploy_action.go b/actions/ostree_deploy_action.go index dab2265..7d6e6bd 100644 --- a/actions/ostree_deploy_action.go +++ b/actions/ostree_deploy_action.go @@ -18,6 +18,7 @@ Yaml syntax: setup-fstab: bool setup-kernel-cmdline: bool appendkernelcmdline: arguments + collection-id: org.apertis.example Mandatory properties: @@ -44,6 +45,8 @@ action to the configured commandline. - tls-client-cert-path -- path to client certificate to use for the remote repository - tls-client-key-path -- path to client certificate key to use for the remote repository + +- collection-id -- Collection ID ref binding (require libostree 2018.6). */ package actions @@ -70,6 +73,7 @@ type OstreeDeployAction struct { AppendKernelCmdline string `yaml:"append-kernel-cmdline"` TlsClientCertPath string `yaml:"tls-client-cert-path"` TlsClientKeyPath string `yaml:"tls-client-key-path"` + CollectionID string `yaml:"collection-id"` } func NewOstreeDeployAction() *OstreeDeployAction { @@ -140,7 +144,9 @@ func (ot *OstreeDeployAction) Run(context *debos.DebosContext) error { /* FIXME: add support for gpg signing commits so this is no longer needed */ opts := ostree.RemoteOptions{NoGpgVerify: true, TlsClientCertPath: ot.TlsClientCertPath, - TlsClientKeyPath: ot.TlsClientKeyPath} + TlsClientKeyPath: ot.TlsClientKeyPath, + CollectionId: ot.CollectionID, + } err = dstRepo.RemoteAdd("origin", ot.RemoteRepository, opts, nil) if err != nil { diff --git a/actions/recipe.go b/actions/recipe.go new file mode 100644 index 0000000..1da6a6b --- /dev/null +++ b/actions/recipe.go @@ -0,0 +1,267 @@ +/* +Package 'recipe' implements actions mapping to YAML recipe. + +Recipe syntax + +Recipe is a YAML file which is pre-processed though Golang +text templating engine (https://golang.org/pkg/text/template) + +Recipe is composed of 2 parts: + +- header + +- actions + +Comments are allowed and should be prefixed with '#' symbol. + + # Declare variable 'Var' + {{- $Var := "Value" -}} + + # Header + architecture: arm64 + + # Actions are executed in listed order + actions: + - action: ActionName1 + property1: true + + - action: ActionName2 + # Use value of variable 'Var' defined above + property2: {{$Var}} + +Mandatory properties for receipt: + +- architecture -- target architecture + +- actions -- at least one action should be listed + +Supported actions + +- apt -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Apt_Action + +- debootstrap -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Debootstrap_Action + +- download -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Download_Action + +- filesystem-deploy -- https://godoc.org/github.com/go-debos/debos/actions#hdr-FilesystemDeploy_Action + +- image-partition -- https://godoc.org/github.com/go-debos/debos/actions#hdr-ImagePartition_Action + +- ostree-commit -- https://godoc.org/github.com/go-debos/debos/actions#hdr-OstreeCommit_Action + +- ostree-deploy -- https://godoc.org/github.com/go-debos/debos/actions#hdr-OstreeDeploy_Action + +- overlay -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Overlay_Action + +- pack -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Pack_Action + +- raw -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Raw_Action + +- recipe -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Recipe_Action + +- run -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Run_Action + +- unpack -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Unpack_Action +*/ +package actions + +import ( + "bytes" + "fmt" + "github.com/go-debos/debos" + "gopkg.in/yaml.v2" + "path" + "text/template" + "log" + "strings" + "reflect" +) + +/* the YamlAction just embed the Action interface and implements the + * UnmarshalYAML function so it can select the concrete implementer of a + * specific action at unmarshaling time */ +type YamlAction struct { + debos.Action +} + +type Recipe struct { + Architecture string + Actions []YamlAction +} + +func (y *YamlAction) UnmarshalYAML(unmarshal func(interface{}) error) error { + var aux debos.BaseAction + + err := unmarshal(&aux) + if err != nil { + return err + } + + switch aux.Action { + case "debootstrap": + y.Action = NewDebootstrapAction() + case "pack": + y.Action = &PackAction{} + case "unpack": + y.Action = &UnpackAction{} + case "run": + y.Action = &RunAction{} + case "apt": + y.Action = &AptAction{} + case "ostree-commit": + y.Action = &OstreeCommitAction{} + case "ostree-deploy": + y.Action = NewOstreeDeployAction() + case "overlay": + y.Action = &OverlayAction{} + case "image-partition": + y.Action = &ImagePartitionAction{} + case "filesystem-deploy": + y.Action = NewFilesystemDeployAction() + case "raw": + y.Action = &RawAction{} + case "download": + y.Action = &DownloadAction{} + case "recipe": + y.Action = &RecipeAction{} + default: + return fmt.Errorf("Unknown action: %v", aux.Action) + } + + unmarshal(y.Action) + + return nil +} + +func sector(s int) int { + return s * 512 +} + +func DumpActionStruct(iface interface{}) string { + var a []string + + s := reflect.ValueOf(iface) + t := reflect.TypeOf(iface) + + for i := 0; i < t.NumField(); i++ { + f := s.Field(i) + // Dump only exported entries + if f.CanInterface() { + str := fmt.Sprintf("%s: %v", s.Type().Field(i).Name, f.Interface()) + a = append(a, str) + } + } + + return strings.Join(a, ", ") +} + +const tabs = 2 + +func DumpActions(iface interface{}, depth int) { + tab := strings.Repeat(" ", depth * tabs) + entries := reflect.ValueOf(iface) + + for i := 0; i < entries.NumField(); i++ { + if entries.Type().Field(i).Name == "Actions" { + log.Printf("%s %s:\n", tab, entries.Type().Field(i).Name) + actions := reflect.ValueOf(entries.Field(i).Interface()) + for j := 0; j < actions.Len(); j++ { + yaml := reflect.ValueOf(actions.Index(j).Interface()) + DumpActionFields(yaml.Field(0).Interface(), depth + 1) + } + } else { + log.Printf("%s %s: %v\n", tab, entries.Type().Field(i).Name, entries.Field(i).Interface()) + } + } +} + +func DumpActionFields(iface interface{}, depth int) { + tab := strings.Repeat(" ", depth * tabs) + entries := reflect.ValueOf(iface).Elem() + + for i := 0; i < entries.NumField(); i++ { + f := entries.Field(i) + // Dump only exported entries + if f.CanInterface() { + switch f.Kind() { + case reflect.Struct: + if entries.Type().Field(i).Type.String() == "debos.BaseAction" { + // BaseAction is the only struct embbed in Action ActionFields + // dump it at the same level + log.Printf("%s- %s", tab, DumpActionStruct(f.Interface())) + } + + case reflect.Slice: + s := reflect.ValueOf(f.Interface()) + if s.Len() > 0 && s.Index(0).Kind() == reflect.Struct { + log.Printf("%s %s:\n", tab, entries.Type().Field(i).Name) + for j := 0; j < s.Len(); j++ { + if s.Index(j).Kind() == reflect.Struct { + log.Printf("%s { %s }", tab, DumpActionStruct(s.Index(j).Interface())) + } + } + } else { + log.Printf("%s %s: %s\n", tab, entries.Type().Field(i).Name, f) + } + + default: + log.Printf("%s %s: %v\n", tab, entries.Type().Field(i).Name, f.Interface()) + } + } + } +} + +/* +Parse method reads YAML recipe file and map all steps to appropriate actions. + +- file -- is the path to configuration file + +- templateVars -- optional argument allowing to use custom map for templating +engine. Multiple template maps have no effect; only first map will be used. +*/ +func (r *Recipe) Parse(file string, printRecipe bool, dump bool, templateVars ...map[string]string) error { + t := template.New(path.Base(file)) + funcs := template.FuncMap{ + "sector": sector, + } + t.Funcs(funcs) + + if _, err := t.ParseFiles(file); err != nil { + return err + } + + if len(templateVars) == 0 { + templateVars = append(templateVars, make(map[string]string)) + } + + data := new(bytes.Buffer) + if err := t.Execute(data, templateVars[0]); err != nil { + return err + } + + if printRecipe || dump { + log.Printf("Recipe '%s':", file) + } + + if printRecipe { + log.Printf("%s", data) + } + + if err := yaml.Unmarshal(data.Bytes(), &r); err != nil { + return err + } + + if dump { + DumpActions(reflect.ValueOf(*r).Interface(), 0) + } + + if len(r.Architecture) == 0 { + return fmt.Errorf("Recipe file must have 'architecture' property") + } + + if len(r.Actions) == 0 { + return fmt.Errorf("Recipe file must have at least one action") + } + + return nil +} diff --git a/actions/recipe_action.go b/actions/recipe_action.go new file mode 100644 index 0000000..e48f650 --- /dev/null +++ b/actions/recipe_action.go @@ -0,0 +1,143 @@ +/* +Recipe Action + +Include a recipe. + +Yaml syntax: + - action: recipe + recipe: path to recipe + variables: + key: value + +Mandatory properties: + +- recipe -- includes the recipe actions at the given path. + +Optional properties: + +- variables -- overrides or adds new template variables. + +*/ +package actions + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "github.com/go-debos/debos" + "github.com/go-debos/fakemachine" +) + +type RecipeAction struct { + debos.BaseAction `yaml:",inline"` + Recipe string + Variables map[string]string + Actions Recipe `yaml:"-"` + templateVars map[string]string +} + +func (recipe *RecipeAction) Verify(context *debos.DebosContext) error { + if len(recipe.Recipe) == 0 { + return errors.New("'recipe' property can't be empty") + } + + file := recipe.Recipe + if !filepath.IsAbs(file) { + file = filepath.Clean(context.RecipeDir + "/" + recipe.Recipe) + } + + if _, err := os.Stat(file); os.IsNotExist(err) { + return err + } + + // Initialise template vars + recipe.templateVars = make(map[string]string) + recipe.templateVars["included_recipe"] = "true" + recipe.templateVars["architecture"] = context.Architecture + + // Add Variables to template vars + for k, v := range recipe.Variables { + recipe.templateVars[k] = v + } + + if err := recipe.Actions.Parse(file, context.PrintRecipe, context.Verbose, recipe.templateVars); err != nil { + return err + } + + if context.Architecture != recipe.Actions.Architecture { + return fmt.Errorf("Expect architecture '%s' but got '%s'", context.Architecture, recipe.Actions.Architecture) + } + + for _, a := range recipe.Actions.Actions { + if err := a.Verify(context); err != nil { + return err + } + } + + return nil +} + +func (recipe *RecipeAction) PreMachine(context *debos.DebosContext, m *fakemachine.Machine, args *[]string) error { + // TODO: check args? + + for _, a := range recipe.Actions.Actions { + if err := a.PreMachine(context, m, args); err != nil { + return err + } + } + + return nil +} + +func (recipe *RecipeAction) PreNoMachine(context *debos.DebosContext) error { + for _, a := range recipe.Actions.Actions { + if err := a.PreNoMachine(context); err != nil { + return err + } + } + + return nil +} + +func (recipe *RecipeAction) Run(context *debos.DebosContext) error { + recipe.LogStart() + + for _, a := range recipe.Actions.Actions { + if err := a.Run(context); err != nil { + return err + } + } + + return nil +} + +func (recipe *RecipeAction) Cleanup(context *debos.DebosContext) error { + for _, a := range recipe.Actions.Actions { + if err := a.Cleanup(context); err != nil { + return err + } + } + + return nil +} + +func (recipe *RecipeAction) PostMachine(context *debos.DebosContext) error { + for _, a := range recipe.Actions.Actions { + if err := a.PostMachine(context); err != nil { + return err + } + } + + return nil +} + +func (recipe *RecipeAction) PostMachineCleanup(context *debos.DebosContext) error { + for _, a := range recipe.Actions.Actions { + if err := a.PostMachineCleanup(context); err != nil { + return err + } + } + + return nil +} diff --git a/actions/recipe_test.go b/actions/recipe_test.go new file mode 100644 index 0000000..972bf61 --- /dev/null +++ b/actions/recipe_test.go @@ -0,0 +1,370 @@ +package actions_test + +import ( + "github.com/go-debos/debos" + "github.com/go-debos/debos/actions" + "github.com/stretchr/testify/assert" + "io/ioutil" + "os" + "testing" + "strings" +) + +type testRecipe struct { + recipe string + err string +} + +// Test if incorrect file has been passed +func TestParse_incorrect_file(t *testing.T) { + var err error + + var tests = []struct { + filename string + err string + }{ + { + "non-existing.yaml", + "open non-existing.yaml: no such file or directory", + }, + { + "/proc", + "read /proc: is a directory", + }, + } + + for _, test := range tests { + r := actions.Recipe{} + err = r.Parse(test.filename, false, false) + assert.EqualError(t, err, test.err) + } +} + +// Check common recipe syntax +func TestParse_syntax(t *testing.T) { + + var tests = []testRecipe{ + // Test if all actions are supported + {` +architecture: arm64 + +actions: + - action: apt + - action: debootstrap + - action: download + - action: filesystem-deploy + - action: image-partition + - action: ostree-commit + - action: ostree-deploy + - action: overlay + - action: pack + - action: raw + - action: run + - action: unpack + - action: recipe +`, + "", // Do not expect failure + }, + // Test of unknown action in list + {` +architecture: arm64 + +actions: + - action: test_unknown_action +`, + "Unknown action: test_unknown_action", + }, + // Test if 'architecture' property absence + {` +actions: + - action: raw +`, + "Recipe file must have 'architecture' property", + }, + // Test if no actions listed + {` +architecture: arm64 +`, + "Recipe file must have at least one action", + }, + // Test of wrong syntax in Yaml + {`wrong`, + "yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `wrong` into actions.Recipe", + }, + // Test if no actions listed + {` +architecture: arm64 +`, + "Recipe file must have at least one action", + }, + } + + for _, test := range tests { + runTest(t, test) + } +} + +// Check template engine +func TestParse_template(t *testing.T) { + + var test = testRecipe{ + // Test template variables + ` +{{ $action:= or .action "download" }} +architecture: arm64 +actions: + - action: {{ $action }} +`, + "", // Do not expect failure + } + + { // Test of embedded template + r := runTest(t, test) + assert.Equalf(t, r.Actions[0].String(), "download", + "Fail to use embedded variable definition from recipe:%s\n", + test.recipe) + } + + { // Test of user-defined template variable + var templateVars = map[string]string{ + "action": "pack", + } + + r := runTest(t, test, templateVars) + assert.Equalf(t, r.Actions[0].String(), "pack", + "Fail to redefine variable with user-defined map:%s\n", + test.recipe) + } +} + +// Test of 'sector' function embedded to recipe package +func TestParse_sector(t *testing.T) { + var testSector = testRecipe{ + // Fail with unknown action + ` +architecture: arm64 + +actions: + - action: {{ sector 42 }} +`, + "Unknown action: 21504", + } + runTest(t, testSector) +} + +func runTest(t *testing.T, test testRecipe, templateVars ...map[string]string) actions.Recipe { + file, err := ioutil.TempFile(os.TempDir(), "recipe") + assert.Empty(t, err) + defer os.Remove(file.Name()) + + file.WriteString(test.recipe) + file.Close() + + r := actions.Recipe{} + if len(templateVars) == 0 { + err = r.Parse(file.Name(), false, false) + } else { + err = r.Parse(file.Name(), false, false, templateVars[0]) + } + + failed := false + + if len(test.err) > 0 { + // Expected error? + failed = !assert.EqualError(t, err, test.err) + } else { + // Unexpected error + failed = !assert.Empty(t, err) + } + + if failed { + t.Logf("Failed recipe:%s\n", test.recipe) + } + + return r +} + +type subRecipe struct { + name string + recipe string +} + +type testSubRecipe struct { + recipe string + subrecipe subRecipe + err string +} + +func TestSubRecipe(t *testing.T) { + // Embedded recipes + var recipeAmd64 = subRecipe { + "amd64.yaml", + ` +architecture: amd64 + +actions: + - action: run + command: ok.sh +`, + } + var recipeInheritedArch = subRecipe { + "inherited.yaml", + ` +{{- $architecture := or .architecture "armhf" }} +architecture: {{ $architecture }} + +actions: + - action: run + command: ok.sh +`, + } + var recipeArmhf = subRecipe { + "armhf.yaml", + ` +architecture: armhf + +actions: + - action: run + command: ok.sh +`, + } + var recipeIncluded = subRecipe { + "included.yaml", + ` +{{- $included_recipe := or .included_recipe "false"}} +architecture: amd64 + +actions: + - action: run + command: ok.sh + {{- if ne $included_recipe "true" }} + - action: recipe + recipe: armhf.yaml + {{- end }} +`, + } + + // test recipes + var tests = []testSubRecipe { + { + // Test recipe same architecture OK + ` +architecture: amd64 + +actions: + - action: recipe + recipe: amd64.yaml +`, + recipeAmd64, + "", // Do not expect failure + }, + { + // Test recipe with inherited architecture OK + ` +architecture: amd64 + +actions: + - action: recipe + recipe: inherited.yaml +`, + recipeInheritedArch, + "", // Do not expect failure + }, + { + // Fail with unknown recipe + ` +architecture: amd64 + +actions: + - action: recipe + recipe: unknown_recipe.yaml +`, + recipeAmd64, + "stat /tmp/unknown_recipe.yaml: no such file or directory", + }, + { + // Fail with different architecture recipe + ` +architecture: amd64 + +actions: + - action: recipe + recipe: armhf.yaml +`, + recipeArmhf, + "Expect architecture 'amd64' but got 'armhf'", + }, + { + // Test included_recipe prevents parsing OK + ` +architecture: amd64 + +actions: + - action: recipe + recipe: included.yaml +`, + recipeIncluded, + "", // Do not expect failure + }, + } + + for _, test := range tests { + runTestWithSubRecipes(t, test) + } +} + +func runTestWithSubRecipes(t *testing.T, test testSubRecipe, templateVars ...map[string]string) actions.Recipe { + var context debos.DebosContext + dir, err := ioutil.TempDir("", "go-debos") + assert.Empty(t, err) + defer os.RemoveAll(dir) + + file, err := ioutil.TempFile(dir, "recipe") + assert.Empty(t, err) + defer os.Remove(file.Name()) + + file.WriteString(test.recipe) + file.Close() + + file_subrecipe, err := os.Create(dir + "/" + test.subrecipe.name) + assert.Empty(t, err) + defer os.Remove(file_subrecipe.Name()) + + file_subrecipe.WriteString(test.subrecipe.recipe) + file_subrecipe.Close() + + r := actions.Recipe{} + if len(templateVars) == 0 { + err = r.Parse(file.Name(), false, false) + } else { + err = r.Parse(file.Name(), false, false, templateVars[0]) + } + + // Should not expect error during parse + failed := !assert.Empty(t, err) + + if !failed { + context.Architecture = r.Architecture + context.RecipeDir = dir + + for _, a := range r.Actions { + if err = a.Verify(&context); err != nil { + break + } + } + + if len(test.err) > 0 { + // Expected error? + failed = !assert.EqualError(t, err, strings.Replace(test.err, "/tmp", dir, 1)) + } else { + // Unexpected error + failed = !assert.Empty(t, err) + } + } + + if failed { + t.Logf("Failed recipe:%s\n", test.recipe) + } + + return r +} |