diff options
Diffstat (limited to 'ops/kontemplate/context')
18 files changed, 742 insertions, 0 deletions
diff --git a/ops/kontemplate/context/context.go b/ops/kontemplate/context/context.go new file mode 100644 index 000000000000..2d0378a0ec23 --- /dev/null +++ b/ops/kontemplate/context/context.go @@ -0,0 +1,266 @@ +// Copyright (C) 2016-2019 Vincent Ambo <mail@tazj.in> +// +// This file is part of Kontemplate. +// +// Kontemplate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package context + +import ( + "fmt" + "path" + "strings" + + "github.com/tazjin/kontemplate/util" +) + +type ResourceSet struct { + // Name of the resource set. This can be used in include/exclude statements during kontemplate runs. + Name string `json:"name"` + + // Path to the folder containing the files for this resource set. This defaults to the value of the 'name' field + // if unset. + Path string `json:"path"` + + // Values to include when interpolating resources from this resource set. + Values map[string]interface{} `json:"values"` + + // Args to pass on to kubectl for this resource set. + Args []string `json:"args"` + + // Nested resource sets to include + Include []ResourceSet `json:"include"` + + // Parent resource set for flattened resource sets. Should not be manually specified. + Parent string +} + +type Context struct { + // The name of the kubectl context + Name string `json:"context"` + + // Global variables that should be accessible by all resource sets + Global map[string]interface{} `json:"global"` + + // File names of YAML or JSON files including extra variables that should be globally accessible + VariableImportFiles []string `json:"import"` + + // The resource sets to include in this context + ResourceSets []ResourceSet `json:"include"` + + // Variables imported from additional files + ImportedVars map[string]interface{} + + // Explicitly set variables (via `--var`) that should override all others + ExplicitVars map[string]interface{} + + // This field represents the absolute path to the context base directory and should not be manually specified. + BaseDir string +} + +func contextLoadingError(filename string, cause error) error { + return fmt.Errorf("Context loading failed on file %s due to: \n%v", filename, cause) +} + +// Attempt to load and deserialise a Context from the specified file. +func LoadContext(filename string, explicitVars *[]string) (*Context, error) { + var ctx Context + err := util.LoadData(filename, &ctx) + + if err != nil { + return nil, contextLoadingError(filename, err) + } + + ctx.BaseDir = path.Dir(filename) + + // Prepare the resource sets by resolving parents etc. + ctx.ResourceSets = flattenPrepareResourceSetPaths(&ctx.BaseDir, &ctx.ResourceSets) + + // Add variables explicitly specified on the command line + ctx.ExplicitVars, err = loadExplicitVars(explicitVars) + if err != nil { + return nil, fmt.Errorf("Error setting explicit variables: %v\n", err) + } + + // Add variables loaded from import files + ctx.ImportedVars, err = ctx.loadImportedVariables() + if err != nil { + return nil, contextLoadingError(filename, err) + } + + // Merge variables defined at different levels. The + // `mergeContextValues` function is documented with the merge + // hierarchy. + ctx.ResourceSets = ctx.mergeContextValues() + + if err != nil { + return nil, contextLoadingError(filename, err) + } + + return &ctx, nil +} + +// Kontemplate supports specifying additional variable files with the +// `import` keyword. This function loads those variable files and +// merges them together with the context's other global variables. +func (ctx *Context) loadImportedVariables() (map[string]interface{}, error) { + allImportedVars := make(map[string]interface{}) + + for _, file := range ctx.VariableImportFiles { + // Ensure that the filename is not merged with the baseDir if + // it is set to an absolute path. + var filePath string + if path.IsAbs(file) { + filePath = file + } else { + filePath = path.Join(ctx.BaseDir, file) + } + + var importedVars map[string]interface{} + err := util.LoadData(filePath, &importedVars) + + if err != nil { + return nil, err + } + + allImportedVars = *util.Merge(&allImportedVars, &importedVars) + } + + return allImportedVars, nil +} + +// Correctly prepares the file paths for resource sets by inferring implicit paths and flattening resource set +// collections, i.e. resource sets that themselves have an additional 'include' field set. +// Those will be regarded as a short-hand for including multiple resource sets from a subfolder. +// See https://github.com/tazjin/kontemplate/issues/9 for more information. +func flattenPrepareResourceSetPaths(baseDir *string, rs *[]ResourceSet) []ResourceSet { + flattened := make([]ResourceSet, 0) + + for _, r := range *rs { + // If a path is not explicitly specified it should default to the resource set name. + // This is also the classic behaviour prior to kontemplate 1.2 + if r.Path == "" { + r.Path = r.Name + } + + // Paths are made absolute by resolving them relative to the context base, + // unless absolute paths were specified. + if !path.IsAbs(r.Path) { + r.Path = path.Join(*baseDir, r.Path) + } + + if len(r.Include) == 0 { + flattened = append(flattened, r) + } else { + for _, subResourceSet := range r.Include { + if subResourceSet.Path == "" { + subResourceSet.Path = subResourceSet.Name + } + + subResourceSet.Parent = r.Name + subResourceSet.Name = path.Join(r.Name, subResourceSet.Name) + subResourceSet.Path = path.Join(r.Path, subResourceSet.Path) + subResourceSet.Values = *util.Merge(&r.Values, &subResourceSet.Values) + flattened = append(flattened, subResourceSet) + } + } + } + + return flattened +} + +// Merges the context and resource set variables according in the +// desired precedence order. +// +// For now the reasoning behind the merge order is from least specific +// in relation to the cluster configuration, which means that the +// precedence is (in ascending order): +// +// 1. Default values in resource sets. +// 2. Values imported from files (via `import:`) +// 3. Global values in a cluster configuration +// 4. Values set in a resource set's `include`-section +// 5. Explicit values set on the CLI (`--var`) +// +// For a discussion on the reasoning behind this order, please consult +// https://github.com/tazjin/kontemplate/issues/142 +func (ctx *Context) mergeContextValues() []ResourceSet { + updated := make([]ResourceSet, len(ctx.ResourceSets)) + + // Merging has to happen separately for every individual + // resource set to make use of the default values: + for i, rs := range ctx.ResourceSets { + // Begin by loading default values from the resource + // sets configuration. + // + // Resource sets are used across different cluster + // contexts and the default values in them have the + // lowest precedence. + defaultValues := loadDefaultValues(&rs, ctx) + + // Continue by merging default values with values + // imported from external files. Those values are also + // used across cluster contexts, but have higher + // precedence than defaults. + merged := util.Merge(defaultValues, &ctx.ImportedVars) + + // Merge global values defined in the cluster context: + merged = util.Merge(merged, &ctx.Global) + + // Merge values configured in the resource set's + // `include` section: + merged = util.Merge(merged, &rs.Values) + + // Merge values defined explicitly on the CLI: + merged = util.Merge(merged, &ctx.ExplicitVars) + + // Continue with the newly merged resource set: + rs.Values = *merged + updated[i] = rs + } + + return updated +} + +// Loads default values for a resource set collection from +// path/to/set/default.{json|yaml}. +func loadDefaultValues(rs *ResourceSet, c *Context) *map[string]interface{} { + var defaultVars map[string]interface{} + + for _, filename := range util.DefaultFilenames { + err := util.LoadData(path.Join(rs.Path, filename), &defaultVars) + if err == nil { + return &defaultVars + } + } + + // The actual error is not inspected here. The reasoning for + // this is that in case of serious problems (e.g. permission + // issues with the folder / folder not existing) failure will + // occur a bit later anyways. + // + // Otherwise we'd have to differentiate between + // file-not-found-errors (no default values specified) and + // other errors here. + return &rs.Values +} + +// Prepares the variables specified explicitly via `--var` when +// executing kontemplate for adding to the context. +func loadExplicitVars(vars *[]string) (map[string]interface{}, error) { + explicitVars := make(map[string]interface{}, len(*vars)) + + for _, v := range *vars { + varParts := strings.SplitN(v, "=", 2) + if len(varParts) != 2 { + return nil, fmt.Errorf(`invalid explicit variable provided (%s), name and value should be separated with "="`, v) + } + + explicitVars[varParts[0]] = varParts[1] + } + + return explicitVars, nil +} diff --git a/ops/kontemplate/context/context_test.go b/ops/kontemplate/context/context_test.go new file mode 100644 index 000000000000..7ecd9d587d24 --- /dev/null +++ b/ops/kontemplate/context/context_test.go @@ -0,0 +1,353 @@ +// Copyright (C) 2016-2019 Vincent Ambo <mail@tazj.in> +// +// This file is part of Kontemplate. +// +// Kontemplate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package context + +import ( + "reflect" + "testing" +) + +var noExplicitVars []string = make([]string, 0) + +func TestLoadFlatContextFromFile(t *testing.T) { + ctx, err := LoadContext("testdata/flat-test.yaml", &noExplicitVars) + + if err != nil { + t.Error(err) + t.Fail() + } + + expected := Context{ + Name: "k8s.prod.mydomain.com", + Global: map[string]interface{}{ + "globalVar": "lizards", + }, + ResourceSets: []ResourceSet{ + { + Name: "some-api", + Path: "testdata/some-api", + Values: map[string]interface{}{ + "apiPort": float64(4567), // yep! + "importantFeature": true, + "version": "1.0-0e6884d", + "globalVar": "lizards", + }, + Include: nil, + Parent: "", + }, + }, + BaseDir: "testdata", + ImportedVars: make(map[string]interface{}, 0), + ExplicitVars: make(map[string]interface{}, 0), + } + + if !reflect.DeepEqual(*ctx, expected) { + t.Error("Loaded context and expected context did not match") + t.Fail() + } +} + +func TestLoadContextWithArgs(t *testing.T) { + ctx, err := LoadContext("testdata/flat-with-args-test.yaml", &noExplicitVars) + + if err != nil { + t.Error(err) + t.Fail() + } + + expected := Context{ + Name: "k8s.prod.mydomain.com", + ResourceSets: []ResourceSet{ + { + Name: "some-api", + Path: "testdata/some-api", + Values: make(map[string]interface{}, 0), + Args: []string{ + "--as=some-user", + "--as-group=hello:world", + "--as-banana", + "true", + }, + Include: nil, + Parent: "", + }, + }, + BaseDir: "testdata", + ImportedVars: make(map[string]interface{}, 0), + ExplicitVars: make(map[string]interface{}, 0), + } + + if !reflect.DeepEqual(*ctx, expected) { + t.Error("Loaded context and expected context did not match") + t.Fail() + } +} + +func TestLoadContextWithResourceSetCollections(t *testing.T) { + ctx, err := LoadContext("testdata/collections-test.yaml", &noExplicitVars) + + if err != nil { + t.Error(err) + t.Fail() + } + + expected := Context{ + Name: "k8s.prod.mydomain.com", + Global: map[string]interface{}{ + "globalVar": "lizards", + }, + ResourceSets: []ResourceSet{ + { + Name: "some-api", + Path: "testdata/some-api", + Values: map[string]interface{}{ + "apiPort": float64(4567), // yep! + "importantFeature": true, + "version": "1.0-0e6884d", + "globalVar": "lizards", + }, + Include: nil, + Parent: "", + }, + { + Name: "collection/nested", + Path: "testdata/collection/nested", + Values: map[string]interface{}{ + "lizards": "good", + "globalVar": "lizards", + }, + Include: nil, + Parent: "collection", + }, + }, + BaseDir: "testdata", + ImportedVars: make(map[string]interface{}, 0), + ExplicitVars: make(map[string]interface{}, 0), + } + + if !reflect.DeepEqual(*ctx, expected) { + t.Error("Loaded context and expected context did not match") + t.Fail() + } + +} + +func TestSubresourceVariableInheritance(t *testing.T) { + ctx, err := LoadContext("testdata/parent-variables.yaml", &noExplicitVars) + + if err != nil { + t.Error(err) + t.Fail() + } + + expected := Context{ + Name: "k8s.prod.mydomain.com", + ResourceSets: []ResourceSet{ + { + Name: "parent/child", + Path: "testdata/parent/child", + Values: map[string]interface{}{ + "foo": "bar", + "bar": "baz", + }, + Include: nil, + Parent: "parent", + }, + }, + BaseDir: "testdata", + ImportedVars: make(map[string]interface{}, 0), + ExplicitVars: make(map[string]interface{}, 0), + } + + if !reflect.DeepEqual(*ctx, expected) { + t.Error("Loaded and expected context did not match") + t.Fail() + } +} + +func TestSubresourceVariableInheritanceOverride(t *testing.T) { + ctx, err := LoadContext("testdata/parent-variable-override.yaml", &noExplicitVars) + + if err != nil { + t.Error(err) + t.Fail() + } + + expected := Context{ + Name: "k8s.prod.mydomain.com", + ResourceSets: []ResourceSet{ + { + Name: "parent/child", + Path: "testdata/parent/child", + Values: map[string]interface{}{ + "foo": "newvalue", + }, + Include: nil, + Parent: "parent", + }, + }, + BaseDir: "testdata", + ImportedVars: make(map[string]interface{}, 0), + ExplicitVars: make(map[string]interface{}, 0), + } + + if !reflect.DeepEqual(*ctx, expected) { + t.Error("Loaded and expected context did not match") + t.Fail() + } +} + +func TestDefaultValuesLoading(t *testing.T) { + ctx, err := LoadContext("testdata/default-loading.yaml", &noExplicitVars) + if err != nil { + t.Error(err) + t.Fail() + } + + rs := ctx.ResourceSets[0] + if rs.Values["defaultValues"] != "loaded" { + t.Errorf("Default values not loaded from YAML file") + t.Fail() + } + + if rs.Values["override"] != "notAtAll" { + t.Error("Default values should not override other values") + t.Fail() + } +} + +func TestImportValuesLoading(t *testing.T) { + ctx, err := LoadContext("testdata/import-vars-simple.yaml", &noExplicitVars) + if err != nil { + t.Error(err) + t.Fail() + } + + expected := map[string]interface{}{ + "override": "true", + "music": map[string]interface{}{ + "artist": "Pallida", + "track": "Tractor Beam", + }, + } + + if !reflect.DeepEqual(ctx.ImportedVars, expected) { + t.Error("Expected imported values after loading imports did not match!") + t.Fail() + } +} + +func TestExplicitPathLoading(t *testing.T) { + ctx, err := LoadContext("testdata/explicit-path.yaml", &noExplicitVars) + if err != nil { + t.Error(err) + t.Fail() + } + + expected := Context{ + Name: "k8s.prod.mydomain.com", + ResourceSets: []ResourceSet{ + { + Name: "some-api-europe", + Path: "testdata/some-api", + Values: map[string]interface{}{ + "location": "europe", + }, + Include: nil, + Parent: "", + }, + { + Name: "some-api-asia", + Path: "testdata/some-api", + Values: map[string]interface{}{ + "location": "asia", + }, + Include: nil, + Parent: "", + }, + }, + BaseDir: "testdata", + ImportedVars: make(map[string]interface{}, 0), + ExplicitVars: make(map[string]interface{}, 0), + } + + if !reflect.DeepEqual(*ctx, expected) { + t.Error("Loaded context and expected context did not match") + t.Fail() + } +} + +func TestExplicitSubresourcePathLoading(t *testing.T) { + ctx, err := LoadContext("testdata/explicit-subresource-path.yaml", &noExplicitVars) + if err != nil { + t.Error(err) + t.Fail() + } + + expected := Context{ + Name: "k8s.prod.mydomain.com", + ResourceSets: []ResourceSet{ + { + Name: "parent/child", + Path: "testdata/parent-path/child-path", + Parent: "parent", + Values: make(map[string]interface{}, 0), + }, + }, + BaseDir: "testdata", + ImportedVars: make(map[string]interface{}, 0), + ExplicitVars: make(map[string]interface{}, 0), + } + + if !reflect.DeepEqual(*ctx, expected) { + t.Error("Loaded context and expected context did not match") + t.Fail() + } +} + +func TestSetVariablesFromArguments(t *testing.T) { + vars := []string{"version=some-service-version"} + ctx, _ := LoadContext("testdata/default-loading.yaml", &vars) + + if version := ctx.ExplicitVars["version"]; version != "some-service-version" { + t.Errorf(`Expected variable "version" to have value "some-service-version" but was "%s"`, version) + } +} + +func TestSetInvalidVariablesFromArguments(t *testing.T) { + vars := []string{"version: some-service-version"} + _, err := LoadContext("testdata/default-loading.yaml", &vars) + + if err == nil { + t.Error("Expected invalid variable to return an error") + } +} + +// This test ensures that variables are merged in the correct order. +// Please consult the test data in `testdata/merging`. +func TestValueMergePrecedence(t *testing.T) { + cliVars:= []string{"cliVar=cliVar"} + ctx, _ := LoadContext("testdata/merging/context.yaml", &cliVars) + + expected := map[string]interface{}{ + "defaultVar": "defaultVar", + "importVar": "importVar", + "globalVar": "globalVar", + "includeVar": "includeVar", + "cliVar": "cliVar", + } + + result := ctx.ResourceSets[0].Values + + if !reflect.DeepEqual(expected, result) { + t.Errorf("Merged values did not match expected result: \n%v", result) + t.Fail() + } +} diff --git a/ops/kontemplate/context/testdata/collections-test.yaml b/ops/kontemplate/context/testdata/collections-test.yaml new file mode 100644 index 000000000000..a619c8cfddcc --- /dev/null +++ b/ops/kontemplate/context/testdata/collections-test.yaml @@ -0,0 +1,15 @@ +--- +context: k8s.prod.mydomain.com +global: + globalVar: lizards +include: + - name: some-api + values: + version: 1.0-0e6884d + importantFeature: true + apiPort: 4567 + - name: collection + include: + - name: nested + values: + lizards: good diff --git a/ops/kontemplate/context/testdata/default-loading.yaml b/ops/kontemplate/context/testdata/default-loading.yaml new file mode 100644 index 000000000000..d589c99b4eaf --- /dev/null +++ b/ops/kontemplate/context/testdata/default-loading.yaml @@ -0,0 +1,6 @@ +--- +context: default-loading +include: + - name: default + values: + override: notAtAll \ No newline at end of file diff --git a/ops/kontemplate/context/testdata/default/default.yaml b/ops/kontemplate/context/testdata/default/default.yaml new file mode 100644 index 000000000000..0ffa3cd81f24 --- /dev/null +++ b/ops/kontemplate/context/testdata/default/default.yaml @@ -0,0 +1,2 @@ +defaultValues: loaded +override: noop \ No newline at end of file diff --git a/ops/kontemplate/context/testdata/explicit-path.yaml b/ops/kontemplate/context/testdata/explicit-path.yaml new file mode 100644 index 000000000000..2c81f83c0919 --- /dev/null +++ b/ops/kontemplate/context/testdata/explicit-path.yaml @@ -0,0 +1,11 @@ +--- +context: k8s.prod.mydomain.com +include: + - name: some-api-europe + path: some-api + values: + location: europe + - name: some-api-asia + path: some-api + values: + location: asia diff --git a/ops/kontemplate/context/testdata/explicit-subresource-path.yaml b/ops/kontemplate/context/testdata/explicit-subresource-path.yaml new file mode 100644 index 000000000000..6cf86183229e --- /dev/null +++ b/ops/kontemplate/context/testdata/explicit-subresource-path.yaml @@ -0,0 +1,8 @@ +--- +context: k8s.prod.mydomain.com +include: + - name: parent + path: parent-path + include: + - name: child + path: child-path diff --git a/ops/kontemplate/context/testdata/flat-test.yaml b/ops/kontemplate/context/testdata/flat-test.yaml new file mode 100644 index 000000000000..dd7804f719c3 --- /dev/null +++ b/ops/kontemplate/context/testdata/flat-test.yaml @@ -0,0 +1,10 @@ +--- +context: k8s.prod.mydomain.com +global: + globalVar: lizards +include: + - name: some-api + values: + version: 1.0-0e6884d + importantFeature: true + apiPort: 4567 diff --git a/ops/kontemplate/context/testdata/flat-with-args-test.yaml b/ops/kontemplate/context/testdata/flat-with-args-test.yaml new file mode 100644 index 000000000000..29d3334fb54d --- /dev/null +++ b/ops/kontemplate/context/testdata/flat-with-args-test.yaml @@ -0,0 +1,9 @@ +--- +context: k8s.prod.mydomain.com +include: + - name: some-api + args: + - --as=some-user + - --as-group=hello:world + - --as-banana + - "true" diff --git a/ops/kontemplate/context/testdata/import-vars-simple.yaml b/ops/kontemplate/context/testdata/import-vars-simple.yaml new file mode 100644 index 000000000000..12244e1ab174 --- /dev/null +++ b/ops/kontemplate/context/testdata/import-vars-simple.yaml @@ -0,0 +1,5 @@ +--- +context: k8s.prod.mydomain.com +import: + - test-vars.yaml +include: [] diff --git a/ops/kontemplate/context/testdata/merging/context.yaml b/ops/kontemplate/context/testdata/merging/context.yaml new file mode 100644 index 000000000000..df30d3d8cbe3 --- /dev/null +++ b/ops/kontemplate/context/testdata/merging/context.yaml @@ -0,0 +1,15 @@ +# This context file is intended to test the merge hierarchy of +# variables defined at different levels. +--- +context: merging.in.kontemplate.works +global: + globalVar: globalVar + includeVar: should be overridden (global) + cliVar: should be overridden (global) +import: + - import-vars.yaml +include: + - name: resource + values: + includeVar: includeVar + cliVar: should be overridden (include) diff --git a/ops/kontemplate/context/testdata/merging/import-vars.yaml b/ops/kontemplate/context/testdata/merging/import-vars.yaml new file mode 100644 index 000000000000..2a51352571a6 --- /dev/null +++ b/ops/kontemplate/context/testdata/merging/import-vars.yaml @@ -0,0 +1,4 @@ +importVar: importVar +globalVar: should be overridden (import) +includeVar: should be overridden (import) +cliVar: should be overridden (import) diff --git a/ops/kontemplate/context/testdata/merging/resource/default.yaml b/ops/kontemplate/context/testdata/merging/resource/default.yaml new file mode 100644 index 000000000000..040a19aaba25 --- /dev/null +++ b/ops/kontemplate/context/testdata/merging/resource/default.yaml @@ -0,0 +1,5 @@ +defaultVar: defaultVar +importVar: should be overridden (default) +globalVar: should be overridden (default) +includeVar: should be overridden (default) +cliVar: should be overridden (default) diff --git a/ops/kontemplate/context/testdata/merging/resource/output.yaml b/ops/kontemplate/context/testdata/merging/resource/output.yaml new file mode 100644 index 000000000000..5920b2720780 --- /dev/null +++ b/ops/kontemplate/context/testdata/merging/resource/output.yaml @@ -0,0 +1,5 @@ +defaultVar: {{ .defaultVar }} +importVar: {{ .importVar }} +globalVar: {{ .globalVar }} +includeVar: {{ .includeVar }} +cliVar: {{ .cliVar }} diff --git a/ops/kontemplate/context/testdata/parent-variable-override.yaml b/ops/kontemplate/context/testdata/parent-variable-override.yaml new file mode 100644 index 000000000000..42676c3028fe --- /dev/null +++ b/ops/kontemplate/context/testdata/parent-variable-override.yaml @@ -0,0 +1,10 @@ +--- +context: k8s.prod.mydomain.com +include: + - name: parent + values: + foo: bar + include: + - name: child + values: + foo: newvalue diff --git a/ops/kontemplate/context/testdata/parent-variables.yaml b/ops/kontemplate/context/testdata/parent-variables.yaml new file mode 100644 index 000000000000..8459fd30405b --- /dev/null +++ b/ops/kontemplate/context/testdata/parent-variables.yaml @@ -0,0 +1,10 @@ +--- +context: k8s.prod.mydomain.com +include: + - name: parent + values: + foo: bar + include: + - name: child + values: + bar: baz diff --git a/ops/kontemplate/context/testdata/test-vars-override.yaml b/ops/kontemplate/context/testdata/test-vars-override.yaml new file mode 100644 index 000000000000..5215c559c136 --- /dev/null +++ b/ops/kontemplate/context/testdata/test-vars-override.yaml @@ -0,0 +1,3 @@ +--- +override: 3 +place: Oslo diff --git a/ops/kontemplate/context/testdata/test-vars.yaml b/ops/kontemplate/context/testdata/test-vars.yaml new file mode 100644 index 000000000000..af27bdc455bf --- /dev/null +++ b/ops/kontemplate/context/testdata/test-vars.yaml @@ -0,0 +1,5 @@ +--- +override: 'true' +music: + artist: Pallida + track: Tractor Beam |