From 795a97466527a5f02e79e47b7fb316c78ffde667 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Fri, 20 Dec 2019 22:13:07 +0000 Subject: chore(kontemplate): Prepare kontemplate for depot-merge This merge will not yet include moving over to buildGo.nix, as support for testing and such is not present in that library yet. --- ops/kontemplate/templater/dns.go | 35 +++ ops/kontemplate/templater/pass.go | 34 +++ ops/kontemplate/templater/templater.go | 236 +++++++++++++++++++++ ops/kontemplate/templater/templater_test.go | 205 ++++++++++++++++++ .../templater/testdata/test-default.txt | 1 + .../templater/testdata/test-insertTemplate.txt | 1 + .../templater/testdata/test-template.txt | 1 + 7 files changed, 513 insertions(+) create mode 100644 ops/kontemplate/templater/dns.go create mode 100644 ops/kontemplate/templater/pass.go create mode 100644 ops/kontemplate/templater/templater.go create mode 100644 ops/kontemplate/templater/templater_test.go create mode 100644 ops/kontemplate/templater/testdata/test-default.txt create mode 100644 ops/kontemplate/templater/testdata/test-insertTemplate.txt create mode 100644 ops/kontemplate/templater/testdata/test-template.txt (limited to 'ops/kontemplate/templater') diff --git a/ops/kontemplate/templater/dns.go b/ops/kontemplate/templater/dns.go new file mode 100644 index 000000000000..6cd974dd9324 --- /dev/null +++ b/ops/kontemplate/templater/dns.go @@ -0,0 +1,35 @@ +// Copyright (C) 2016-2019 Vincent Ambo +// +// This file is part of Kontemplate. +// +// Kontemplate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This file contains the implementation of a template function for retrieving +// IP addresses from DNS + +package templater + +import ( + "fmt" + "net" + "os" +) + +func GetIPsFromDNS(host string) ([]interface{}, error) { + fmt.Fprintf(os.Stderr, "Attempting to look up IP for %s in DNS\n", host) + ips, err := net.LookupIP(host) + + if err != nil { + return nil, fmt.Errorf("IP address lookup failed: %v", err) + } + + var result []interface{} = make([]interface{}, len(ips)) + for i, ip := range ips { + result[i] = ip + } + + return result, nil +} diff --git a/ops/kontemplate/templater/pass.go b/ops/kontemplate/templater/pass.go new file mode 100644 index 000000000000..f7fbcb433de5 --- /dev/null +++ b/ops/kontemplate/templater/pass.go @@ -0,0 +1,34 @@ +// Copyright (C) 2016-2019 Vincent Ambo +// +// This file is part of Kontemplate. +// +// Kontemplate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This file contains the implementation of a template function for retrieving +// variables from 'pass', the standard UNIX password manager. + +package templater + +import ( + "fmt" + "os" + "os/exec" + "strings" +) + +func GetFromPass(key string) (string, error) { + fmt.Fprintf(os.Stderr, "Attempting to look up %s in pass\n", key) + pass := exec.Command("pass", "show", key) + + output, err := pass.CombinedOutput() + if err != nil { + return "", fmt.Errorf("Pass lookup failed: %s (%v)", output, err) + } + + trimmed := strings.TrimSpace(string(output)) + + return trimmed, nil +} diff --git a/ops/kontemplate/templater/templater.go b/ops/kontemplate/templater/templater.go new file mode 100644 index 000000000000..a8f0c670a603 --- /dev/null +++ b/ops/kontemplate/templater/templater.go @@ -0,0 +1,236 @@ +// Copyright (C) 2016-2019 Vincent Ambo +// +// This file is part of Kontemplate. +// +// Kontemplate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package templater + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "strings" + "text/template" + + "github.com/Masterminds/sprig" + "github.com/tazjin/kontemplate/context" + "github.com/tazjin/kontemplate/util" +) + +const failOnMissingKeys string = "missingkey=error" + +type RenderedResource struct { + Filename string + Rendered string +} + +type RenderedResourceSet struct { + Name string + Resources []RenderedResource + Args []string +} + +func LoadAndApplyTemplates(include *[]string, exclude *[]string, c *context.Context) ([]RenderedResourceSet, error) { + limitedResourceSets := applyLimits(&c.ResourceSets, include, exclude) + renderedResourceSets := make([]RenderedResourceSet, 0) + + if len(*limitedResourceSets) == 0 { + return renderedResourceSets, fmt.Errorf("No valid resource sets included!") + } + + for _, rs := range *limitedResourceSets { + set, err := processResourceSet(c, &rs) + + if err != nil { + return nil, err + } + + renderedResourceSets = append(renderedResourceSets, *set) + } + + return renderedResourceSets, nil +} + +func processResourceSet(ctx *context.Context, rs *context.ResourceSet) (*RenderedResourceSet, error) { + fmt.Fprintf(os.Stderr, "Loading resources for %s\n", rs.Name) + + fileInfo, err := os.Stat(rs.Path) + if err != nil { + return nil, err + } + + var files []os.FileInfo + var resources []RenderedResource + + // Treat single-file resource paths separately from resource + // sets containing multiple templates + if fileInfo.IsDir() { + // Explicitly discard this error, which will give us an empty + // list of files instead. + // This will end up printing a warning to the user, but it + // won't stop the rest of the process. + files, _ = ioutil.ReadDir(rs.Path) + resources, err = processFiles(ctx, rs, files) + if err != nil { + return nil, err + } + } else { + resource, err := templateFile(ctx, rs, rs.Path) + if err != nil { + return nil, err + } + + resources = []RenderedResource{resource} + } + + return &RenderedResourceSet{ + Name: rs.Name, + Resources: resources, + Args: rs.Args, + }, nil +} + +func processFiles(ctx *context.Context, rs *context.ResourceSet, files []os.FileInfo) ([]RenderedResource, error) { + resources := make([]RenderedResource, 0) + + for _, file := range files { + if !file.IsDir() && isResourceFile(file) { + path := path.Join(rs.Path, file.Name()) + res, err := templateFile(ctx, rs, path) + + if err != nil { + return resources, err + } + + resources = append(resources, res) + } + } + + return resources, nil +} + +func templateFile(ctx *context.Context, rs *context.ResourceSet, filepath string) (RenderedResource, error) { + var resource RenderedResource + + tpl, err := template.New(path.Base(filepath)).Funcs(templateFuncs(ctx, rs)).Option(failOnMissingKeys).ParseFiles(filepath) + if err != nil { + return resource, fmt.Errorf("Could not load template %s: %v", filepath, err) + } + + var b bytes.Buffer + err = tpl.Execute(&b, rs.Values) + if err != nil { + return resource, fmt.Errorf("Error while templating %s: %v", filepath, err) + } + + resource = RenderedResource{ + Filename: path.Base(filepath), + Rendered: b.String(), + } + + return resource, nil +} + +// Applies the limits of explicitly included or excluded resources and returns the updated resource set. +// Exclude takes priority over include +func applyLimits(rs *[]context.ResourceSet, include *[]string, exclude *[]string) *[]context.ResourceSet { + if len(*include) == 0 && len(*exclude) == 0 { + return rs + } + + // Exclude excluded resource sets + excluded := make([]context.ResourceSet, 0) + for _, r := range *rs { + if !matchesResourceSet(exclude, &r) { + excluded = append(excluded, r) + } + } + + // Include included resource sets + if len(*include) == 0 { + return &excluded + } + included := make([]context.ResourceSet, 0) + for _, r := range excluded { + if matchesResourceSet(include, &r) { + included = append(included, r) + } + } + + return &included +} + +// Check whether an include/exclude string slice matches a resource set +func matchesResourceSet(s *[]string, rs *context.ResourceSet) bool { + for _, r := range *s { + r = strings.TrimSuffix(r, "/") + if r == rs.Name || r == rs.Parent { + return true + } + } + + return false +} + +func templateFuncs(c *context.Context, rs *context.ResourceSet) template.FuncMap { + m := sprig.TxtFuncMap() + m["json"] = func(data interface{}) string { + b, _ := json.Marshal(data) + return string(b) + } + m["passLookup"] = GetFromPass + m["gitHEAD"] = func() (string, error) { + out, err := exec.Command("git", "-C", c.BaseDir, "rev-parse", "HEAD").Output() + if err != nil { + return "", err + } + output := strings.TrimSpace(string(out)) + return output, nil + } + m["lookupIPAddr"] = GetIPsFromDNS + m["insertFile"] = func(file string) (string, error) { + data, err := ioutil.ReadFile(path.Join(rs.Path, file)) + if err != nil { + return "", err + } + + return string(data), nil + } + m["insertTemplate"] = func(file string) (string, error) { + data, err := templateFile(c, rs, path.Join(rs.Path, file)) + if err != nil { + return "", err + } + + return data.Rendered, nil + } + m["default"] = func(defaultVal interface{}, varName string) interface{} { + if val, ok := rs.Values[varName]; ok { + return val + } + + return defaultVal + } + return m +} + +// Checks whether a file is a resource file (i.e. is YAML or JSON) and not a default values file. +func isResourceFile(f os.FileInfo) bool { + for _, defaultFile := range util.DefaultFilenames { + if f.Name() == defaultFile { + return false + } + } + + return strings.HasSuffix(f.Name(), "yaml") || + strings.HasSuffix(f.Name(), "yml") || + strings.HasSuffix(f.Name(), "json") +} diff --git a/ops/kontemplate/templater/templater_test.go b/ops/kontemplate/templater/templater_test.go new file mode 100644 index 000000000000..c20858c203b4 --- /dev/null +++ b/ops/kontemplate/templater/templater_test.go @@ -0,0 +1,205 @@ +// Copyright (C) 2016-2019 Vincent Ambo +// +// This file is part of Kontemplate. +// +// Kontemplate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package templater + +import ( + "github.com/tazjin/kontemplate/context" + "reflect" + "strings" + "testing" +) + +func TestApplyNoLimits(t *testing.T) { + resources := []context.ResourceSet{ + { + Name: "testResourceSet1", + }, + { + Name: "testResourceSet2", + }, + } + + result := applyLimits(&resources, &[]string{}, &[]string{}) + + if !reflect.DeepEqual(resources, *result) { + t.Error("Resource set slice changed, but shouldn't have.") + t.Errorf("Expected: %v\nResult: %v\n", resources, *result) + t.Fail() + } +} + +func TestApplyIncludeLimits(t *testing.T) { + resources := []context.ResourceSet{ + { + Name: "testResourceSet1", + }, + { + Name: "testResourceSet2", + }, + { + Name: "testResourceSet3", + Parent: "included", + }, + } + + includes := []string{"testResourceSet1", "included"} + + result := applyLimits(&resources, &includes, &[]string{}) + + expected := []context.ResourceSet{ + { + Name: "testResourceSet1", + }, + { + Name: "testResourceSet3", + Parent: "included", + }, + } + + if !reflect.DeepEqual(expected, *result) { + t.Error("Result does not contain expected resource sets.") + t.Errorf("Expected: %v\nResult: %v\n", expected, *result) + t.Fail() + } +} + +func TestApplyExcludeLimits(t *testing.T) { + resources := []context.ResourceSet{ + { + Name: "testResourceSet1", + }, + { + Name: "testResourceSet2", + }, + { + Name: "testResourceSet3", + Parent: "included", + }, + } + + exclude := []string{"testResourceSet2"} + + result := applyLimits(&resources, &[]string{}, &exclude) + + expected := []context.ResourceSet{ + { + Name: "testResourceSet1", + }, + { + Name: "testResourceSet3", + Parent: "included", + }, + } + + if !reflect.DeepEqual(expected, *result) { + t.Error("Result does not contain expected resource sets.") + t.Errorf("Expected: %v\nResult: %v\n", expected, *result) + t.Fail() + } +} + +func TestApplyLimitsExcludeIncludePrecedence(t *testing.T) { + resources := []context.ResourceSet{ + { + Name: "collection/nested1", + Parent: "collection", + }, + { + Name: "collection/nested2", + Parent: "collection", + }, + { + Name: "collection/nested3", + Parent: "collection", + }, + { + Name: "something-else", + }, + } + + include := []string{"collection"} + exclude := []string{"collection/nested2"} + + result := applyLimits(&resources, &include, &exclude) + + expected := []context.ResourceSet{ + { + Name: "collection/nested1", + Parent: "collection", + }, + { + Name: "collection/nested3", + Parent: "collection", + }, + } + + if !reflect.DeepEqual(expected, *result) { + t.Error("Result does not contain expected resource sets.") + t.Errorf("Expected: %v\nResult: %v\n", expected, *result) + t.Fail() + } +} + +func TestFailOnMissingKeys(t *testing.T) { + ctx := context.Context{} + resourceSet := context.ResourceSet{} + + _, err := templateFile(&ctx, &resourceSet, "testdata/test-template.txt") + + if err == nil { + t.Errorf("Template with missing keys should have failed.\n") + t.Fail() + } + + if !strings.Contains(err.Error(), "map has no entry for key \"testName\"") { + t.Errorf("Templating failed with unexpected error: %v\n", err) + } +} + +func TestDefaultTemplateFunction(t *testing.T) { + ctx := context.Context{} + resourceSet := context.ResourceSet{} + + res, err := templateFile(&ctx, &resourceSet, "testdata/test-default.txt") + + if err != nil { + t.Errorf("Templating with default values should have succeeded.\n") + t.Fail() + } + + if res.Rendered != "defaultValue\n" { + t.Error("Result does not contain expected rendered default value.") + t.Fail() + } +} + +func TestInsertTemplateFunction(t *testing.T) { + ctx := context.Context{} + resourceSet := context.ResourceSet{ + Path: "testdata", + Values: map[string]interface{}{ + "testName": "TestInsertTemplateFunction", + }, + } + + res, err := templateFile(&ctx, &resourceSet, "testdata/test-insertTemplate.txt") + + if err != nil { + t.Error(err) + t.Errorf("Templating with an insertTemplate call should have succeeded.\n") + t.Fail() + } + + if res.Rendered != "Inserting \"Template for test TestInsertTemplateFunction\".\n" { + t.Error("Result does not contain expected rendered template value.") + t.Error(res.Rendered) + t.Fail() + } +} diff --git a/ops/kontemplate/templater/testdata/test-default.txt b/ops/kontemplate/templater/testdata/test-default.txt new file mode 100644 index 000000000000..4f7997bd69d2 --- /dev/null +++ b/ops/kontemplate/templater/testdata/test-default.txt @@ -0,0 +1 @@ +{{ default "defaultValue" "missingVar" }} diff --git a/ops/kontemplate/templater/testdata/test-insertTemplate.txt b/ops/kontemplate/templater/testdata/test-insertTemplate.txt new file mode 100644 index 000000000000..8155e174fe00 --- /dev/null +++ b/ops/kontemplate/templater/testdata/test-insertTemplate.txt @@ -0,0 +1 @@ +Inserting "{{ insertTemplate "test-template.txt" | trim }}". diff --git a/ops/kontemplate/templater/testdata/test-template.txt b/ops/kontemplate/templater/testdata/test-template.txt new file mode 100644 index 000000000000..06f1cfc630c3 --- /dev/null +++ b/ops/kontemplate/templater/testdata/test-template.txt @@ -0,0 +1 @@ +Template for test {{ .testName }} -- cgit 1.4.1