merge(kontemplate): Integrate kontemplate at //depot/ops/kontemplate r/251

7 files changed, 513 insertions, 0 deletions

diff --git a/ops/kontemplate/templater/dns.go b/ops/kontemplate/templater/dns.go
new file mode 100644
index 000000000000..6cd974dd9324
--- /dev/null
+++ b/ops/kontemplate/templater/dns.go
@@ -0,0 +1,35 @@
+// Copyright (C) 2016-2019  Vincent Ambo <mail@tazj.in>
+//
+// This file is part of Kontemplate.
+//
+// Kontemplate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This file contains the implementation of a template function for retrieving
+// IP addresses from DNS
+
+package templater
+
+import (
+	"fmt"
+	"net"
+	"os"
+)
+
+func GetIPsFromDNS(host string) ([]interface{}, error) {
+	fmt.Fprintf(os.Stderr, "Attempting to look up IP for %s in DNS\n", host)
+	ips, err := net.LookupIP(host)
+
+	if err != nil {
+		return nil, fmt.Errorf("IP address lookup failed: %v", err)
+	}
+
+	var result []interface{} = make([]interface{}, len(ips))
+	for i, ip := range ips {
+		result[i] = ip
+	}
+
+	return result, nil
+}
diff --git a/ops/kontemplate/templater/pass.go b/ops/kontemplate/templater/pass.go
new file mode 100644
index 000000000000..f7fbcb433de5
--- /dev/null
+++ b/ops/kontemplate/templater/pass.go
@@ -0,0 +1,34 @@
+// Copyright (C) 2016-2019  Vincent Ambo <mail@tazj.in>
+//
+// This file is part of Kontemplate.
+//
+// Kontemplate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This file contains the implementation of a template function for retrieving
+// variables from 'pass', the standard UNIX password manager.
+
+package templater
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+func GetFromPass(key string) (string, error) {
+	fmt.Fprintf(os.Stderr, "Attempting to look up %s in pass\n", key)
+	pass := exec.Command("pass", "show", key)
+
+	output, err := pass.CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("Pass lookup failed: %s (%v)", output, err)
+	}
+
+	trimmed := strings.TrimSpace(string(output))
+
+	return trimmed, nil
+}
diff --git a/ops/kontemplate/templater/templater.go b/ops/kontemplate/templater/templater.go
new file mode 100644
index 000000000000..a8f0c670a603
--- /dev/null
+++ b/ops/kontemplate/templater/templater.go
@@ -0,0 +1,236 @@
+// Copyright (C) 2016-2019  Vincent Ambo <mail@tazj.in>
+//
+// This file is part of Kontemplate.
+//
+// Kontemplate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+package templater
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+	"text/template"
+
+	"github.com/Masterminds/sprig"
+	"github.com/tazjin/kontemplate/context"
+	"github.com/tazjin/kontemplate/util"
+)
+
+const failOnMissingKeys string = "missingkey=error"
+
+type RenderedResource struct {
+	Filename string
+	Rendered string
+}
+
+type RenderedResourceSet struct {
+	Name      string
+	Resources []RenderedResource
+	Args      []string
+}
+
+func LoadAndApplyTemplates(include *[]string, exclude *[]string, c *context.Context) ([]RenderedResourceSet, error) {
+	limitedResourceSets := applyLimits(&c.ResourceSets, include, exclude)
+	renderedResourceSets := make([]RenderedResourceSet, 0)
+
+	if len(*limitedResourceSets) == 0 {
+		return renderedResourceSets, fmt.Errorf("No valid resource sets included!")
+	}
+
+	for _, rs := range *limitedResourceSets {
+		set, err := processResourceSet(c, &rs)
+
+		if err != nil {
+			return nil, err
+		}
+
+		renderedResourceSets = append(renderedResourceSets, *set)
+	}
+
+	return renderedResourceSets, nil
+}
+
+func processResourceSet(ctx *context.Context, rs *context.ResourceSet) (*RenderedResourceSet, error) {
+	fmt.Fprintf(os.Stderr, "Loading resources for %s\n", rs.Name)
+
+	fileInfo, err := os.Stat(rs.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	var files []os.FileInfo
+	var resources []RenderedResource
+
+	// Treat single-file resource paths separately from resource
+	// sets containing multiple templates
+	if fileInfo.IsDir() {
+		// Explicitly discard this error, which will give us an empty
+		// list of files instead.
+		// This will end up printing a warning to the user, but it
+		// won't stop the rest of the process.
+		files, _ = ioutil.ReadDir(rs.Path)
+		resources, err = processFiles(ctx, rs, files)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		resource, err := templateFile(ctx, rs, rs.Path)
+		if err != nil {
+			return nil, err
+		}
+
+		resources = []RenderedResource{resource}
+	}
+
+	return &RenderedResourceSet{
+		Name:      rs.Name,
+		Resources: resources,
+		Args:      rs.Args,
+	}, nil
+}
+
+func processFiles(ctx *context.Context, rs *context.ResourceSet, files []os.FileInfo) ([]RenderedResource, error) {
+	resources := make([]RenderedResource, 0)
+
+	for _, file := range files {
+		if !file.IsDir() && isResourceFile(file) {
+			path := path.Join(rs.Path, file.Name())
+			res, err := templateFile(ctx, rs, path)
+
+			if err != nil {
+				return resources, err
+			}
+
+			resources = append(resources, res)
+		}
+	}
+
+	return resources, nil
+}
+
+func templateFile(ctx *context.Context, rs *context.ResourceSet, filepath string) (RenderedResource, error) {
+	var resource RenderedResource
+
+	tpl, err := template.New(path.Base(filepath)).Funcs(templateFuncs(ctx, rs)).Option(failOnMissingKeys).ParseFiles(filepath)
+	if err != nil {
+		return resource, fmt.Errorf("Could not load template %s: %v", filepath, err)
+	}
+
+	var b bytes.Buffer
+	err = tpl.Execute(&b, rs.Values)
+	if err != nil {
+		return resource, fmt.Errorf("Error while templating %s: %v", filepath, err)
+	}
+
+	resource = RenderedResource{
+		Filename: path.Base(filepath),
+		Rendered: b.String(),
+	}
+
+	return resource, nil
+}
+
+// Applies the limits of explicitly included or excluded resources and returns the updated resource set.
+// Exclude takes priority over include
+func applyLimits(rs *[]context.ResourceSet, include *[]string, exclude *[]string) *[]context.ResourceSet {
+	if len(*include) == 0 && len(*exclude) == 0 {
+		return rs
+	}
+
+	// Exclude excluded resource sets
+	excluded := make([]context.ResourceSet, 0)
+	for _, r := range *rs {
+		if !matchesResourceSet(exclude, &r) {
+			excluded = append(excluded, r)
+		}
+	}
+
+	// Include included resource sets
+	if len(*include) == 0 {
+		return &excluded
+	}
+	included := make([]context.ResourceSet, 0)
+	for _, r := range excluded {
+		if matchesResourceSet(include, &r) {
+			included = append(included, r)
+		}
+	}
+
+	return &included
+}
+
+// Check whether an include/exclude string slice matches a resource set
+func matchesResourceSet(s *[]string, rs *context.ResourceSet) bool {
+	for _, r := range *s {
+		r = strings.TrimSuffix(r, "/")
+		if r == rs.Name || r == rs.Parent {
+			return true
+		}
+	}
+
+	return false
+}
+
+func templateFuncs(c *context.Context, rs *context.ResourceSet) template.FuncMap {
+	m := sprig.TxtFuncMap()
+	m["json"] = func(data interface{}) string {
+		b, _ := json.Marshal(data)
+		return string(b)
+	}
+	m["passLookup"] = GetFromPass
+	m["gitHEAD"] = func() (string, error) {
+		out, err := exec.Command("git", "-C", c.BaseDir, "rev-parse", "HEAD").Output()
+		if err != nil {
+			return "", err
+		}
+		output := strings.TrimSpace(string(out))
+		return output, nil
+	}
+	m["lookupIPAddr"] = GetIPsFromDNS
+	m["insertFile"] = func(file string) (string, error) {
+		data, err := ioutil.ReadFile(path.Join(rs.Path, file))
+		if err != nil {
+			return "", err
+		}
+
+		return string(data), nil
+	}
+	m["insertTemplate"] = func(file string) (string, error) {
+		data, err := templateFile(c, rs, path.Join(rs.Path, file))
+		if err != nil {
+			return "", err
+		}
+
+		return data.Rendered, nil
+	}
+	m["default"] = func(defaultVal interface{}, varName string) interface{} {
+		if val, ok := rs.Values[varName]; ok {
+			return val
+		}
+
+		return defaultVal
+	}
+	return m
+}
+
+// Checks whether a file is a resource file (i.e. is YAML or JSON) and not a default values file.
+func isResourceFile(f os.FileInfo) bool {
+	for _, defaultFile := range util.DefaultFilenames {
+		if f.Name() == defaultFile {
+			return false
+		}
+	}
+
+	return strings.HasSuffix(f.Name(), "yaml") ||
+		strings.HasSuffix(f.Name(), "yml") ||
+		strings.HasSuffix(f.Name(), "json")
+}
diff --git a/ops/kontemplate/templater/templater_test.go b/ops/kontemplate/templater/templater_test.go
new file mode 100644
index 000000000000..c20858c203b4
--- /dev/null
+++ b/ops/kontemplate/templater/templater_test.go
@@ -0,0 +1,205 @@
+// Copyright (C) 2016-2019  Vincent Ambo <mail@tazj.in>
+//
+// This file is part of Kontemplate.
+//
+// Kontemplate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+package templater
+
+import (
+	"github.com/tazjin/kontemplate/context"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestApplyNoLimits(t *testing.T) {
+	resources := []context.ResourceSet{
+		{
+			Name: "testResourceSet1",
+		},
+		{
+			Name: "testResourceSet2",
+		},
+	}
+
+	result := applyLimits(&resources, &[]string{}, &[]string{})
+
+	if !reflect.DeepEqual(resources, *result) {
+		t.Error("Resource set slice changed, but shouldn't have.")
+		t.Errorf("Expected: %v\nResult: %v\n", resources, *result)
+		t.Fail()
+	}
+}
+
+func TestApplyIncludeLimits(t *testing.T) {
+	resources := []context.ResourceSet{
+		{
+			Name: "testResourceSet1",
+		},
+		{
+			Name: "testResourceSet2",
+		},
+		{
+			Name:   "testResourceSet3",
+			Parent: "included",
+		},
+	}
+
+	includes := []string{"testResourceSet1", "included"}
+
+	result := applyLimits(&resources, &includes, &[]string{})
+
+	expected := []context.ResourceSet{
+		{
+			Name: "testResourceSet1",
+		},
+		{
+			Name:   "testResourceSet3",
+			Parent: "included",
+		},
+	}
+
+	if !reflect.DeepEqual(expected, *result) {
+		t.Error("Result does not contain expected resource sets.")
+		t.Errorf("Expected: %v\nResult: %v\n", expected, *result)
+		t.Fail()
+	}
+}
+
+func TestApplyExcludeLimits(t *testing.T) {
+	resources := []context.ResourceSet{
+		{
+			Name: "testResourceSet1",
+		},
+		{
+			Name: "testResourceSet2",
+		},
+		{
+			Name:   "testResourceSet3",
+			Parent: "included",
+		},
+	}
+
+	exclude := []string{"testResourceSet2"}
+
+	result := applyLimits(&resources, &[]string{}, &exclude)
+
+	expected := []context.ResourceSet{
+		{
+			Name: "testResourceSet1",
+		},
+		{
+			Name:   "testResourceSet3",
+			Parent: "included",
+		},
+	}
+
+	if !reflect.DeepEqual(expected, *result) {
+		t.Error("Result does not contain expected resource sets.")
+		t.Errorf("Expected: %v\nResult: %v\n", expected, *result)
+		t.Fail()
+	}
+}
+
+func TestApplyLimitsExcludeIncludePrecedence(t *testing.T) {
+	resources := []context.ResourceSet{
+		{
+			Name:   "collection/nested1",
+			Parent: "collection",
+		},
+		{
+			Name:   "collection/nested2",
+			Parent: "collection",
+		},
+		{
+			Name:   "collection/nested3",
+			Parent: "collection",
+		},
+		{
+			Name: "something-else",
+		},
+	}
+
+	include := []string{"collection"}
+	exclude := []string{"collection/nested2"}
+
+	result := applyLimits(&resources, &include, &exclude)
+
+	expected := []context.ResourceSet{
+		{
+			Name:   "collection/nested1",
+			Parent: "collection",
+		},
+		{
+			Name:   "collection/nested3",
+			Parent: "collection",
+		},
+	}
+
+	if !reflect.DeepEqual(expected, *result) {
+		t.Error("Result does not contain expected resource sets.")
+		t.Errorf("Expected: %v\nResult: %v\n", expected, *result)
+		t.Fail()
+	}
+}
+
+func TestFailOnMissingKeys(t *testing.T) {
+	ctx := context.Context{}
+	resourceSet := context.ResourceSet{}
+
+	_, err := templateFile(&ctx, &resourceSet, "testdata/test-template.txt")
+
+	if err == nil {
+		t.Errorf("Template with missing keys should have failed.\n")
+		t.Fail()
+	}
+
+	if !strings.Contains(err.Error(), "map has no entry for key \"testName\"") {
+		t.Errorf("Templating failed with unexpected error: %v\n", err)
+	}
+}
+
+func TestDefaultTemplateFunction(t *testing.T) {
+	ctx := context.Context{}
+	resourceSet := context.ResourceSet{}
+
+	res, err := templateFile(&ctx, &resourceSet, "testdata/test-default.txt")
+
+	if err != nil {
+		t.Errorf("Templating with default values should have succeeded.\n")
+		t.Fail()
+	}
+
+	if res.Rendered != "defaultValue\n" {
+		t.Error("Result does not contain expected rendered default value.")
+		t.Fail()
+	}
+}
+
+func TestInsertTemplateFunction(t *testing.T) {
+	ctx := context.Context{}
+	resourceSet := context.ResourceSet{
+		Path: "testdata",
+		Values: map[string]interface{}{
+			"testName":        "TestInsertTemplateFunction",
+		},
+	}
+
+	res, err := templateFile(&ctx, &resourceSet, "testdata/test-insertTemplate.txt")
+
+	if err != nil {
+		t.Error(err)
+		t.Errorf("Templating with an insertTemplate call should have succeeded.\n")
+		t.Fail()
+	}
+
+	if res.Rendered != "Inserting \"Template for test TestInsertTemplateFunction\".\n" {
+		t.Error("Result does not contain expected rendered template value.")
+		t.Error(res.Rendered)
+		t.Fail()
+	}
+}
diff --git a/ops/kontemplate/templater/testdata/test-default.txt b/ops/kontemplate/templater/testdata/test-default.txt
new file mode 100644
index 000000000000..4f7997bd69d2
--- /dev/null
+++ b/ops/kontemplate/templater/testdata/test-default.txt
@@ -0,0 +1 @@
+{{ default "defaultValue" "missingVar" }}
diff --git a/ops/kontemplate/templater/testdata/test-insertTemplate.txt b/ops/kontemplate/templater/testdata/test-insertTemplate.txt
new file mode 100644
index 000000000000..8155e174fe00
--- /dev/null
+++ b/ops/kontemplate/templater/testdata/test-insertTemplate.txt
@@ -0,0 +1 @@
+Inserting "{{ insertTemplate "test-template.txt" | trim }}".
diff --git a/ops/kontemplate/templater/testdata/test-template.txt b/ops/kontemplate/templater/testdata/test-template.txt
new file mode 100644
index 000000000000..06f1cfc630c3
--- /dev/null
+++ b/ops/kontemplate/templater/testdata/test-template.txt
@@ -0,0 +1 @@
+Template for test {{ .testName }}