about summary refs log tree commit diff
diff options
context:
space:
mode:
-rw-r--r--tools/nixery/.travis.yml1
-rw-r--r--tools/nixery/build-image/default.nix5
-rw-r--r--tools/nixery/build-image/load-pkgs.nix54
-rw-r--r--tools/nixery/default.nix3
-rw-r--r--tools/nixery/server/builder/builder.go7
-rw-r--r--tools/nixery/server/config/config.go66
-rw-r--r--tools/nixery/server/config/pkgsource.go155
-rw-r--r--tools/nixery/server/main.go6
8 files changed, 192 insertions, 105 deletions
diff --git a/tools/nixery/.travis.yml b/tools/nixery/.travis.yml
index 471037f36474..f670ab0e2cf5 100644
--- a/tools/nixery/.travis.yml
+++ b/tools/nixery/.travis.yml
@@ -31,6 +31,7 @@ script:
       -e GOOGLE_APPLICATION_CREDENTIALS=/var/nixery/key.json \
       -e GCS_SIGNING_ACCOUNT="${GCS_SIGNING_ACCOUNT}" \
       -e GCS_SIGNING_KEY=/var/nixery/gcs.pem \
+      -e NIXERY_CHANNEL=nixos-unstable \
       ${IMG}
 
   # print all of the container's logs regardless of success
diff --git a/tools/nixery/build-image/default.nix b/tools/nixery/build-image/default.nix
index 0d3002cb404e..6b1cea6f0ca2 100644
--- a/tools/nixery/build-image/default.nix
+++ b/tools/nixery/build-image/default.nix
@@ -20,11 +20,12 @@
 
   # Because of the insanity occuring below, this function must mirror
   # all arguments of build-image.nix.
-, pkgSource ? "nixpkgs!nixos-19.03"
+, srcType ? "nixpkgs"
+, srcArgs ? "nixos-19.03"
 , tag ? null, name ? null, packages ? null, maxLayers ? null
 }@args:
 
-let pkgs = import ./load-pkgs.nix { inherit pkgSource; };
+let pkgs = import ./load-pkgs.nix { inherit srcType srcArgs; };
 in with pkgs; rec {
 
   groupLayers = buildGoPackage {
diff --git a/tools/nixery/build-image/load-pkgs.nix b/tools/nixery/build-image/load-pkgs.nix
index 3e8b450c45d2..cceebfc14dae 100644
--- a/tools/nixery/build-image/load-pkgs.nix
+++ b/tools/nixery/build-image/load-pkgs.nix
@@ -12,17 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Load a Nix package set from a source specified in one of the following
-# formats:
-#
-# 1. nixpkgs!$channel (e.g. nixpkgs!nixos-19.03)
-# 2. git!$repo!$rev (e.g. git!git@github.com:NixOS/nixpkgs.git!master)
-# 3. path!$path (e.g. path!/var/local/nixpkgs)
-#
-# '!' was chosen as the separator because `builtins.split` does not
-# support regex escapes and there are few other candidates. It
-# doesn't matter much because this is invoked by the server.
-{ pkgSource, args ? { } }:
+# Load a Nix package set from one of the supported source types
+# (nixpkgs, git, path).
+{ srcType, srcArgs, importArgs ? { } }:
 
 with builtins;
 let
@@ -32,42 +24,22 @@ let
     let
       url =
         "https://github.com/NixOS/nixpkgs-channels/archive/${channel}.tar.gz";
-    in import (fetchTarball url) args;
+    in import (fetchTarball url) importArgs;
 
   # If a git repository is requested, it is retrieved via
   # builtins.fetchGit which defaults to the git configuration of the
   # outside environment. This means that user-configured SSH
   # credentials etc. are going to work as expected.
-  fetchImportGit = url: rev:
-    let
-      # builtins.fetchGit needs to know whether 'rev' is a reference
-      # (e.g. a branch/tag) or a revision (i.e. a commit hash)
-      #
-      # Since this data is being extrapolated from the supplied image
-      # tag, we have to guess if we want to avoid specifying a format.
-      #
-      # There are some additional caveats around whether the default
-      # branch contains the specified revision, which need to be
-      # explained to users.
-      spec = if (stringLength rev) == 40 then {
-        inherit url rev;
-      } else {
-        inherit url;
-        ref = rev;
-      };
-    in import (fetchGit spec) args;
+  fetchImportGit = spec: import (fetchGit spec) importArgs;
 
   # No special handling is used for paths, so users are expected to pass one
   # that will work natively with Nix.
-  importPath = path: import (toPath path) args;
-
-  source = split "!" pkgSource;
-  sourceType = elemAt source 0;
-in if sourceType == "nixpkgs" then
-  fetchImportChannel (elemAt source 2)
-else if sourceType == "git" then
-  fetchImportGit (elemAt source 2) (elemAt source 4)
-else if sourceType == "path" then
-  importPath (elemAt source 2)
+  importPath = path: import (toPath path) importArgs;
+in if srcType == "nixpkgs" then
+  fetchImportChannel srcArgs
+else if srcType == "git" then
+  fetchImportGit (fromJSON srcArgs)
+else if srcType == "path" then
+  importPath srcArgs
 else
-  throw ("Invalid package set source specification: ${pkgSource}")
+  throw ("Invalid package set source specification: ${srcType} (${srcArgs})")
diff --git a/tools/nixery/default.nix b/tools/nixery/default.nix
index 1f908b609897..f7a2a1712bfb 100644
--- a/tools/nixery/default.nix
+++ b/tools/nixery/default.nix
@@ -28,7 +28,8 @@ rec {
 
   # Implementation of the image building & layering logic
   nixery-build-image = (import ./build-image {
-    pkgSource = "path!${<nixpkgs>}";
+    srcType = "path";
+    srcArgs = <nixpkgs>;
   }).wrapper;
 
   # Use mdBook to build a static asset page which Nixery can then
diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/server/builder/builder.go
index 35a2c2f71283..ce88d2dd894f 100644
--- a/tools/nixery/server/builder/builder.go
+++ b/tools/nixery/server/builder/builder.go
@@ -118,15 +118,16 @@ func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, ima
 			return nil, err
 		}
 
+		srcType, srcArgs := cfg.Pkgs.Render(image.Tag)
+
 		args := []string{
 			"--timeout", cfg.Timeout,
 			"--argstr", "name", image.Name,
 			"--argstr", "packages", string(packages),
+			"--argstr", "srcType", srcType,
+			"--argstr", "srcArgs", srcArgs,
 		}
 
-		if cfg.Pkgs != nil {
-			args = append(args, "--argstr", "pkgSource", cfg.Pkgs.Render(image.Tag))
-		}
 		cmd := exec.Command("nixery-build-image", args...)
 
 		outpipe, err := cmd.StdoutPipe()
diff --git a/tools/nixery/server/config/config.go b/tools/nixery/server/config/config.go
index 5fba0e658ae0..ea1bb1ab4532 100644
--- a/tools/nixery/server/config/config.go
+++ b/tools/nixery/server/config/config.go
@@ -18,7 +18,6 @@
 package config
 
 import (
-	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
@@ -26,58 +25,6 @@ import (
 	"cloud.google.com/go/storage"
 )
 
-// pkgSource represents the source from which the Nix package set used
-// by Nixery is imported. Users configure the source by setting one of
-// the supported environment variables.
-type PkgSource struct {
-	srcType string
-	args    string
-}
-
-// Convert the package source into the representation required by Nix.
-func (p *PkgSource) Render(tag string) string {
-	// The 'git' source requires a tag to be present.
-	if p.srcType == "git" {
-		if tag == "latest" || tag == "" {
-			tag = "master"
-		}
-
-		return fmt.Sprintf("git!%s!%s", p.args, tag)
-	}
-
-	return fmt.Sprintf("%s!%s", p.srcType, p.args)
-}
-
-// Retrieve a package source from the environment. If no source is
-// specified, the Nix code will default to a recent NixOS channel.
-func pkgSourceFromEnv() *PkgSource {
-	if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" {
-		log.Printf("Using Nix package set from Nix channel %q\n", channel)
-		return &PkgSource{
-			srcType: "nixpkgs",
-			args:    channel,
-		}
-	}
-
-	if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" {
-		log.Printf("Using Nix package set from git repository at %q\n", git)
-		return &PkgSource{
-			srcType: "git",
-			args:    git,
-		}
-	}
-
-	if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" {
-		log.Printf("Using Nix package set from path %q\n", path)
-		return &PkgSource{
-			srcType: "path",
-			args:    path,
-		}
-	}
-
-	return nil
-}
-
 // Load (optional) GCS bucket signing data from the GCS_SIGNING_KEY and
 // GCS_SIGNING_ACCOUNT envvars.
 func signingOptsFromEnv() *storage.SignedURLOptions {
@@ -118,18 +65,23 @@ type Config struct {
 	Bucket  string                    // GCS bucket to cache & serve layers
 	Signing *storage.SignedURLOptions // Signing options to use for GCS URLs
 	Port    string                    // Port on which to launch HTTP server
-	Pkgs    *PkgSource                // Source for Nix package set
+	Pkgs    PkgSource                 // Source for Nix package set
 	Timeout string                    // Timeout for a single Nix builder (seconds)
 	WebDir  string                    // Directory with static web assets
 }
 
-func FromEnv() *Config {
+func FromEnv() (*Config, error) {
+	pkgs, err := pkgSourceFromEnv()
+	if err != nil {
+		return nil, err
+	}
+
 	return &Config{
 		Bucket:  getConfig("BUCKET", "GCS bucket for layer storage", ""),
 		Port:    getConfig("PORT", "HTTP port", ""),
-		Pkgs:    pkgSourceFromEnv(),
+		Pkgs:    pkgs,
 		Signing: signingOptsFromEnv(),
 		Timeout: getConfig("NIX_TIMEOUT", "Nix builder timeout", "60"),
 		WebDir:  getConfig("WEB_DIR", "Static web file dir", ""),
-	}
+	}, nil
 }
diff --git a/tools/nixery/server/config/pkgsource.go b/tools/nixery/server/config/pkgsource.go
new file mode 100644
index 000000000000..61bea33dfe62
--- /dev/null
+++ b/tools/nixery/server/config/pkgsource.go
@@ -0,0 +1,155 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+package config
+
+import (
+	"crypto/sha1"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"regexp"
+	"strings"
+)
+
+// PkgSource represents the source from which the Nix package set used
+// by Nixery is imported. Users configure the source by setting one of
+// the supported environment variables.
+type PkgSource interface {
+	// Convert the package source into the representation required
+	// for calling Nix.
+	Render(tag string) (string, string)
+
+	// Create a key by which builds for this source and iamge
+	// combination can be cached.
+	//
+	// The empty string means that this value is not cacheable due
+	// to the package source being a moving target (such as a
+	// channel).
+	CacheKey(pkgs []string, tag string) string
+}
+
+type GitSource struct {
+	repository string
+}
+
+// Regex to determine whether a git reference is a commit hash or
+// something else (branch/tag).
+//
+// Used to check whether a git reference is cacheable, and to pass the
+// correct git structure to Nix.
+//
+// Note: If a user creates a branch or tag with the name of a commit
+// and references it intentionally, this heuristic will fail.
+var commitRegex = regexp.MustCompile(`^[0-9a-f]{40}$`)
+
+func (g *GitSource) Render(tag string) (string, string) {
+	args := map[string]string{
+		"url": g.repository,
+	}
+
+	// The 'git' source requires a tag to be present. If the user
+	// has not specified one, it is assumed that the default
+	// 'master' branch should be used.
+	if tag == "latest" || tag == "" {
+		tag = "master"
+	}
+
+	if commitRegex.MatchString(tag) {
+		args["rev"] = tag
+	} else {
+		args["ref"] = tag
+	}
+
+	j, _ := json.Marshal(args)
+
+	return "git", string(j)
+}
+
+func (g *GitSource) CacheKey(pkgs []string, tag string) string {
+	// Only full commit hashes can be used for caching, as
+	// everything else is potentially a moving target.
+	if !commitRegex.MatchString(tag) {
+		return ""
+	}
+
+	unhashed := strings.Join(pkgs, "") + tag
+	hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed)))
+
+	return hashed
+}
+
+type NixChannel struct {
+	channel string
+}
+
+func (n *NixChannel) Render(tag string) (string, string) {
+	return "nixpkgs", n.channel
+}
+
+func (n *NixChannel) CacheKey(pkgs []string, tag string) string {
+	// Since Nix channels are downloaded from the nixpkgs-channels
+	// Github, users can specify full commit hashes as the
+	// "channel", in which case builds are cacheable.
+	if !commitRegex.MatchString(n.channel) {
+		return ""
+	}
+
+	unhashed := strings.Join(pkgs, "") + n.channel
+	hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed)))
+
+	return hashed
+}
+
+type PkgsPath struct {
+	path string
+}
+
+func (p *PkgsPath) Render(tag string) (string, string) {
+	return "path", p.path
+}
+
+func (p *PkgsPath) CacheKey(pkgs []string, tag string) string {
+	// Path-based builds are not currently cacheable because we
+	// have no local hash of the package folder's state easily
+	// available.
+	return ""
+}
+
+// Retrieve a package source from the environment. If no source is
+// specified, the Nix code will default to a recent NixOS channel.
+func pkgSourceFromEnv() (PkgSource, error) {
+	if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" {
+		log.Printf("Using Nix package set from Nix channel %q\n", channel)
+		return &NixChannel{
+			channel: channel,
+		}, nil
+	}
+
+	if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" {
+		log.Printf("Using Nix package set from git repository at %q\n", git)
+		return &GitSource{
+			repository: git,
+		}, nil
+	}
+
+	if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" {
+		log.Printf("Using Nix package set from path %q\n", path)
+		return &PkgsPath{
+			path: path,
+		}, nil
+	}
+
+	return nil, fmt.Errorf("no valid package source has been specified")
+}
diff --git a/tools/nixery/server/main.go b/tools/nixery/server/main.go
index da181249b285..49db1cdf8a5f 100644
--- a/tools/nixery/server/main.go
+++ b/tools/nixery/server/main.go
@@ -190,7 +190,11 @@ func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 }
 
 func main() {
-	cfg := config.FromEnv()
+	cfg, err := config.FromEnv()
+	if err != nil {
+		log.Fatalln("Failed to load configuration", err)
+	}
+
 	ctx := context.Background()
 	bucket := prepareBucket(&ctx, cfg)
 	cache := builder.NewCache()