From 980f5e218761fa340b746e6336db62abf63c953a Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sun, 8 Sep 2019 21:53:22 +0100 Subject: refactor(server): Move package source management logic to server Introduces three new types representing each of the possible package sources and moves the logic for specifying the package source to the server. Concrete changes: * Determining whether a specified git reference is a commit vs. a branch/tag is now done in the server, and is done more precisely by using a regular expression. * Package sources now have a new `CacheKey` function which can be used to retrieve a key under which a build manifest can be cached *if* the package source is not a moving target (i.e. a full git commit hash of either nixpkgs or a private repository). This function is not yet used. * Users *must* now specify a package source, Nixery no longer defaults to anything and will fail to launch if no source is configured. --- tools/nixery/.travis.yml | 1 + tools/nixery/build-image/default.nix | 5 +- tools/nixery/build-image/load-pkgs.nix | 54 +++-------- tools/nixery/default.nix | 3 +- tools/nixery/server/builder/builder.go | 7 +- tools/nixery/server/config/config.go | 66 ++------------ tools/nixery/server/config/pkgsource.go | 155 ++++++++++++++++++++++++++++++++ tools/nixery/server/main.go | 6 +- 8 files changed, 192 insertions(+), 105 deletions(-) create mode 100644 tools/nixery/server/config/pkgsource.go (limited to 'tools') diff --git a/tools/nixery/.travis.yml b/tools/nixery/.travis.yml index 471037f36474..f670ab0e2cf5 100644 --- a/tools/nixery/.travis.yml +++ b/tools/nixery/.travis.yml @@ -31,6 +31,7 @@ script: -e GOOGLE_APPLICATION_CREDENTIALS=/var/nixery/key.json \ -e GCS_SIGNING_ACCOUNT="${GCS_SIGNING_ACCOUNT}" \ -e GCS_SIGNING_KEY=/var/nixery/gcs.pem \ + -e NIXERY_CHANNEL=nixos-unstable \ ${IMG} # print all of the container's logs regardless of success diff --git a/tools/nixery/build-image/default.nix b/tools/nixery/build-image/default.nix index 0d3002cb404e..6b1cea6f0ca2 100644 --- a/tools/nixery/build-image/default.nix +++ b/tools/nixery/build-image/default.nix @@ -20,11 +20,12 @@ # Because of the insanity occuring below, this function must mirror # all arguments of build-image.nix. -, pkgSource ? "nixpkgs!nixos-19.03" +, srcType ? "nixpkgs" +, srcArgs ? "nixos-19.03" , tag ? null, name ? null, packages ? null, maxLayers ? null }@args: -let pkgs = import ./load-pkgs.nix { inherit pkgSource; }; +let pkgs = import ./load-pkgs.nix { inherit srcType srcArgs; }; in with pkgs; rec { groupLayers = buildGoPackage { diff --git a/tools/nixery/build-image/load-pkgs.nix b/tools/nixery/build-image/load-pkgs.nix index 3e8b450c45d2..cceebfc14dae 100644 --- a/tools/nixery/build-image/load-pkgs.nix +++ b/tools/nixery/build-image/load-pkgs.nix @@ -12,17 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Load a Nix package set from a source specified in one of the following -# formats: -# -# 1. nixpkgs!$channel (e.g. nixpkgs!nixos-19.03) -# 2. git!$repo!$rev (e.g. git!git@github.com:NixOS/nixpkgs.git!master) -# 3. path!$path (e.g. path!/var/local/nixpkgs) -# -# '!' was chosen as the separator because `builtins.split` does not -# support regex escapes and there are few other candidates. It -# doesn't matter much because this is invoked by the server. -{ pkgSource, args ? { } }: +# Load a Nix package set from one of the supported source types +# (nixpkgs, git, path). +{ srcType, srcArgs, importArgs ? { } }: with builtins; let @@ -32,42 +24,22 @@ let let url = "https://github.com/NixOS/nixpkgs-channels/archive/${channel}.tar.gz"; - in import (fetchTarball url) args; + in import (fetchTarball url) importArgs; # If a git repository is requested, it is retrieved via # builtins.fetchGit which defaults to the git configuration of the # outside environment. This means that user-configured SSH # credentials etc. are going to work as expected. - fetchImportGit = url: rev: - let - # builtins.fetchGit needs to know whether 'rev' is a reference - # (e.g. a branch/tag) or a revision (i.e. a commit hash) - # - # Since this data is being extrapolated from the supplied image - # tag, we have to guess if we want to avoid specifying a format. - # - # There are some additional caveats around whether the default - # branch contains the specified revision, which need to be - # explained to users. - spec = if (stringLength rev) == 40 then { - inherit url rev; - } else { - inherit url; - ref = rev; - }; - in import (fetchGit spec) args; + fetchImportGit = spec: import (fetchGit spec) importArgs; # No special handling is used for paths, so users are expected to pass one # that will work natively with Nix. - importPath = path: import (toPath path) args; - - source = split "!" pkgSource; - sourceType = elemAt source 0; -in if sourceType == "nixpkgs" then - fetchImportChannel (elemAt source 2) -else if sourceType == "git" then - fetchImportGit (elemAt source 2) (elemAt source 4) -else if sourceType == "path" then - importPath (elemAt source 2) + importPath = path: import (toPath path) importArgs; +in if srcType == "nixpkgs" then + fetchImportChannel srcArgs +else if srcType == "git" then + fetchImportGit (fromJSON srcArgs) +else if srcType == "path" then + importPath srcArgs else - throw ("Invalid package set source specification: ${pkgSource}") + throw ("Invalid package set source specification: ${srcType} (${srcArgs})") diff --git a/tools/nixery/default.nix b/tools/nixery/default.nix index 1f908b609897..f7a2a1712bfb 100644 --- a/tools/nixery/default.nix +++ b/tools/nixery/default.nix @@ -28,7 +28,8 @@ rec { # Implementation of the image building & layering logic nixery-build-image = (import ./build-image { - pkgSource = "path!${}"; + srcType = "path"; + srcArgs = ; }).wrapper; # Use mdBook to build a static asset page which Nixery can then diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/server/builder/builder.go index 35a2c2f71283..ce88d2dd894f 100644 --- a/tools/nixery/server/builder/builder.go +++ b/tools/nixery/server/builder/builder.go @@ -118,15 +118,16 @@ func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, ima return nil, err } + srcType, srcArgs := cfg.Pkgs.Render(image.Tag) + args := []string{ "--timeout", cfg.Timeout, "--argstr", "name", image.Name, "--argstr", "packages", string(packages), + "--argstr", "srcType", srcType, + "--argstr", "srcArgs", srcArgs, } - if cfg.Pkgs != nil { - args = append(args, "--argstr", "pkgSource", cfg.Pkgs.Render(image.Tag)) - } cmd := exec.Command("nixery-build-image", args...) outpipe, err := cmd.StdoutPipe() diff --git a/tools/nixery/server/config/config.go b/tools/nixery/server/config/config.go index 5fba0e658ae0..ea1bb1ab4532 100644 --- a/tools/nixery/server/config/config.go +++ b/tools/nixery/server/config/config.go @@ -18,7 +18,6 @@ package config import ( - "fmt" "io/ioutil" "log" "os" @@ -26,58 +25,6 @@ import ( "cloud.google.com/go/storage" ) -// pkgSource represents the source from which the Nix package set used -// by Nixery is imported. Users configure the source by setting one of -// the supported environment variables. -type PkgSource struct { - srcType string - args string -} - -// Convert the package source into the representation required by Nix. -func (p *PkgSource) Render(tag string) string { - // The 'git' source requires a tag to be present. - if p.srcType == "git" { - if tag == "latest" || tag == "" { - tag = "master" - } - - return fmt.Sprintf("git!%s!%s", p.args, tag) - } - - return fmt.Sprintf("%s!%s", p.srcType, p.args) -} - -// Retrieve a package source from the environment. If no source is -// specified, the Nix code will default to a recent NixOS channel. -func pkgSourceFromEnv() *PkgSource { - if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" { - log.Printf("Using Nix package set from Nix channel %q\n", channel) - return &PkgSource{ - srcType: "nixpkgs", - args: channel, - } - } - - if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" { - log.Printf("Using Nix package set from git repository at %q\n", git) - return &PkgSource{ - srcType: "git", - args: git, - } - } - - if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" { - log.Printf("Using Nix package set from path %q\n", path) - return &PkgSource{ - srcType: "path", - args: path, - } - } - - return nil -} - // Load (optional) GCS bucket signing data from the GCS_SIGNING_KEY and // GCS_SIGNING_ACCOUNT envvars. func signingOptsFromEnv() *storage.SignedURLOptions { @@ -118,18 +65,23 @@ type Config struct { Bucket string // GCS bucket to cache & serve layers Signing *storage.SignedURLOptions // Signing options to use for GCS URLs Port string // Port on which to launch HTTP server - Pkgs *PkgSource // Source for Nix package set + Pkgs PkgSource // Source for Nix package set Timeout string // Timeout for a single Nix builder (seconds) WebDir string // Directory with static web assets } -func FromEnv() *Config { +func FromEnv() (*Config, error) { + pkgs, err := pkgSourceFromEnv() + if err != nil { + return nil, err + } + return &Config{ Bucket: getConfig("BUCKET", "GCS bucket for layer storage", ""), Port: getConfig("PORT", "HTTP port", ""), - Pkgs: pkgSourceFromEnv(), + Pkgs: pkgs, Signing: signingOptsFromEnv(), Timeout: getConfig("NIX_TIMEOUT", "Nix builder timeout", "60"), WebDir: getConfig("WEB_DIR", "Static web file dir", ""), - } + }, nil } diff --git a/tools/nixery/server/config/pkgsource.go b/tools/nixery/server/config/pkgsource.go new file mode 100644 index 000000000000..61bea33dfe62 --- /dev/null +++ b/tools/nixery/server/config/pkgsource.go @@ -0,0 +1,155 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +package config + +import ( + "crypto/sha1" + "encoding/json" + "fmt" + "log" + "os" + "regexp" + "strings" +) + +// PkgSource represents the source from which the Nix package set used +// by Nixery is imported. Users configure the source by setting one of +// the supported environment variables. +type PkgSource interface { + // Convert the package source into the representation required + // for calling Nix. + Render(tag string) (string, string) + + // Create a key by which builds for this source and iamge + // combination can be cached. + // + // The empty string means that this value is not cacheable due + // to the package source being a moving target (such as a + // channel). + CacheKey(pkgs []string, tag string) string +} + +type GitSource struct { + repository string +} + +// Regex to determine whether a git reference is a commit hash or +// something else (branch/tag). +// +// Used to check whether a git reference is cacheable, and to pass the +// correct git structure to Nix. +// +// Note: If a user creates a branch or tag with the name of a commit +// and references it intentionally, this heuristic will fail. +var commitRegex = regexp.MustCompile(`^[0-9a-f]{40}$`) + +func (g *GitSource) Render(tag string) (string, string) { + args := map[string]string{ + "url": g.repository, + } + + // The 'git' source requires a tag to be present. If the user + // has not specified one, it is assumed that the default + // 'master' branch should be used. + if tag == "latest" || tag == "" { + tag = "master" + } + + if commitRegex.MatchString(tag) { + args["rev"] = tag + } else { + args["ref"] = tag + } + + j, _ := json.Marshal(args) + + return "git", string(j) +} + +func (g *GitSource) CacheKey(pkgs []string, tag string) string { + // Only full commit hashes can be used for caching, as + // everything else is potentially a moving target. + if !commitRegex.MatchString(tag) { + return "" + } + + unhashed := strings.Join(pkgs, "") + tag + hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed))) + + return hashed +} + +type NixChannel struct { + channel string +} + +func (n *NixChannel) Render(tag string) (string, string) { + return "nixpkgs", n.channel +} + +func (n *NixChannel) CacheKey(pkgs []string, tag string) string { + // Since Nix channels are downloaded from the nixpkgs-channels + // Github, users can specify full commit hashes as the + // "channel", in which case builds are cacheable. + if !commitRegex.MatchString(n.channel) { + return "" + } + + unhashed := strings.Join(pkgs, "") + n.channel + hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed))) + + return hashed +} + +type PkgsPath struct { + path string +} + +func (p *PkgsPath) Render(tag string) (string, string) { + return "path", p.path +} + +func (p *PkgsPath) CacheKey(pkgs []string, tag string) string { + // Path-based builds are not currently cacheable because we + // have no local hash of the package folder's state easily + // available. + return "" +} + +// Retrieve a package source from the environment. If no source is +// specified, the Nix code will default to a recent NixOS channel. +func pkgSourceFromEnv() (PkgSource, error) { + if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" { + log.Printf("Using Nix package set from Nix channel %q\n", channel) + return &NixChannel{ + channel: channel, + }, nil + } + + if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" { + log.Printf("Using Nix package set from git repository at %q\n", git) + return &GitSource{ + repository: git, + }, nil + } + + if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" { + log.Printf("Using Nix package set from path %q\n", path) + return &PkgsPath{ + path: path, + }, nil + } + + return nil, fmt.Errorf("no valid package source has been specified") +} diff --git a/tools/nixery/server/main.go b/tools/nixery/server/main.go index da181249b285..49db1cdf8a5f 100644 --- a/tools/nixery/server/main.go +++ b/tools/nixery/server/main.go @@ -190,7 +190,11 @@ func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } func main() { - cfg := config.FromEnv() + cfg, err := config.FromEnv() + if err != nil { + log.Fatalln("Failed to load configuration", err) + } + ctx := context.Background() bucket := prepareBucket(&ctx, cfg) cache := builder.NewCache() -- cgit 1.4.1