diff options
| -rw-r--r-- | tools/nixery/.travis.yml | 1 | ||||
| -rw-r--r-- | tools/nixery/build-image/default.nix | 5 | ||||
| -rw-r--r-- | tools/nixery/build-image/load-pkgs.nix | 54 | ||||
| -rw-r--r-- | tools/nixery/default.nix | 3 | ||||
| -rw-r--r-- | tools/nixery/server/builder/builder.go | 7 | ||||
| -rw-r--r-- | tools/nixery/server/config/config.go | 66 | ||||
| -rw-r--r-- | tools/nixery/server/config/pkgsource.go | 155 | ||||
| -rw-r--r-- | tools/nixery/server/main.go | 6 |
8 files changed, 192 insertions, 105 deletions
diff --git a/tools/nixery/.travis.yml b/tools/nixery/.travis.yml index 471037f364..f670ab0e2c 100644 --- a/tools/nixery/.travis.yml +++ b/tools/nixery/.travis.yml @@ -31,6 +31,7 @@ script: -e GOOGLE_APPLICATION_CREDENTIALS=/var/nixery/key.json \ -e GCS_SIGNING_ACCOUNT="${GCS_SIGNING_ACCOUNT}" \ -e GCS_SIGNING_KEY=/var/nixery/gcs.pem \ + -e NIXERY_CHANNEL=nixos-unstable \ ${IMG} # print all of the container's logs regardless of success diff --git a/tools/nixery/build-image/default.nix b/tools/nixery/build-image/default.nix index 0d3002cb40..6b1cea6f0c 100644 --- a/tools/nixery/build-image/default.nix +++ b/tools/nixery/build-image/default.nix @@ -20,11 +20,12 @@ # Because of the insanity occuring below, this function must mirror # all arguments of build-image.nix. -, pkgSource ? "nixpkgs!nixos-19.03" +, srcType ? "nixpkgs" +, srcArgs ? "nixos-19.03" , tag ? null, name ? null, packages ? null, maxLayers ? null }@args: -let pkgs = import ./load-pkgs.nix { inherit pkgSource; }; +let pkgs = import ./load-pkgs.nix { inherit srcType srcArgs; }; in with pkgs; rec { groupLayers = buildGoPackage { diff --git a/tools/nixery/build-image/load-pkgs.nix b/tools/nixery/build-image/load-pkgs.nix index 3e8b450c45..cceebfc14d 100644 --- a/tools/nixery/build-image/load-pkgs.nix +++ b/tools/nixery/build-image/load-pkgs.nix @@ -12,17 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Load a Nix package set from a source specified in one of the following -# formats: -# -# 1. nixpkgs!$channel (e.g. nixpkgs!nixos-19.03) -# 2. git!$repo!$rev (e.g. git!git@github.com:NixOS/nixpkgs.git!master) -# 3. path!$path (e.g. path!/var/local/nixpkgs) -# -# '!' was chosen as the separator because `builtins.split` does not -# support regex escapes and there are few other candidates. It -# doesn't matter much because this is invoked by the server. -{ pkgSource, args ? { } }: +# Load a Nix package set from one of the supported source types +# (nixpkgs, git, path). +{ srcType, srcArgs, importArgs ? { } }: with builtins; let @@ -32,42 +24,22 @@ let let url = "https://github.com/NixOS/nixpkgs-channels/archive/${channel}.tar.gz"; - in import (fetchTarball url) args; + in import (fetchTarball url) importArgs; # If a git repository is requested, it is retrieved via # builtins.fetchGit which defaults to the git configuration of the # outside environment. This means that user-configured SSH # credentials etc. are going to work as expected. - fetchImportGit = url: rev: - let - # builtins.fetchGit needs to know whether 'rev' is a reference - # (e.g. a branch/tag) or a revision (i.e. a commit hash) - # - # Since this data is being extrapolated from the supplied image - # tag, we have to guess if we want to avoid specifying a format. - # - # There are some additional caveats around whether the default - # branch contains the specified revision, which need to be - # explained to users. - spec = if (stringLength rev) == 40 then { - inherit url rev; - } else { - inherit url; - ref = rev; - }; - in import (fetchGit spec) args; + fetchImportGit = spec: import (fetchGit spec) importArgs; # No special handling is used for paths, so users are expected to pass one # that will work natively with Nix. - importPath = path: import (toPath path) args; - - source = split "!" pkgSource; - sourceType = elemAt source 0; -in if sourceType == "nixpkgs" then - fetchImportChannel (elemAt source 2) -else if sourceType == "git" then - fetchImportGit (elemAt source 2) (elemAt source 4) -else if sourceType == "path" then - importPath (elemAt source 2) + importPath = path: import (toPath path) importArgs; +in if srcType == "nixpkgs" then + fetchImportChannel srcArgs +else if srcType == "git" then + fetchImportGit (fromJSON srcArgs) +else if srcType == "path" then + importPath srcArgs else - throw ("Invalid package set source specification: ${pkgSource}") + throw ("Invalid package set source specification: ${srcType} (${srcArgs})") diff --git a/tools/nixery/default.nix b/tools/nixery/default.nix index 1f908b6098..f7a2a1712b 100644 --- a/tools/nixery/default.nix +++ b/tools/nixery/default.nix @@ -28,7 +28,8 @@ rec { # Implementation of the image building & layering logic nixery-build-image = (import ./build-image { - pkgSource = "path!${<nixpkgs>}"; + srcType = "path"; + srcArgs = <nixpkgs>; }).wrapper; # Use mdBook to build a static asset page which Nixery can then diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/server/builder/builder.go index 35a2c2f712..ce88d2dd89 100644 --- a/tools/nixery/server/builder/builder.go +++ b/tools/nixery/server/builder/builder.go @@ -118,15 +118,16 @@ func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, ima return nil, err } + srcType, srcArgs := cfg.Pkgs.Render(image.Tag) + args := []string{ "--timeout", cfg.Timeout, "--argstr", "name", image.Name, "--argstr", "packages", string(packages), + "--argstr", "srcType", srcType, + "--argstr", "srcArgs", srcArgs, } - if cfg.Pkgs != nil { - args = append(args, "--argstr", "pkgSource", cfg.Pkgs.Render(image.Tag)) - } cmd := exec.Command("nixery-build-image", args...) outpipe, err := cmd.StdoutPipe() diff --git a/tools/nixery/server/config/config.go b/tools/nixery/server/config/config.go index 5fba0e658a..ea1bb1ab45 100644 --- a/tools/nixery/server/config/config.go +++ b/tools/nixery/server/config/config.go @@ -18,7 +18,6 @@ package config import ( - "fmt" "io/ioutil" "log" "os" @@ -26,58 +25,6 @@ import ( "cloud.google.com/go/storage" ) -// pkgSource represents the source from which the Nix package set used -// by Nixery is imported. Users configure the source by setting one of -// the supported environment variables. -type PkgSource struct { - srcType string - args string -} - -// Convert the package source into the representation required by Nix. -func (p *PkgSource) Render(tag string) string { - // The 'git' source requires a tag to be present. - if p.srcType == "git" { - if tag == "latest" || tag == "" { - tag = "master" - } - - return fmt.Sprintf("git!%s!%s", p.args, tag) - } - - return fmt.Sprintf("%s!%s", p.srcType, p.args) -} - -// Retrieve a package source from the environment. If no source is -// specified, the Nix code will default to a recent NixOS channel. -func pkgSourceFromEnv() *PkgSource { - if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" { - log.Printf("Using Nix package set from Nix channel %q\n", channel) - return &PkgSource{ - srcType: "nixpkgs", - args: channel, - } - } - - if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" { - log.Printf("Using Nix package set from git repository at %q\n", git) - return &PkgSource{ - srcType: "git", - args: git, - } - } - - if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" { - log.Printf("Using Nix package set from path %q\n", path) - return &PkgSource{ - srcType: "path", - args: path, - } - } - - return nil -} - // Load (optional) GCS bucket signing data from the GCS_SIGNING_KEY and // GCS_SIGNING_ACCOUNT envvars. func signingOptsFromEnv() *storage.SignedURLOptions { @@ -118,18 +65,23 @@ type Config struct { Bucket string // GCS bucket to cache & serve layers Signing *storage.SignedURLOptions // Signing options to use for GCS URLs Port string // Port on which to launch HTTP server - Pkgs *PkgSource // Source for Nix package set + Pkgs PkgSource // Source for Nix package set Timeout string // Timeout for a single Nix builder (seconds) WebDir string // Directory with static web assets } -func FromEnv() *Config { +func FromEnv() (*Config, error) { + pkgs, err := pkgSourceFromEnv() + if err != nil { + return nil, err + } + return &Config{ Bucket: getConfig("BUCKET", "GCS bucket for layer storage", ""), Port: getConfig("PORT", "HTTP port", ""), - Pkgs: pkgSourceFromEnv(), + Pkgs: pkgs, Signing: signingOptsFromEnv(), Timeout: getConfig("NIX_TIMEOUT", "Nix builder timeout", "60"), WebDir: getConfig("WEB_DIR", "Static web file dir", ""), - } + }, nil } diff --git a/tools/nixery/server/config/pkgsource.go b/tools/nixery/server/config/pkgsource.go new file mode 100644 index 0000000000..61bea33dfe --- /dev/null +++ b/tools/nixery/server/config/pkgsource.go @@ -0,0 +1,155 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +package config + +import ( + "crypto/sha1" + "encoding/json" + "fmt" + "log" + "os" + "regexp" + "strings" +) + +// PkgSource represents the source from which the Nix package set used +// by Nixery is imported. Users configure the source by setting one of +// the supported environment variables. +type PkgSource interface { + // Convert the package source into the representation required + // for calling Nix. + Render(tag string) (string, string) + + // Create a key by which builds for this source and iamge + // combination can be cached. + // + // The empty string means that this value is not cacheable due + // to the package source being a moving target (such as a + // channel). + CacheKey(pkgs []string, tag string) string +} + +type GitSource struct { + repository string +} + +// Regex to determine whether a git reference is a commit hash or +// something else (branch/tag). +// +// Used to check whether a git reference is cacheable, and to pass the +// correct git structure to Nix. +// +// Note: If a user creates a branch or tag with the name of a commit +// and references it intentionally, this heuristic will fail. +var commitRegex = regexp.MustCompile(`^[0-9a-f]{40}$`) + +func (g *GitSource) Render(tag string) (string, string) { + args := map[string]string{ + "url": g.repository, + } + + // The 'git' source requires a tag to be present. If the user + // has not specified one, it is assumed that the default + // 'master' branch should be used. + if tag == "latest" || tag == "" { + tag = "master" + } + + if commitRegex.MatchString(tag) { + args["rev"] = tag + } else { + args["ref"] = tag + } + + j, _ := json.Marshal(args) + + return "git", string(j) +} + +func (g *GitSource) CacheKey(pkgs []string, tag string) string { + // Only full commit hashes can be used for caching, as + // everything else is potentially a moving target. + if !commitRegex.MatchString(tag) { + return "" + } + + unhashed := strings.Join(pkgs, "") + tag + hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed))) + + return hashed +} + +type NixChannel struct { + channel string +} + +func (n *NixChannel) Render(tag string) (string, string) { + return "nixpkgs", n.channel +} + +func (n *NixChannel) CacheKey(pkgs []string, tag string) string { + // Since Nix channels are downloaded from the nixpkgs-channels + // Github, users can specify full commit hashes as the + // "channel", in which case builds are cacheable. + if !commitRegex.MatchString(n.channel) { + return "" + } + + unhashed := strings.Join(pkgs, "") + n.channel + hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed))) + + return hashed +} + +type PkgsPath struct { + path string +} + +func (p *PkgsPath) Render(tag string) (string, string) { + return "path", p.path +} + +func (p *PkgsPath) CacheKey(pkgs []string, tag string) string { + // Path-based builds are not currently cacheable because we + // have no local hash of the package folder's state easily + // available. + return "" +} + +// Retrieve a package source from the environment. If no source is +// specified, the Nix code will default to a recent NixOS channel. +func pkgSourceFromEnv() (PkgSource, error) { + if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" { + log.Printf("Using Nix package set from Nix channel %q\n", channel) + return &NixChannel{ + channel: channel, + }, nil + } + + if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" { + log.Printf("Using Nix package set from git repository at %q\n", git) + return &GitSource{ + repository: git, + }, nil + } + + if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" { + log.Printf("Using Nix package set from path %q\n", path) + return &PkgsPath{ + path: path, + }, nil + } + + return nil, fmt.Errorf("no valid package source has been specified") +} diff --git a/tools/nixery/server/main.go b/tools/nixery/server/main.go index da181249b2..49db1cdf8a 100644 --- a/tools/nixery/server/main.go +++ b/tools/nixery/server/main.go @@ -190,7 +190,11 @@ func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } func main() { - cfg := config.FromEnv() + cfg, err := config.FromEnv() + if err != nil { + log.Fatalln("Failed to load configuration", err) + } + ctx := context.Background() bucket := prepareBucket(&ctx, cfg) cache := builder.NewCache() |