diff options
Diffstat (limited to 'tools/nixery/server')
| -rw-r--r-- | tools/nixery/server/builder/builder.go | 7 | ||||
| -rw-r--r-- | tools/nixery/server/config/config.go | 66 | ||||
| -rw-r--r-- | tools/nixery/server/config/pkgsource.go | 155 | ||||
| -rw-r--r-- | tools/nixery/server/main.go | 6 |
4 files changed, 173 insertions, 61 deletions
diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/server/builder/builder.go index 35a2c2f71283..ce88d2dd894f 100644 --- a/tools/nixery/server/builder/builder.go +++ b/tools/nixery/server/builder/builder.go @@ -118,15 +118,16 @@ func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, ima return nil, err } + srcType, srcArgs := cfg.Pkgs.Render(image.Tag) + args := []string{ "--timeout", cfg.Timeout, "--argstr", "name", image.Name, "--argstr", "packages", string(packages), + "--argstr", "srcType", srcType, + "--argstr", "srcArgs", srcArgs, } - if cfg.Pkgs != nil { - args = append(args, "--argstr", "pkgSource", cfg.Pkgs.Render(image.Tag)) - } cmd := exec.Command("nixery-build-image", args...) outpipe, err := cmd.StdoutPipe() diff --git a/tools/nixery/server/config/config.go b/tools/nixery/server/config/config.go index 5fba0e658ae0..ea1bb1ab4532 100644 --- a/tools/nixery/server/config/config.go +++ b/tools/nixery/server/config/config.go @@ -18,7 +18,6 @@ package config import ( - "fmt" "io/ioutil" "log" "os" @@ -26,58 +25,6 @@ import ( "cloud.google.com/go/storage" ) -// pkgSource represents the source from which the Nix package set used -// by Nixery is imported. Users configure the source by setting one of -// the supported environment variables. -type PkgSource struct { - srcType string - args string -} - -// Convert the package source into the representation required by Nix. -func (p *PkgSource) Render(tag string) string { - // The 'git' source requires a tag to be present. - if p.srcType == "git" { - if tag == "latest" || tag == "" { - tag = "master" - } - - return fmt.Sprintf("git!%s!%s", p.args, tag) - } - - return fmt.Sprintf("%s!%s", p.srcType, p.args) -} - -// Retrieve a package source from the environment. If no source is -// specified, the Nix code will default to a recent NixOS channel. -func pkgSourceFromEnv() *PkgSource { - if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" { - log.Printf("Using Nix package set from Nix channel %q\n", channel) - return &PkgSource{ - srcType: "nixpkgs", - args: channel, - } - } - - if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" { - log.Printf("Using Nix package set from git repository at %q\n", git) - return &PkgSource{ - srcType: "git", - args: git, - } - } - - if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" { - log.Printf("Using Nix package set from path %q\n", path) - return &PkgSource{ - srcType: "path", - args: path, - } - } - - return nil -} - // Load (optional) GCS bucket signing data from the GCS_SIGNING_KEY and // GCS_SIGNING_ACCOUNT envvars. func signingOptsFromEnv() *storage.SignedURLOptions { @@ -118,18 +65,23 @@ type Config struct { Bucket string // GCS bucket to cache & serve layers Signing *storage.SignedURLOptions // Signing options to use for GCS URLs Port string // Port on which to launch HTTP server - Pkgs *PkgSource // Source for Nix package set + Pkgs PkgSource // Source for Nix package set Timeout string // Timeout for a single Nix builder (seconds) WebDir string // Directory with static web assets } -func FromEnv() *Config { +func FromEnv() (*Config, error) { + pkgs, err := pkgSourceFromEnv() + if err != nil { + return nil, err + } + return &Config{ Bucket: getConfig("BUCKET", "GCS bucket for layer storage", ""), Port: getConfig("PORT", "HTTP port", ""), - Pkgs: pkgSourceFromEnv(), + Pkgs: pkgs, Signing: signingOptsFromEnv(), Timeout: getConfig("NIX_TIMEOUT", "Nix builder timeout", "60"), WebDir: getConfig("WEB_DIR", "Static web file dir", ""), - } + }, nil } diff --git a/tools/nixery/server/config/pkgsource.go b/tools/nixery/server/config/pkgsource.go new file mode 100644 index 000000000000..61bea33dfe62 --- /dev/null +++ b/tools/nixery/server/config/pkgsource.go @@ -0,0 +1,155 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +package config + +import ( + "crypto/sha1" + "encoding/json" + "fmt" + "log" + "os" + "regexp" + "strings" +) + +// PkgSource represents the source from which the Nix package set used +// by Nixery is imported. Users configure the source by setting one of +// the supported environment variables. +type PkgSource interface { + // Convert the package source into the representation required + // for calling Nix. + Render(tag string) (string, string) + + // Create a key by which builds for this source and iamge + // combination can be cached. + // + // The empty string means that this value is not cacheable due + // to the package source being a moving target (such as a + // channel). + CacheKey(pkgs []string, tag string) string +} + +type GitSource struct { + repository string +} + +// Regex to determine whether a git reference is a commit hash or +// something else (branch/tag). +// +// Used to check whether a git reference is cacheable, and to pass the +// correct git structure to Nix. +// +// Note: If a user creates a branch or tag with the name of a commit +// and references it intentionally, this heuristic will fail. +var commitRegex = regexp.MustCompile(`^[0-9a-f]{40}$`) + +func (g *GitSource) Render(tag string) (string, string) { + args := map[string]string{ + "url": g.repository, + } + + // The 'git' source requires a tag to be present. If the user + // has not specified one, it is assumed that the default + // 'master' branch should be used. + if tag == "latest" || tag == "" { + tag = "master" + } + + if commitRegex.MatchString(tag) { + args["rev"] = tag + } else { + args["ref"] = tag + } + + j, _ := json.Marshal(args) + + return "git", string(j) +} + +func (g *GitSource) CacheKey(pkgs []string, tag string) string { + // Only full commit hashes can be used for caching, as + // everything else is potentially a moving target. + if !commitRegex.MatchString(tag) { + return "" + } + + unhashed := strings.Join(pkgs, "") + tag + hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed))) + + return hashed +} + +type NixChannel struct { + channel string +} + +func (n *NixChannel) Render(tag string) (string, string) { + return "nixpkgs", n.channel +} + +func (n *NixChannel) CacheKey(pkgs []string, tag string) string { + // Since Nix channels are downloaded from the nixpkgs-channels + // Github, users can specify full commit hashes as the + // "channel", in which case builds are cacheable. + if !commitRegex.MatchString(n.channel) { + return "" + } + + unhashed := strings.Join(pkgs, "") + n.channel + hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed))) + + return hashed +} + +type PkgsPath struct { + path string +} + +func (p *PkgsPath) Render(tag string) (string, string) { + return "path", p.path +} + +func (p *PkgsPath) CacheKey(pkgs []string, tag string) string { + // Path-based builds are not currently cacheable because we + // have no local hash of the package folder's state easily + // available. + return "" +} + +// Retrieve a package source from the environment. If no source is +// specified, the Nix code will default to a recent NixOS channel. +func pkgSourceFromEnv() (PkgSource, error) { + if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" { + log.Printf("Using Nix package set from Nix channel %q\n", channel) + return &NixChannel{ + channel: channel, + }, nil + } + + if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" { + log.Printf("Using Nix package set from git repository at %q\n", git) + return &GitSource{ + repository: git, + }, nil + } + + if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" { + log.Printf("Using Nix package set from path %q\n", path) + return &PkgsPath{ + path: path, + }, nil + } + + return nil, fmt.Errorf("no valid package source has been specified") +} diff --git a/tools/nixery/server/main.go b/tools/nixery/server/main.go index da181249b285..49db1cdf8a5f 100644 --- a/tools/nixery/server/main.go +++ b/tools/nixery/server/main.go @@ -190,7 +190,11 @@ func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } func main() { - cfg := config.FromEnv() + cfg, err := config.FromEnv() + if err != nil { + log.Fatalln("Failed to load configuration", err) + } + ctx := context.Background() bucket := prepareBucket(&ctx, cfg) cache := builder.NewCache() |