From 1fa93fe6f640bfdbb5e9ecedb2dbf2cacc5e8945 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Mon, 12 Aug 2019 15:14:24 +0100 Subject: refactor: Move registry server to a subfolder --- tools/nixery/default.nix | 22 +- tools/nixery/go-deps.nix | 120 ---------- tools/nixery/main.go | 492 ---------------------------------------- tools/nixery/server/default.nix | 16 ++ tools/nixery/server/go-deps.nix | 111 +++++++++ tools/nixery/server/main.go | 492 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 621 insertions(+), 632 deletions(-) delete mode 100644 tools/nixery/go-deps.nix delete mode 100644 tools/nixery/main.go create mode 100644 tools/nixery/server/default.nix create mode 100644 tools/nixery/server/go-deps.nix create mode 100644 tools/nixery/server/main.go (limited to 'tools') diff --git a/tools/nixery/default.nix b/tools/nixery/default.nix index 092c76e9c5b9..dee5713c64af 100644 --- a/tools/nixery/default.nix +++ b/tools/nixery/default.nix @@ -23,29 +23,11 @@ rec { # Users will usually not want to use this directly, instead see the # 'nixery' derivation below, which automatically includes runtime # data dependencies. - nixery-server = buildGoPackage { - name = "nixery-server"; - - # Technically people should not be building Nixery through 'go get' - # or similar (as other required files will not be included), but - # buildGoPackage requires a package path. - goPackagePath = "github.com/google/nixery"; - goDeps = ./go-deps.nix; - src = ./.; - - meta = { - description = "Container image build serving Nix-backed images"; - homepage = "https://github.com/google/nixery"; - license = lib.licenses.asl20; - maintainers = [ lib.maintainers.tazjin ]; - }; - }; + nixery-server = callPackage ./server {}; # Nix expression (unimported!) which is used by Nixery to build # container images. - nixery-builder = runCommand "build-registry-image.nix" {} '' - cat ${./build-registry-image.nix} > $out - ''; + nixery-builder = ./build-registry-image.nix; # nixpkgs currently has an old version of mdBook. A new version is # built here, but eventually the update will be upstreamed diff --git a/tools/nixery/go-deps.nix b/tools/nixery/go-deps.nix deleted file mode 100644 index ebd1576db5dd..000000000000 --- a/tools/nixery/go-deps.nix +++ /dev/null @@ -1,120 +0,0 @@ -# This file was generated by https://github.com/kamilchm/go2nix v1.3.0 -[ - { - goPackagePath = "cloud.google.com/go"; - fetch = { - type = "git"; - url = "https://code.googlesource.com/gocloud"; - rev = "77f6a3a292a7dbf66a5329de0d06326f1906b450"; - sha256 = "1c9pkx782nbcp8jnl5lprcbzf97van789ky5qsncjgywjyymhigi"; - }; - } - { - goPackagePath = "github.com/golang/protobuf"; - fetch = { - type = "git"; - url = "https://github.com/golang/protobuf"; - rev = "6c65a5562fc06764971b7c5d05c76c75e84bdbf7"; - sha256 = "1k1wb4zr0qbwgpvz9q5ws9zhlal8hq7dmq62pwxxriksayl6hzym"; - }; - } - { - goPackagePath = "github.com/googleapis/gax-go"; - fetch = { - type = "git"; - url = "https://github.com/googleapis/gax-go"; - rev = "bd5b16380fd03dc758d11cef74ba2e3bc8b0e8c2"; - sha256 = "1lxawwngv6miaqd25s3ba0didfzylbwisd2nz7r4gmbmin6jsjrx"; - }; - } - { - goPackagePath = "github.com/hashicorp/golang-lru"; - fetch = { - type = "git"; - url = "https://github.com/hashicorp/golang-lru"; - rev = "59383c442f7d7b190497e9bb8fc17a48d06cd03f"; - sha256 = "0yzwl592aa32vfy73pl7wdc21855w17zssrp85ckw2nisky8rg9c"; - }; - } - { - goPackagePath = "go.opencensus.io"; - fetch = { - type = "git"; - url = "https://github.com/census-instrumentation/opencensus-go"; - rev = "b4a14686f0a98096416fe1b4cb848e384fb2b22b"; - sha256 = "1aidyp301v5ngwsnnc8v1s09vvbsnch1jc4vd615f7qv77r9s7dn"; - }; - } - { - goPackagePath = "golang.org/x/net"; - fetch = { - type = "git"; - url = "https://go.googlesource.com/net"; - rev = "da137c7871d730100384dbcf36e6f8fa493aef5b"; - sha256 = "1qsiyr3irmb6ii06hivm9p2c7wqyxczms1a9v1ss5698yjr3fg47"; - }; - } - { - goPackagePath = "golang.org/x/oauth2"; - fetch = { - type = "git"; - url = "https://go.googlesource.com/oauth2"; - rev = "0f29369cfe4552d0e4bcddc57cc75f4d7e672a33"; - sha256 = "06jwpvx0x2gjn2y959drbcir5kd7vg87k0r1216abk6rrdzzrzi2"; - }; - } - { - goPackagePath = "golang.org/x/sys"; - fetch = { - type = "git"; - url = "https://go.googlesource.com/sys"; - rev = "51ab0e2deafac1f46c46ad59cf0921be2f180c3d"; - sha256 = "0xdhpckbql3bsqkpc2k5b1cpnq3q1qjqjjq2j3p707rfwb8nm91a"; - }; - } - { - goPackagePath = "golang.org/x/text"; - fetch = { - type = "git"; - url = "https://go.googlesource.com/text"; - rev = "342b2e1fbaa52c93f31447ad2c6abc048c63e475"; - sha256 = "0flv9idw0jm5nm8lx25xqanbkqgfiym6619w575p7nrdh0riqwqh"; - }; - } - { - goPackagePath = "gonum.org/v1/gonum"; - fetch = { - type = "git"; - url = "https://github.com/gonum/gonum"; - rev = "ced62fe5104b907b6c16cb7e575c17b2e62ceddd"; - sha256 = "1b7q6haabnp53igpmvr6a2414yralhbrldixx4kbxxg1apy8jdjg"; - }; - } - { - goPackagePath = "google.golang.org/api"; - fetch = { - type = "git"; - url = "https://code.googlesource.com/google-api-go-client"; - rev = "069bea57b1be6ad0671a49ea7a1128025a22b73f"; - sha256 = "19q2b610lkf3z3y9hn6rf11dd78xr9q4340mdyri7kbijlj2r44q"; - }; - } - { - goPackagePath = "google.golang.org/genproto"; - fetch = { - type = "git"; - url = "https://github.com/google/go-genproto"; - rev = "c506a9f9061087022822e8da603a52fc387115a8"; - sha256 = "03hh80aqi58dqi5ykj4shk3chwkzrgq2f3k6qs5qhgvmcy79y2py"; - }; - } - { - goPackagePath = "google.golang.org/grpc"; - fetch = { - type = "git"; - url = "https://github.com/grpc/grpc-go"; - rev = "977142214c45640483838b8672a43c46f89f90cb"; - sha256 = "05wig23l2sil3bfdv19gq62sya7hsabqj9l8pzr1sm57qsvj218d"; - }; - } -] diff --git a/tools/nixery/main.go b/tools/nixery/main.go deleted file mode 100644 index d20ede2eb587..000000000000 --- a/tools/nixery/main.go +++ /dev/null @@ -1,492 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -// Package main provides the implementation of a container registry that -// transparently builds container images based on Nix derivations. -// -// The Nix derivation used for image creation is responsible for creating -// objects that are compatible with the registry API. The targeted registry -// protocol is currently Docker's. -// -// When an image is requested, the required contents are parsed out of the -// request and a Nix-build is initiated that eventually responds with the -// manifest as well as information linking each layer digest to a local -// filesystem path. -package main - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "log" - "net/http" - "os" - "os/exec" - "regexp" - "strings" - "time" - - "cloud.google.com/go/storage" -) - -// pkgSource represents the source from which the Nix package set used -// by Nixery is imported. Users configure the source by setting one of -// the supported environment variables. -type pkgSource struct { - srcType string - args string -} - -// Convert the package source into the representation required by Nix. -func (p *pkgSource) renderSource(tag string) string { - // The 'git' source requires a tag to be present. - if p.srcType == "git" { - if tag == "latest" || tag == "" { - tag = "master" - } - - return fmt.Sprintf("git!%s!%s", p.args, tag) - } - - return fmt.Sprintf("%s!%s", p.srcType, p.args) -} - -// Retrieve a package source from the environment. If no source is -// specified, the Nix code will default to a recent NixOS channel. -func pkgSourceFromEnv() *pkgSource { - if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" { - log.Printf("Using Nix package set from Nix channel %q\n", channel) - return &pkgSource{ - srcType: "nixpkgs", - args: channel, - } - } - - if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" { - log.Printf("Using Nix package set from git repository at %q\n", git) - return &pkgSource{ - srcType: "git", - args: git, - } - } - - if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" { - log.Printf("Using Nix package set from path %q\n", path) - return &pkgSource{ - srcType: "path", - args: path, - } - } - - return nil -} - -// Load (optional) GCS bucket signing data from the GCS_SIGNING_KEY and -// GCS_SIGNING_ACCOUNT envvars. -func signingOptsFromEnv() *storage.SignedURLOptions { - path := os.Getenv("GCS_SIGNING_KEY") - id := os.Getenv("GCS_SIGNING_ACCOUNT") - - if path == "" || id == "" { - log.Println("GCS URL signing disabled") - return nil - } - - log.Printf("GCS URL signing enabled with account %q\n", id) - k, err := ioutil.ReadFile(path) - if err != nil { - log.Fatalf("Failed to read GCS signing key: %s\n", err) - } - - return &storage.SignedURLOptions{ - GoogleAccessID: id, - PrivateKey: k, - Method: "GET", - } -} - -// config holds the Nixery configuration options. -type config struct { - bucket string // GCS bucket to cache & serve layers - signing *storage.SignedURLOptions // Signing options to use for GCS URLs - builder string // Nix derivation for building images - port string // Port on which to launch HTTP server - pkgs *pkgSource // Source for Nix package set -} - -// ManifestMediaType is the Content-Type used for the manifest itself. This -// corresponds to the "Image Manifest V2, Schema 2" described on this page: -// -// https://docs.docker.com/registry/spec/manifest-v2-2/ -const manifestMediaType string = "application/vnd.docker.distribution.manifest.v2+json" - -// Image represents the information necessary for building a container image. -// This can be either a list of package names (corresponding to keys in the -// nixpkgs set) or a Nix expression that results in a *list* of derivations. -type image struct { - name string - tag string - - // Names of packages to include in the image. These must correspond - // directly to top-level names of Nix packages in the nixpkgs tree. - packages []string -} - -// BuildResult represents the output of calling the Nix derivation responsible -// for building registry images. -// -// The `layerLocations` field contains the local filesystem paths to each -// individual image layer that will need to be served, while the `manifest` -// field contains the JSON-representation of the manifest that needs to be -// served to the client. -// -// The later field is simply treated as opaque JSON and passed through. -type BuildResult struct { - Error string `json:"error"` - Pkgs []string `json:"pkgs"` - - Manifest json.RawMessage `json:"manifest"` - LayerLocations map[string]struct { - Path string `json:"path"` - Md5 []byte `json:"md5"` - } `json:"layerLocations"` -} - -// imageFromName parses an image name into the corresponding structure which can -// be used to invoke Nix. -// -// It will expand convenience names under the hood (see the `convenienceNames` -// function below). -func imageFromName(name string, tag string) image { - packages := strings.Split(name, "/") - return image{ - name: name, - tag: tag, - packages: convenienceNames(packages), - } -} - -// convenienceNames expands convenience package names defined by Nixery which -// let users include commonly required sets of tools in a container quickly. -// -// Convenience names must be specified as the first package in an image. -// -// Currently defined convenience names are: -// -// * `shell`: Includes bash, coreutils and other common command-line tools -// * `builder`: All of the above and the standard build environment -func convenienceNames(packages []string) []string { - shellPackages := []string{"bashInteractive", "coreutils", "moreutils", "nano"} - - if packages[0] == "shell" { - return append(packages[1:], shellPackages...) - } - - return packages -} - -// Call out to Nix and request that an image be built. Nix will, upon success, -// return a manifest for the container image. -func buildImage(ctx *context.Context, cfg *config, image *image, bucket *storage.BucketHandle) (*BuildResult, error) { - packages, err := json.Marshal(image.packages) - if err != nil { - return nil, err - } - - args := []string{ - "--no-out-link", - "--show-trace", - "--argstr", "name", image.name, - "--argstr", "packages", string(packages), cfg.builder, - } - - if cfg.pkgs != nil { - args = append(args, "--argstr", "pkgSource", cfg.pkgs.renderSource(image.tag)) - } - cmd := exec.Command("nix-build", args...) - - outpipe, err := cmd.StdoutPipe() - if err != nil { - return nil, err - } - - errpipe, err := cmd.StderrPipe() - if err != nil { - return nil, err - } - - if err = cmd.Start(); err != nil { - log.Println("Error starting nix-build:", err) - return nil, err - } - log.Printf("Started Nix image build for '%s'", image.name) - - stdout, _ := ioutil.ReadAll(outpipe) - stderr, _ := ioutil.ReadAll(errpipe) - - if err = cmd.Wait(); err != nil { - // TODO(tazjin): Propagate errors upwards in a usable format. - log.Printf("nix-build execution error: %s\nstdout: %s\nstderr: %s\n", err, stdout, stderr) - return nil, err - } - - log.Println("Finished Nix image build") - - buildOutput, err := ioutil.ReadFile(strings.TrimSpace(string(stdout))) - if err != nil { - return nil, err - } - - // The build output returned by Nix is deserialised to add all - // contained layers to the bucket. Only the manifest itself is - // re-serialised to JSON and returned. - var result BuildResult - err = json.Unmarshal(buildOutput, &result) - if err != nil { - return nil, err - } - - for layer, meta := range result.LayerLocations { - err = uploadLayer(ctx, bucket, layer, meta.Path, meta.Md5) - if err != nil { - return nil, err - } - } - - return &result, nil -} - -// uploadLayer uploads a single layer to Cloud Storage bucket. Before writing -// any data the bucket is probed to see if the file already exists. -// -// If the file does exist, its MD5 hash is verified to ensure that the stored -// file is not - for example - a fragment of a previous, incomplete upload. -func uploadLayer(ctx *context.Context, bucket *storage.BucketHandle, layer string, path string, md5 []byte) error { - layerKey := fmt.Sprintf("layers/%s", layer) - obj := bucket.Object(layerKey) - - // Before uploading a layer to the bucket, probe whether it already - // exists. - // - // If it does and the MD5 checksum matches the expected one, the layer - // upload can be skipped. - attrs, err := obj.Attrs(*ctx) - - if err == nil && bytes.Equal(attrs.MD5, md5) { - log.Printf("Layer sha256:%s already exists in bucket, skipping upload", layer) - } else { - writer := obj.NewWriter(*ctx) - file, err := os.Open(path) - - if err != nil { - return fmt.Errorf("failed to open layer %s from path %s: %v", layer, path, err) - } - - size, err := io.Copy(writer, file) - if err != nil { - return fmt.Errorf("failed to write layer %s to Cloud Storage: %v", layer, err) - } - - if err = writer.Close(); err != nil { - return fmt.Errorf("failed to write layer %s to Cloud Storage: %v", layer, err) - } - - log.Printf("Uploaded layer sha256:%s (%v bytes written)\n", layer, size) - } - - return nil -} - -// layerRedirect constructs the public URL of the layer object in the Cloud -// Storage bucket, signs it and redirects the user there. -// -// Signing the URL allows unauthenticated clients to retrieve objects from the -// bucket. -// -// The Docker client is known to follow redirects, but this might not be true -// for all other registry clients. -func constructLayerUrl(cfg *config, digest string) (string, error) { - log.Printf("Redirecting layer '%s' request to bucket '%s'\n", digest, cfg.bucket) - object := "layers/" + digest - - if cfg.signing != nil { - opts := *cfg.signing - opts.Expires = time.Now().Add(5 * time.Minute) - return storage.SignedURL(cfg.bucket, object, &opts) - } else { - return ("https://storage.googleapis.com/" + cfg.bucket + "/" + object), nil - } -} - -// prepareBucket configures the handle to a Cloud Storage bucket in which -// individual layers will be stored after Nix builds. Nixery does not directly -// serve layers to registry clients, instead it redirects them to the public -// URLs of the Cloud Storage bucket. -// -// The bucket is required for Nixery to function correctly, hence fatal errors -// are generated in case it fails to be set up correctly. -func prepareBucket(ctx *context.Context, cfg *config) *storage.BucketHandle { - client, err := storage.NewClient(*ctx) - if err != nil { - log.Fatalln("Failed to set up Cloud Storage client:", err) - } - - bkt := client.Bucket(cfg.bucket) - - if _, err := bkt.Attrs(*ctx); err != nil { - log.Fatalln("Could not access configured bucket", err) - } - - return bkt -} - -// Regexes matching the V2 Registry API routes. This only includes the -// routes required for serving images, since pushing and other such -// functionality is not available. -var ( - manifestRegex = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/manifests/([\w|\-|\.|\_]+)$`) - layerRegex = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/blobs/sha256:(\w+)$`) -) - -// Error format corresponding to the registry protocol V2 specification. This -// allows feeding back errors to clients in a way that can be presented to -// users. -type registryError struct { - Code string `json:"code"` - Message string `json:"message"` -} - -type registryErrors struct { - Errors []registryError `json:"errors"` -} - -func writeError(w http.ResponseWriter, status int, code, message string) { - err := registryErrors{ - Errors: []registryError{ - {code, message}, - }, - } - json, _ := json.Marshal(err) - - w.WriteHeader(status) - w.Header().Add("Content-Type", "application/json") - w.Write(json) -} - -type registryHandler struct { - cfg *config - ctx *context.Context - bucket *storage.BucketHandle -} - -func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - // Acknowledge that we speak V2 with an empty response - if r.RequestURI == "/v2/" { - return - } - - // Serve the manifest (straight from Nix) - manifestMatches := manifestRegex.FindStringSubmatch(r.RequestURI) - if len(manifestMatches) == 3 { - imageName := manifestMatches[1] - imageTag := manifestMatches[2] - log.Printf("Requesting manifest for image %q at tag %q", imageName, imageTag) - image := imageFromName(imageName, imageTag) - buildResult, err := buildImage(h.ctx, h.cfg, &image, h.bucket) - - if err != nil { - writeError(w, 500, "UNKNOWN", "image build failure") - log.Println("Failed to build image manifest", err) - return - } - - // Some error types have special handling, which is applied - // here. - if buildResult.Error == "not_found" { - s := fmt.Sprintf("Could not find Nix packages: %v", buildResult.Pkgs) - writeError(w, 404, "MANIFEST_UNKNOWN", s) - log.Println(s) - return - } - - // This marshaling error is ignored because we know that this - // field represents valid JSON data. - manifest, _ := json.Marshal(buildResult.Manifest) - w.Header().Add("Content-Type", manifestMediaType) - w.Write(manifest) - return - } - - // Serve an image layer. For this we need to first ask Nix for - // the manifest, then proceed to extract the correct layer from - // it. - layerMatches := layerRegex.FindStringSubmatch(r.RequestURI) - if len(layerMatches) == 3 { - digest := layerMatches[2] - url, err := constructLayerUrl(h.cfg, digest) - - if err != nil { - log.Printf("Failed to sign GCS URL: %s\n", err) - writeError(w, 500, "UNKNOWN", "could not serve layer") - return - } - - w.Header().Set("Location", url) - w.WriteHeader(303) - return - } - - log.Printf("Unsupported registry route: %s\n", r.RequestURI) - w.WriteHeader(404) -} - -func getConfig(key, desc string) string { - value := os.Getenv(key) - if value == "" { - log.Fatalln(desc + " must be specified") - } - - return value -} - -func main() { - cfg := &config{ - bucket: getConfig("BUCKET", "GCS bucket for layer storage"), - builder: getConfig("NIX_BUILDER", "Nix image builder code"), - port: getConfig("PORT", "HTTP port"), - pkgs: pkgSourceFromEnv(), - signing: signingOptsFromEnv(), - } - - ctx := context.Background() - bucket := prepareBucket(&ctx, cfg) - - log.Printf("Starting Kubernetes Nix controller on port %s\n", cfg.port) - - // All /v2/ requests belong to the registry handler. - http.Handle("/v2/", ®istryHandler{ - cfg: cfg, - ctx: &ctx, - bucket: bucket, - }) - - // All other roots are served by the static file server. - webDir := http.Dir(getConfig("WEB_DIR", "Static web file dir")) - http.Handle("/", http.FileServer(webDir)) - - log.Fatal(http.ListenAndServe(":"+cfg.port, nil)) -} diff --git a/tools/nixery/server/default.nix b/tools/nixery/server/default.nix new file mode 100644 index 000000000000..394d2b27b442 --- /dev/null +++ b/tools/nixery/server/default.nix @@ -0,0 +1,16 @@ +{ buildGoPackage, lib }: + +buildGoPackage { + name = "nixery-server"; + goDeps = ./go-deps.nix; + src = ./.; + + goPackagePath = "github.com/google/nixery"; + + meta = { + description = "Container image builder serving Nix-backed images"; + homepage = "https://github.com/google/nixery"; + license = lib.licenses.asl20; + maintainers = [ lib.maintainers.tazjin ]; + }; +} diff --git a/tools/nixery/server/go-deps.nix b/tools/nixery/server/go-deps.nix new file mode 100644 index 000000000000..a223ef0a7021 --- /dev/null +++ b/tools/nixery/server/go-deps.nix @@ -0,0 +1,111 @@ +# This file was generated by https://github.com/kamilchm/go2nix v1.3.0 +[ + { + goPackagePath = "cloud.google.com/go"; + fetch = { + type = "git"; + url = "https://code.googlesource.com/gocloud"; + rev = "77f6a3a292a7dbf66a5329de0d06326f1906b450"; + sha256 = "1c9pkx782nbcp8jnl5lprcbzf97van789ky5qsncjgywjyymhigi"; + }; + } + { + goPackagePath = "github.com/golang/protobuf"; + fetch = { + type = "git"; + url = "https://github.com/golang/protobuf"; + rev = "6c65a5562fc06764971b7c5d05c76c75e84bdbf7"; + sha256 = "1k1wb4zr0qbwgpvz9q5ws9zhlal8hq7dmq62pwxxriksayl6hzym"; + }; + } + { + goPackagePath = "github.com/googleapis/gax-go"; + fetch = { + type = "git"; + url = "https://github.com/googleapis/gax-go"; + rev = "bd5b16380fd03dc758d11cef74ba2e3bc8b0e8c2"; + sha256 = "1lxawwngv6miaqd25s3ba0didfzylbwisd2nz7r4gmbmin6jsjrx"; + }; + } + { + goPackagePath = "github.com/hashicorp/golang-lru"; + fetch = { + type = "git"; + url = "https://github.com/hashicorp/golang-lru"; + rev = "59383c442f7d7b190497e9bb8fc17a48d06cd03f"; + sha256 = "0yzwl592aa32vfy73pl7wdc21855w17zssrp85ckw2nisky8rg9c"; + }; + } + { + goPackagePath = "go.opencensus.io"; + fetch = { + type = "git"; + url = "https://github.com/census-instrumentation/opencensus-go"; + rev = "b4a14686f0a98096416fe1b4cb848e384fb2b22b"; + sha256 = "1aidyp301v5ngwsnnc8v1s09vvbsnch1jc4vd615f7qv77r9s7dn"; + }; + } + { + goPackagePath = "golang.org/x/net"; + fetch = { + type = "git"; + url = "https://go.googlesource.com/net"; + rev = "da137c7871d730100384dbcf36e6f8fa493aef5b"; + sha256 = "1qsiyr3irmb6ii06hivm9p2c7wqyxczms1a9v1ss5698yjr3fg47"; + }; + } + { + goPackagePath = "golang.org/x/oauth2"; + fetch = { + type = "git"; + url = "https://go.googlesource.com/oauth2"; + rev = "0f29369cfe4552d0e4bcddc57cc75f4d7e672a33"; + sha256 = "06jwpvx0x2gjn2y959drbcir5kd7vg87k0r1216abk6rrdzzrzi2"; + }; + } + { + goPackagePath = "golang.org/x/sys"; + fetch = { + type = "git"; + url = "https://go.googlesource.com/sys"; + rev = "51ab0e2deafac1f46c46ad59cf0921be2f180c3d"; + sha256 = "0xdhpckbql3bsqkpc2k5b1cpnq3q1qjqjjq2j3p707rfwb8nm91a"; + }; + } + { + goPackagePath = "golang.org/x/text"; + fetch = { + type = "git"; + url = "https://go.googlesource.com/text"; + rev = "342b2e1fbaa52c93f31447ad2c6abc048c63e475"; + sha256 = "0flv9idw0jm5nm8lx25xqanbkqgfiym6619w575p7nrdh0riqwqh"; + }; + } + { + goPackagePath = "google.golang.org/api"; + fetch = { + type = "git"; + url = "https://code.googlesource.com/google-api-go-client"; + rev = "069bea57b1be6ad0671a49ea7a1128025a22b73f"; + sha256 = "19q2b610lkf3z3y9hn6rf11dd78xr9q4340mdyri7kbijlj2r44q"; + }; + } + { + goPackagePath = "google.golang.org/genproto"; + fetch = { + type = "git"; + url = "https://github.com/google/go-genproto"; + rev = "c506a9f9061087022822e8da603a52fc387115a8"; + sha256 = "03hh80aqi58dqi5ykj4shk3chwkzrgq2f3k6qs5qhgvmcy79y2py"; + }; + } + { + goPackagePath = "google.golang.org/grpc"; + fetch = { + type = "git"; + url = "https://github.com/grpc/grpc-go"; + rev = "977142214c45640483838b8672a43c46f89f90cb"; + sha256 = "05wig23l2sil3bfdv19gq62sya7hsabqj9l8pzr1sm57qsvj218d"; + }; + } +] diff --git a/tools/nixery/server/main.go b/tools/nixery/server/main.go new file mode 100644 index 000000000000..d20ede2eb587 --- /dev/null +++ b/tools/nixery/server/main.go @@ -0,0 +1,492 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +// Package main provides the implementation of a container registry that +// transparently builds container images based on Nix derivations. +// +// The Nix derivation used for image creation is responsible for creating +// objects that are compatible with the registry API. The targeted registry +// protocol is currently Docker's. +// +// When an image is requested, the required contents are parsed out of the +// request and a Nix-build is initiated that eventually responds with the +// manifest as well as information linking each layer digest to a local +// filesystem path. +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "regexp" + "strings" + "time" + + "cloud.google.com/go/storage" +) + +// pkgSource represents the source from which the Nix package set used +// by Nixery is imported. Users configure the source by setting one of +// the supported environment variables. +type pkgSource struct { + srcType string + args string +} + +// Convert the package source into the representation required by Nix. +func (p *pkgSource) renderSource(tag string) string { + // The 'git' source requires a tag to be present. + if p.srcType == "git" { + if tag == "latest" || tag == "" { + tag = "master" + } + + return fmt.Sprintf("git!%s!%s", p.args, tag) + } + + return fmt.Sprintf("%s!%s", p.srcType, p.args) +} + +// Retrieve a package source from the environment. If no source is +// specified, the Nix code will default to a recent NixOS channel. +func pkgSourceFromEnv() *pkgSource { + if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" { + log.Printf("Using Nix package set from Nix channel %q\n", channel) + return &pkgSource{ + srcType: "nixpkgs", + args: channel, + } + } + + if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" { + log.Printf("Using Nix package set from git repository at %q\n", git) + return &pkgSource{ + srcType: "git", + args: git, + } + } + + if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" { + log.Printf("Using Nix package set from path %q\n", path) + return &pkgSource{ + srcType: "path", + args: path, + } + } + + return nil +} + +// Load (optional) GCS bucket signing data from the GCS_SIGNING_KEY and +// GCS_SIGNING_ACCOUNT envvars. +func signingOptsFromEnv() *storage.SignedURLOptions { + path := os.Getenv("GCS_SIGNING_KEY") + id := os.Getenv("GCS_SIGNING_ACCOUNT") + + if path == "" || id == "" { + log.Println("GCS URL signing disabled") + return nil + } + + log.Printf("GCS URL signing enabled with account %q\n", id) + k, err := ioutil.ReadFile(path) + if err != nil { + log.Fatalf("Failed to read GCS signing key: %s\n", err) + } + + return &storage.SignedURLOptions{ + GoogleAccessID: id, + PrivateKey: k, + Method: "GET", + } +} + +// config holds the Nixery configuration options. +type config struct { + bucket string // GCS bucket to cache & serve layers + signing *storage.SignedURLOptions // Signing options to use for GCS URLs + builder string // Nix derivation for building images + port string // Port on which to launch HTTP server + pkgs *pkgSource // Source for Nix package set +} + +// ManifestMediaType is the Content-Type used for the manifest itself. This +// corresponds to the "Image Manifest V2, Schema 2" described on this page: +// +// https://docs.docker.com/registry/spec/manifest-v2-2/ +const manifestMediaType string = "application/vnd.docker.distribution.manifest.v2+json" + +// Image represents the information necessary for building a container image. +// This can be either a list of package names (corresponding to keys in the +// nixpkgs set) or a Nix expression that results in a *list* of derivations. +type image struct { + name string + tag string + + // Names of packages to include in the image. These must correspond + // directly to top-level names of Nix packages in the nixpkgs tree. + packages []string +} + +// BuildResult represents the output of calling the Nix derivation responsible +// for building registry images. +// +// The `layerLocations` field contains the local filesystem paths to each +// individual image layer that will need to be served, while the `manifest` +// field contains the JSON-representation of the manifest that needs to be +// served to the client. +// +// The later field is simply treated as opaque JSON and passed through. +type BuildResult struct { + Error string `json:"error"` + Pkgs []string `json:"pkgs"` + + Manifest json.RawMessage `json:"manifest"` + LayerLocations map[string]struct { + Path string `json:"path"` + Md5 []byte `json:"md5"` + } `json:"layerLocations"` +} + +// imageFromName parses an image name into the corresponding structure which can +// be used to invoke Nix. +// +// It will expand convenience names under the hood (see the `convenienceNames` +// function below). +func imageFromName(name string, tag string) image { + packages := strings.Split(name, "/") + return image{ + name: name, + tag: tag, + packages: convenienceNames(packages), + } +} + +// convenienceNames expands convenience package names defined by Nixery which +// let users include commonly required sets of tools in a container quickly. +// +// Convenience names must be specified as the first package in an image. +// +// Currently defined convenience names are: +// +// * `shell`: Includes bash, coreutils and other common command-line tools +// * `builder`: All of the above and the standard build environment +func convenienceNames(packages []string) []string { + shellPackages := []string{"bashInteractive", "coreutils", "moreutils", "nano"} + + if packages[0] == "shell" { + return append(packages[1:], shellPackages...) + } + + return packages +} + +// Call out to Nix and request that an image be built. Nix will, upon success, +// return a manifest for the container image. +func buildImage(ctx *context.Context, cfg *config, image *image, bucket *storage.BucketHandle) (*BuildResult, error) { + packages, err := json.Marshal(image.packages) + if err != nil { + return nil, err + } + + args := []string{ + "--no-out-link", + "--show-trace", + "--argstr", "name", image.name, + "--argstr", "packages", string(packages), cfg.builder, + } + + if cfg.pkgs != nil { + args = append(args, "--argstr", "pkgSource", cfg.pkgs.renderSource(image.tag)) + } + cmd := exec.Command("nix-build", args...) + + outpipe, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + + errpipe, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + + if err = cmd.Start(); err != nil { + log.Println("Error starting nix-build:", err) + return nil, err + } + log.Printf("Started Nix image build for '%s'", image.name) + + stdout, _ := ioutil.ReadAll(outpipe) + stderr, _ := ioutil.ReadAll(errpipe) + + if err = cmd.Wait(); err != nil { + // TODO(tazjin): Propagate errors upwards in a usable format. + log.Printf("nix-build execution error: %s\nstdout: %s\nstderr: %s\n", err, stdout, stderr) + return nil, err + } + + log.Println("Finished Nix image build") + + buildOutput, err := ioutil.ReadFile(strings.TrimSpace(string(stdout))) + if err != nil { + return nil, err + } + + // The build output returned by Nix is deserialised to add all + // contained layers to the bucket. Only the manifest itself is + // re-serialised to JSON and returned. + var result BuildResult + err = json.Unmarshal(buildOutput, &result) + if err != nil { + return nil, err + } + + for layer, meta := range result.LayerLocations { + err = uploadLayer(ctx, bucket, layer, meta.Path, meta.Md5) + if err != nil { + return nil, err + } + } + + return &result, nil +} + +// uploadLayer uploads a single layer to Cloud Storage bucket. Before writing +// any data the bucket is probed to see if the file already exists. +// +// If the file does exist, its MD5 hash is verified to ensure that the stored +// file is not - for example - a fragment of a previous, incomplete upload. +func uploadLayer(ctx *context.Context, bucket *storage.BucketHandle, layer string, path string, md5 []byte) error { + layerKey := fmt.Sprintf("layers/%s", layer) + obj := bucket.Object(layerKey) + + // Before uploading a layer to the bucket, probe whether it already + // exists. + // + // If it does and the MD5 checksum matches the expected one, the layer + // upload can be skipped. + attrs, err := obj.Attrs(*ctx) + + if err == nil && bytes.Equal(attrs.MD5, md5) { + log.Printf("Layer sha256:%s already exists in bucket, skipping upload", layer) + } else { + writer := obj.NewWriter(*ctx) + file, err := os.Open(path) + + if err != nil { + return fmt.Errorf("failed to open layer %s from path %s: %v", layer, path, err) + } + + size, err := io.Copy(writer, file) + if err != nil { + return fmt.Errorf("failed to write layer %s to Cloud Storage: %v", layer, err) + } + + if err = writer.Close(); err != nil { + return fmt.Errorf("failed to write layer %s to Cloud Storage: %v", layer, err) + } + + log.Printf("Uploaded layer sha256:%s (%v bytes written)\n", layer, size) + } + + return nil +} + +// layerRedirect constructs the public URL of the layer object in the Cloud +// Storage bucket, signs it and redirects the user there. +// +// Signing the URL allows unauthenticated clients to retrieve objects from the +// bucket. +// +// The Docker client is known to follow redirects, but this might not be true +// for all other registry clients. +func constructLayerUrl(cfg *config, digest string) (string, error) { + log.Printf("Redirecting layer '%s' request to bucket '%s'\n", digest, cfg.bucket) + object := "layers/" + digest + + if cfg.signing != nil { + opts := *cfg.signing + opts.Expires = time.Now().Add(5 * time.Minute) + return storage.SignedURL(cfg.bucket, object, &opts) + } else { + return ("https://storage.googleapis.com/" + cfg.bucket + "/" + object), nil + } +} + +// prepareBucket configures the handle to a Cloud Storage bucket in which +// individual layers will be stored after Nix builds. Nixery does not directly +// serve layers to registry clients, instead it redirects them to the public +// URLs of the Cloud Storage bucket. +// +// The bucket is required for Nixery to function correctly, hence fatal errors +// are generated in case it fails to be set up correctly. +func prepareBucket(ctx *context.Context, cfg *config) *storage.BucketHandle { + client, err := storage.NewClient(*ctx) + if err != nil { + log.Fatalln("Failed to set up Cloud Storage client:", err) + } + + bkt := client.Bucket(cfg.bucket) + + if _, err := bkt.Attrs(*ctx); err != nil { + log.Fatalln("Could not access configured bucket", err) + } + + return bkt +} + +// Regexes matching the V2 Registry API routes. This only includes the +// routes required for serving images, since pushing and other such +// functionality is not available. +var ( + manifestRegex = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/manifests/([\w|\-|\.|\_]+)$`) + layerRegex = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/blobs/sha256:(\w+)$`) +) + +// Error format corresponding to the registry protocol V2 specification. This +// allows feeding back errors to clients in a way that can be presented to +// users. +type registryError struct { + Code string `json:"code"` + Message string `json:"message"` +} + +type registryErrors struct { + Errors []registryError `json:"errors"` +} + +func writeError(w http.ResponseWriter, status int, code, message string) { + err := registryErrors{ + Errors: []registryError{ + {code, message}, + }, + } + json, _ := json.Marshal(err) + + w.WriteHeader(status) + w.Header().Add("Content-Type", "application/json") + w.Write(json) +} + +type registryHandler struct { + cfg *config + ctx *context.Context + bucket *storage.BucketHandle +} + +func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Acknowledge that we speak V2 with an empty response + if r.RequestURI == "/v2/" { + return + } + + // Serve the manifest (straight from Nix) + manifestMatches := manifestRegex.FindStringSubmatch(r.RequestURI) + if len(manifestMatches) == 3 { + imageName := manifestMatches[1] + imageTag := manifestMatches[2] + log.Printf("Requesting manifest for image %q at tag %q", imageName, imageTag) + image := imageFromName(imageName, imageTag) + buildResult, err := buildImage(h.ctx, h.cfg, &image, h.bucket) + + if err != nil { + writeError(w, 500, "UNKNOWN", "image build failure") + log.Println("Failed to build image manifest", err) + return + } + + // Some error types have special handling, which is applied + // here. + if buildResult.Error == "not_found" { + s := fmt.Sprintf("Could not find Nix packages: %v", buildResult.Pkgs) + writeError(w, 404, "MANIFEST_UNKNOWN", s) + log.Println(s) + return + } + + // This marshaling error is ignored because we know that this + // field represents valid JSON data. + manifest, _ := json.Marshal(buildResult.Manifest) + w.Header().Add("Content-Type", manifestMediaType) + w.Write(manifest) + return + } + + // Serve an image layer. For this we need to first ask Nix for + // the manifest, then proceed to extract the correct layer from + // it. + layerMatches := layerRegex.FindStringSubmatch(r.RequestURI) + if len(layerMatches) == 3 { + digest := layerMatches[2] + url, err := constructLayerUrl(h.cfg, digest) + + if err != nil { + log.Printf("Failed to sign GCS URL: %s\n", err) + writeError(w, 500, "UNKNOWN", "could not serve layer") + return + } + + w.Header().Set("Location", url) + w.WriteHeader(303) + return + } + + log.Printf("Unsupported registry route: %s\n", r.RequestURI) + w.WriteHeader(404) +} + +func getConfig(key, desc string) string { + value := os.Getenv(key) + if value == "" { + log.Fatalln(desc + " must be specified") + } + + return value +} + +func main() { + cfg := &config{ + bucket: getConfig("BUCKET", "GCS bucket for layer storage"), + builder: getConfig("NIX_BUILDER", "Nix image builder code"), + port: getConfig("PORT", "HTTP port"), + pkgs: pkgSourceFromEnv(), + signing: signingOptsFromEnv(), + } + + ctx := context.Background() + bucket := prepareBucket(&ctx, cfg) + + log.Printf("Starting Kubernetes Nix controller on port %s\n", cfg.port) + + // All /v2/ requests belong to the registry handler. + http.Handle("/v2/", ®istryHandler{ + cfg: cfg, + ctx: &ctx, + bucket: bucket, + }) + + // All other roots are served by the static file server. + webDir := http.Dir(getConfig("WEB_DIR", "Static web file dir")) + http.Handle("/", http.FileServer(webDir)) + + log.Fatal(http.ListenAndServe(":"+cfg.port, nil)) +} -- cgit 1.4.1