diff options
| -rw-r--r-- | tools/nixery/server/builder/builder.go | 208 | ||||
| -rw-r--r-- | tools/nixery/server/config/config.go | 131 | ||||
| -rw-r--r-- | tools/nixery/server/main.go | 330 |
3 files changed, 365 insertions, 304 deletions
diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/server/builder/builder.go new file mode 100644 index 000000000000..c53b702e0537 --- /dev/null +++ b/tools/nixery/server/builder/builder.go @@ -0,0 +1,208 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +// Package builder implements the code required to build images via Nix. Image +// build data is cached for up to 24 hours to avoid duplicated calls to Nix +// (which are costly even if no building is performed). +package builder + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "strings" + + "cloud.google.com/go/storage" + "github.com/google/nixery/config" +) + +// Image represents the information necessary for building a container image. +// This can be either a list of package names (corresponding to keys in the +// nixpkgs set) or a Nix expression that results in a *list* of derivations. +type Image struct { + Name string + Tag string + + // Names of packages to include in the image. These must correspond + // directly to top-level names of Nix packages in the nixpkgs tree. + Packages []string +} + +// ImageFromName parses an image name into the corresponding structure which can +// be used to invoke Nix. +// +// It will expand convenience names under the hood (see the `convenienceNames` +// function below). +func ImageFromName(name string, tag string) Image { + packages := strings.Split(name, "/") + return Image{ + Name: name, + Tag: tag, + Packages: convenienceNames(packages), + } +} + +// BuildResult represents the output of calling the Nix derivation responsible +// for building registry images. +// +// The `layerLocations` field contains the local filesystem paths to each +// individual image layer that will need to be served, while the `manifest` +// field contains the JSON-representation of the manifest that needs to be +// served to the client. +// +// The later field is simply treated as opaque JSON and passed through. +type BuildResult struct { + Error string `json:"error"` + Pkgs []string `json:"pkgs"` + + Manifest json.RawMessage `json:"manifest"` + LayerLocations map[string]struct { + Path string `json:"path"` + Md5 []byte `json:"md5"` + } `json:"layerLocations"` +} + +// convenienceNames expands convenience package names defined by Nixery which +// let users include commonly required sets of tools in a container quickly. +// +// Convenience names must be specified as the first package in an image. +// +// Currently defined convenience names are: +// +// * `shell`: Includes bash, coreutils and other common command-line tools +func convenienceNames(packages []string) []string { + shellPackages := []string{"bashInteractive", "coreutils", "moreutils", "nano"} + + if packages[0] == "shell" { + return append(packages[1:], shellPackages...) + } + + return packages +} + +// Call out to Nix and request that an image be built. Nix will, upon success, +// return a manifest for the container image. +func BuildImage(ctx *context.Context, cfg *config.Config, image *Image, bucket *storage.BucketHandle) (*BuildResult, error) { + packages, err := json.Marshal(image.Packages) + if err != nil { + return nil, err + } + + args := []string{ + "--argstr", "name", image.Name, + "--argstr", "packages", string(packages), + } + + if cfg.Pkgs != nil { + args = append(args, "--argstr", "pkgSource", cfg.Pkgs.Render(image.Tag)) + } + cmd := exec.Command("nixery-build-image", args...) + + outpipe, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + + errpipe, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + + if err = cmd.Start(); err != nil { + log.Println("Error starting nix-build:", err) + return nil, err + } + log.Printf("Started Nix image build for '%s'", image.Name) + + stdout, _ := ioutil.ReadAll(outpipe) + stderr, _ := ioutil.ReadAll(errpipe) + + if err = cmd.Wait(); err != nil { + // TODO(tazjin): Propagate errors upwards in a usable format. + log.Printf("nix-build execution error: %s\nstdout: %s\nstderr: %s\n", err, stdout, stderr) + return nil, err + } + + log.Println("Finished Nix image build") + + buildOutput, err := ioutil.ReadFile(strings.TrimSpace(string(stdout))) + if err != nil { + return nil, err + } + + // The build output returned by Nix is deserialised to add all + // contained layers to the bucket. Only the manifest itself is + // re-serialised to JSON and returned. + var result BuildResult + err = json.Unmarshal(buildOutput, &result) + if err != nil { + return nil, err + } + + for layer, meta := range result.LayerLocations { + err = uploadLayer(ctx, bucket, layer, meta.Path, meta.Md5) + if err != nil { + return nil, err + } + } + + return &result, nil +} + +// uploadLayer uploads a single layer to Cloud Storage bucket. Before writing +// any data the bucket is probed to see if the file already exists. +// +// If the file does exist, its MD5 hash is verified to ensure that the stored +// file is not - for example - a fragment of a previous, incomplete upload. +func uploadLayer(ctx *context.Context, bucket *storage.BucketHandle, layer string, path string, md5 []byte) error { + layerKey := fmt.Sprintf("layers/%s", layer) + obj := bucket.Object(layerKey) + + // Before uploading a layer to the bucket, probe whether it already + // exists. + // + // If it does and the MD5 checksum matches the expected one, the layer + // upload can be skipped. + attrs, err := obj.Attrs(*ctx) + + if err == nil && bytes.Equal(attrs.MD5, md5) { + log.Printf("Layer sha256:%s already exists in bucket, skipping upload", layer) + } else { + writer := obj.NewWriter(*ctx) + file, err := os.Open(path) + + if err != nil { + return fmt.Errorf("failed to open layer %s from path %s: %v", layer, path, err) + } + + size, err := io.Copy(writer, file) + if err != nil { + return fmt.Errorf("failed to write layer %s to Cloud Storage: %v", layer, err) + } + + if err = writer.Close(); err != nil { + return fmt.Errorf("failed to write layer %s to Cloud Storage: %v", layer, err) + } + + log.Printf("Uploaded layer sha256:%s (%v bytes written)\n", layer, size) + } + + return nil +} diff --git a/tools/nixery/server/config/config.go b/tools/nixery/server/config/config.go new file mode 100644 index 000000000000..4e3b70dcdc22 --- /dev/null +++ b/tools/nixery/server/config/config.go @@ -0,0 +1,131 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +// Package config implements structures to store Nixery's configuration at +// runtime as well as the logic for instantiating this configuration from the +// environment. +package config + +import ( + "fmt" + "io/ioutil" + "log" + "os" + + "cloud.google.com/go/storage" +) + +// pkgSource represents the source from which the Nix package set used +// by Nixery is imported. Users configure the source by setting one of +// the supported environment variables. +type PkgSource struct { + srcType string + args string +} + +// Convert the package source into the representation required by Nix. +func (p *PkgSource) Render(tag string) string { + // The 'git' source requires a tag to be present. + if p.srcType == "git" { + if tag == "latest" || tag == "" { + tag = "master" + } + + return fmt.Sprintf("git!%s!%s", p.args, tag) + } + + return fmt.Sprintf("%s!%s", p.srcType, p.args) +} + +// Retrieve a package source from the environment. If no source is +// specified, the Nix code will default to a recent NixOS channel. +func pkgSourceFromEnv() *PkgSource { + if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" { + log.Printf("Using Nix package set from Nix channel %q\n", channel) + return &PkgSource{ + srcType: "nixpkgs", + args: channel, + } + } + + if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" { + log.Printf("Using Nix package set from git repository at %q\n", git) + return &PkgSource{ + srcType: "git", + args: git, + } + } + + if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" { + log.Printf("Using Nix package set from path %q\n", path) + return &PkgSource{ + srcType: "path", + args: path, + } + } + + return nil +} + +// Load (optional) GCS bucket signing data from the GCS_SIGNING_KEY and +// GCS_SIGNING_ACCOUNT envvars. +func signingOptsFromEnv() *storage.SignedURLOptions { + path := os.Getenv("GCS_SIGNING_KEY") + id := os.Getenv("GCS_SIGNING_ACCOUNT") + + if path == "" || id == "" { + log.Println("GCS URL signing disabled") + return nil + } + + log.Printf("GCS URL signing enabled with account %q\n", id) + k, err := ioutil.ReadFile(path) + if err != nil { + log.Fatalf("Failed to read GCS signing key: %s\n", err) + } + + return &storage.SignedURLOptions{ + GoogleAccessID: id, + PrivateKey: k, + Method: "GET", + } +} + +func getConfig(key, desc string) string { + value := os.Getenv(key) + if value == "" { + log.Fatalln(desc + " must be specified") + } + + return value +} + +// config holds the Nixery configuration options. +type Config struct { + Bucket string // GCS bucket to cache & serve layers + Signing *storage.SignedURLOptions // Signing options to use for GCS URLs + Port string // Port on which to launch HTTP server + Pkgs *PkgSource // Source for Nix package set + WebDir string +} + +func FromEnv() *Config { + return &Config{ + Bucket: getConfig("BUCKET", "GCS bucket for layer storage"), + Port: getConfig("PORT", "HTTP port"), + Pkgs: pkgSourceFromEnv(), + Signing: signingOptsFromEnv(), + WebDir: getConfig("WEB_DIR", "Static web file dir"), + } +} diff --git a/tools/nixery/server/main.go b/tools/nixery/server/main.go index 3e015e8587fc..5d7dcd2adfc2 100644 --- a/tools/nixery/server/main.go +++ b/tools/nixery/server/main.go @@ -12,8 +12,8 @@ // License for the specific language governing permissions and limitations under // the License. -// Package main provides the implementation of a container registry that -// transparently builds container images based on Nix derivations. +// The nixery server implements a container registry that transparently builds +// container images based on Nix derivations. // // The Nix derivation used for image creation is responsible for creating // objects that are compatible with the registry API. The targeted registry @@ -26,287 +26,32 @@ package main import ( - "bytes" "context" "encoding/json" "fmt" - "io" - "io/ioutil" "log" "net/http" - "os" - "os/exec" "regexp" - "strings" "time" "cloud.google.com/go/storage" + "github.com/google/nixery/builder" + "github.com/google/nixery/config" ) -// pkgSource represents the source from which the Nix package set used -// by Nixery is imported. Users configure the source by setting one of -// the supported environment variables. -type pkgSource struct { - srcType string - args string -} - -// Convert the package source into the representation required by Nix. -func (p *pkgSource) renderSource(tag string) string { - // The 'git' source requires a tag to be present. - if p.srcType == "git" { - if tag == "latest" || tag == "" { - tag = "master" - } - - return fmt.Sprintf("git!%s!%s", p.args, tag) - } - - return fmt.Sprintf("%s!%s", p.srcType, p.args) -} - -// Retrieve a package source from the environment. If no source is -// specified, the Nix code will default to a recent NixOS channel. -func pkgSourceFromEnv() *pkgSource { - if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" { - log.Printf("Using Nix package set from Nix channel %q\n", channel) - return &pkgSource{ - srcType: "nixpkgs", - args: channel, - } - } - - if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" { - log.Printf("Using Nix package set from git repository at %q\n", git) - return &pkgSource{ - srcType: "git", - args: git, - } - } - - if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" { - log.Printf("Using Nix package set from path %q\n", path) - return &pkgSource{ - srcType: "path", - args: path, - } - } - - return nil -} - -// Load (optional) GCS bucket signing data from the GCS_SIGNING_KEY and -// GCS_SIGNING_ACCOUNT envvars. -func signingOptsFromEnv() *storage.SignedURLOptions { - path := os.Getenv("GCS_SIGNING_KEY") - id := os.Getenv("GCS_SIGNING_ACCOUNT") - - if path == "" || id == "" { - log.Println("GCS URL signing disabled") - return nil - } - - log.Printf("GCS URL signing enabled with account %q\n", id) - k, err := ioutil.ReadFile(path) - if err != nil { - log.Fatalf("Failed to read GCS signing key: %s\n", err) - } - - return &storage.SignedURLOptions{ - GoogleAccessID: id, - PrivateKey: k, - Method: "GET", - } -} - -// config holds the Nixery configuration options. -type config struct { - bucket string // GCS bucket to cache & serve layers - signing *storage.SignedURLOptions // Signing options to use for GCS URLs - port string // Port on which to launch HTTP server - pkgs *pkgSource // Source for Nix package set -} - // ManifestMediaType is the Content-Type used for the manifest itself. This // corresponds to the "Image Manifest V2, Schema 2" described on this page: // // https://docs.docker.com/registry/spec/manifest-v2-2/ const manifestMediaType string = "application/vnd.docker.distribution.manifest.v2+json" -// Image represents the information necessary for building a container image. -// This can be either a list of package names (corresponding to keys in the -// nixpkgs set) or a Nix expression that results in a *list* of derivations. -type image struct { - name string - tag string - - // Names of packages to include in the image. These must correspond - // directly to top-level names of Nix packages in the nixpkgs tree. - packages []string -} - -// BuildResult represents the output of calling the Nix derivation responsible -// for building registry images. -// -// The `layerLocations` field contains the local filesystem paths to each -// individual image layer that will need to be served, while the `manifest` -// field contains the JSON-representation of the manifest that needs to be -// served to the client. -// -// The later field is simply treated as opaque JSON and passed through. -type BuildResult struct { - Error string `json:"error"` - Pkgs []string `json:"pkgs"` - - Manifest json.RawMessage `json:"manifest"` - LayerLocations map[string]struct { - Path string `json:"path"` - Md5 []byte `json:"md5"` - } `json:"layerLocations"` -} - -// imageFromName parses an image name into the corresponding structure which can -// be used to invoke Nix. -// -// It will expand convenience names under the hood (see the `convenienceNames` -// function below). -func imageFromName(name string, tag string) image { - packages := strings.Split(name, "/") - return image{ - name: name, - tag: tag, - packages: convenienceNames(packages), - } -} - -// convenienceNames expands convenience package names defined by Nixery which -// let users include commonly required sets of tools in a container quickly. -// -// Convenience names must be specified as the first package in an image. -// -// Currently defined convenience names are: -// -// * `shell`: Includes bash, coreutils and other common command-line tools -// * `builder`: All of the above and the standard build environment -func convenienceNames(packages []string) []string { - shellPackages := []string{"bashInteractive", "coreutils", "moreutils", "nano"} - - if packages[0] == "shell" { - return append(packages[1:], shellPackages...) - } - - return packages -} - -// Call out to Nix and request that an image be built. Nix will, upon success, -// return a manifest for the container image. -func buildImage(ctx *context.Context, cfg *config, image *image, bucket *storage.BucketHandle) (*BuildResult, error) { - packages, err := json.Marshal(image.packages) - if err != nil { - return nil, err - } - - args := []string{ - "--argstr", "name", image.name, - "--argstr", "packages", string(packages), - } - - if cfg.pkgs != nil { - args = append(args, "--argstr", "pkgSource", cfg.pkgs.renderSource(image.tag)) - } - cmd := exec.Command("nixery-build-image", args...) - - outpipe, err := cmd.StdoutPipe() - if err != nil { - return nil, err - } - - errpipe, err := cmd.StderrPipe() - if err != nil { - return nil, err - } - - if err = cmd.Start(); err != nil { - log.Println("Error starting nix-build:", err) - return nil, err - } - log.Printf("Started Nix image build for '%s'", image.name) - - stdout, _ := ioutil.ReadAll(outpipe) - stderr, _ := ioutil.ReadAll(errpipe) - - if err = cmd.Wait(); err != nil { - // TODO(tazjin): Propagate errors upwards in a usable format. - log.Printf("nix-build execution error: %s\nstdout: %s\nstderr: %s\n", err, stdout, stderr) - return nil, err - } - - log.Println("Finished Nix image build") - - buildOutput, err := ioutil.ReadFile(strings.TrimSpace(string(stdout))) - if err != nil { - return nil, err - } - - // The build output returned by Nix is deserialised to add all - // contained layers to the bucket. Only the manifest itself is - // re-serialised to JSON and returned. - var result BuildResult - err = json.Unmarshal(buildOutput, &result) - if err != nil { - return nil, err - } - - for layer, meta := range result.LayerLocations { - err = uploadLayer(ctx, bucket, layer, meta.Path, meta.Md5) - if err != nil { - return nil, err - } - } - - return &result, nil -} - -// uploadLayer uploads a single layer to Cloud Storage bucket. Before writing -// any data the bucket is probed to see if the file already exists. -// -// If the file does exist, its MD5 hash is verified to ensure that the stored -// file is not - for example - a fragment of a previous, incomplete upload. -func uploadLayer(ctx *context.Context, bucket *storage.BucketHandle, layer string, path string, md5 []byte) error { - layerKey := fmt.Sprintf("layers/%s", layer) - obj := bucket.Object(layerKey) - - // Before uploading a layer to the bucket, probe whether it already - // exists. - // - // If it does and the MD5 checksum matches the expected one, the layer - // upload can be skipped. - attrs, err := obj.Attrs(*ctx) - - if err == nil && bytes.Equal(attrs.MD5, md5) { - log.Printf("Layer sha256:%s already exists in bucket, skipping upload", layer) - } else { - writer := obj.NewWriter(*ctx) - file, err := os.Open(path) - - if err != nil { - return fmt.Errorf("failed to open layer %s from path %s: %v", layer, path, err) - } - - size, err := io.Copy(writer, file) - if err != nil { - return fmt.Errorf("failed to write layer %s to Cloud Storage: %v", layer, err) - } - - if err = writer.Close(); err != nil { - return fmt.Errorf("failed to write layer %s to Cloud Storage: %v", layer, err) - } - - log.Printf("Uploaded layer sha256:%s (%v bytes written)\n", layer, size) - } - - return nil -} +// Regexes matching the V2 Registry API routes. This only includes the +// routes required for serving images, since pushing and other such +// functionality is not available. +var ( + manifestRegex = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/manifests/([\w|\-|\.|\_]+)$`) + layerRegex = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/blobs/sha256:(\w+)$`) +) // layerRedirect constructs the public URL of the layer object in the Cloud // Storage bucket, signs it and redirects the user there. @@ -316,16 +61,16 @@ func uploadLayer(ctx *context.Context, bucket *storage.BucketHandle, layer strin // // The Docker client is known to follow redirects, but this might not be true // for all other registry clients. -func constructLayerUrl(cfg *config, digest string) (string, error) { - log.Printf("Redirecting layer '%s' request to bucket '%s'\n", digest, cfg.bucket) +func constructLayerUrl(cfg *config.Config, digest string) (string, error) { + log.Printf("Redirecting layer '%s' request to bucket '%s'\n", digest, cfg.Bucket) object := "layers/" + digest - if cfg.signing != nil { - opts := *cfg.signing + if cfg.Signing != nil { + opts := *cfg.Signing opts.Expires = time.Now().Add(5 * time.Minute) - return storage.SignedURL(cfg.bucket, object, &opts) + return storage.SignedURL(cfg.Bucket, object, &opts) } else { - return ("https://storage.googleapis.com/" + cfg.bucket + "/" + object), nil + return ("https://storage.googleapis.com/" + cfg.Bucket + "/" + object), nil } } @@ -336,13 +81,13 @@ func constructLayerUrl(cfg *config, digest string) (string, error) { // // The bucket is required for Nixery to function correctly, hence fatal errors // are generated in case it fails to be set up correctly. -func prepareBucket(ctx *context.Context, cfg *config) *storage.BucketHandle { +func prepareBucket(ctx *context.Context, cfg *config.Config) *storage.BucketHandle { client, err := storage.NewClient(*ctx) if err != nil { log.Fatalln("Failed to set up Cloud Storage client:", err) } - bkt := client.Bucket(cfg.bucket) + bkt := client.Bucket(cfg.Bucket) if _, err := bkt.Attrs(*ctx); err != nil { log.Fatalln("Could not access configured bucket", err) @@ -351,14 +96,6 @@ func prepareBucket(ctx *context.Context, cfg *config) *storage.BucketHandle { return bkt } -// Regexes matching the V2 Registry API routes. This only includes the -// routes required for serving images, since pushing and other such -// functionality is not available. -var ( - manifestRegex = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/manifests/([\w|\-|\.|\_]+)$`) - layerRegex = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/blobs/sha256:(\w+)$`) -) - // Error format corresponding to the registry protocol V2 specification. This // allows feeding back errors to clients in a way that can be presented to // users. @@ -385,7 +122,7 @@ func writeError(w http.ResponseWriter, status int, code, message string) { } type registryHandler struct { - cfg *config + cfg *config.Config ctx *context.Context bucket *storage.BucketHandle } @@ -402,8 +139,8 @@ func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { imageName := manifestMatches[1] imageTag := manifestMatches[2] log.Printf("Requesting manifest for image %q at tag %q", imageName, imageTag) - image := imageFromName(imageName, imageTag) - buildResult, err := buildImage(h.ctx, h.cfg, &image, h.bucket) + image := builder.ImageFromName(imageName, imageTag) + buildResult, err := builder.BuildImage(h.ctx, h.cfg, &image, h.bucket) if err != nil { writeError(w, 500, "UNKNOWN", "image build failure") @@ -451,27 +188,12 @@ func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(404) } -func getConfig(key, desc string) string { - value := os.Getenv(key) - if value == "" { - log.Fatalln(desc + " must be specified") - } - - return value -} - func main() { - cfg := &config{ - bucket: getConfig("BUCKET", "GCS bucket for layer storage"), - port: getConfig("PORT", "HTTP port"), - pkgs: pkgSourceFromEnv(), - signing: signingOptsFromEnv(), - } - + cfg := config.FromEnv() ctx := context.Background() bucket := prepareBucket(&ctx, cfg) - log.Printf("Starting Kubernetes Nix controller on port %s\n", cfg.port) + log.Printf("Starting Kubernetes Nix controller on port %s\n", cfg.Port) // All /v2/ requests belong to the registry handler. http.Handle("/v2/", ®istryHandler{ @@ -481,8 +203,8 @@ func main() { }) // All other roots are served by the static file server. - webDir := http.Dir(getConfig("WEB_DIR", "Static web file dir")) + webDir := http.Dir(cfg.WebDir) http.Handle("/", http.FileServer(webDir)) - log.Fatal(http.ListenAndServe(":"+cfg.port, nil)) + log.Fatal(http.ListenAndServe(":"+cfg.Port, nil)) } |