From 2b82f1b71a50b8b1473421cce0eec1a0d7ddc360 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Mon, 11 Nov 2019 21:07:16 +0000 Subject: refactor: Reshuffle file structure for better code layout This gets rid of the package called "server" and instead moves everything into the project root, such that Go actually builds us a binary called `nixery`. This is the first step towards factoring out CLI-based functionality for Nixery. --- tools/nixery/build-image/build-image.nix | 173 --------- tools/nixery/build-image/default.nix | 29 -- tools/nixery/build-image/load-pkgs.nix | 45 --- tools/nixery/builder/archive.go | 113 ++++++ tools/nixery/builder/builder.go | 521 +++++++++++++++++++++++++++ tools/nixery/builder/builder_test.go | 123 +++++++ tools/nixery/builder/cache.go | 236 ++++++++++++ tools/nixery/builder/layers.go | 364 +++++++++++++++++++ tools/nixery/config/config.go | 84 +++++ tools/nixery/config/pkgsource.go | 159 ++++++++ tools/nixery/default.nix | 44 ++- tools/nixery/go-deps.nix | 129 +++++++ tools/nixery/logs/logs.go | 119 ++++++ tools/nixery/main.go | 249 +++++++++++++ tools/nixery/manifest/manifest.go | 141 ++++++++ tools/nixery/popcount/popcount.go | 2 +- tools/nixery/prepare-image/default.nix | 29 ++ tools/nixery/prepare-image/load-pkgs.nix | 45 +++ tools/nixery/prepare-image/prepare-image.nix | 173 +++++++++ tools/nixery/server/builder/archive.go | 116 ------ tools/nixery/server/builder/builder.go | 521 --------------------------- tools/nixery/server/builder/builder_test.go | 123 ------- tools/nixery/server/builder/cache.go | 236 ------------ tools/nixery/server/config/config.go | 84 ----- tools/nixery/server/config/pkgsource.go | 159 -------- tools/nixery/server/default.nix | 62 ---- tools/nixery/server/go-deps.nix | 129 ------- tools/nixery/server/layers/grouping.go | 361 ------------------- tools/nixery/server/logs.go | 119 ------ tools/nixery/server/main.go | 248 ------------- tools/nixery/server/manifest/manifest.go | 141 -------- tools/nixery/server/storage/filesystem.go | 96 ----- tools/nixery/server/storage/gcs.go | 219 ----------- tools/nixery/server/storage/storage.go | 51 --- tools/nixery/shell.nix | 2 +- tools/nixery/storage/filesystem.go | 96 +++++ tools/nixery/storage/gcs.go | 219 +++++++++++ tools/nixery/storage/storage.go | 51 +++ 38 files changed, 2890 insertions(+), 2921 deletions(-) delete mode 100644 tools/nixery/build-image/build-image.nix delete mode 100644 tools/nixery/build-image/default.nix delete mode 100644 tools/nixery/build-image/load-pkgs.nix create mode 100644 tools/nixery/builder/archive.go create mode 100644 tools/nixery/builder/builder.go create mode 100644 tools/nixery/builder/builder_test.go create mode 100644 tools/nixery/builder/cache.go create mode 100644 tools/nixery/builder/layers.go create mode 100644 tools/nixery/config/config.go create mode 100644 tools/nixery/config/pkgsource.go create mode 100644 tools/nixery/go-deps.nix create mode 100644 tools/nixery/logs/logs.go create mode 100644 tools/nixery/main.go create mode 100644 tools/nixery/manifest/manifest.go create mode 100644 tools/nixery/prepare-image/default.nix create mode 100644 tools/nixery/prepare-image/load-pkgs.nix create mode 100644 tools/nixery/prepare-image/prepare-image.nix delete mode 100644 tools/nixery/server/builder/archive.go delete mode 100644 tools/nixery/server/builder/builder.go delete mode 100644 tools/nixery/server/builder/builder_test.go delete mode 100644 tools/nixery/server/builder/cache.go delete mode 100644 tools/nixery/server/config/config.go delete mode 100644 tools/nixery/server/config/pkgsource.go delete mode 100644 tools/nixery/server/default.nix delete mode 100644 tools/nixery/server/go-deps.nix delete mode 100644 tools/nixery/server/layers/grouping.go delete mode 100644 tools/nixery/server/logs.go delete mode 100644 tools/nixery/server/main.go delete mode 100644 tools/nixery/server/manifest/manifest.go delete mode 100644 tools/nixery/server/storage/filesystem.go delete mode 100644 tools/nixery/server/storage/gcs.go delete mode 100644 tools/nixery/server/storage/storage.go create mode 100644 tools/nixery/storage/filesystem.go create mode 100644 tools/nixery/storage/gcs.go create mode 100644 tools/nixery/storage/storage.go (limited to 'tools/nixery') diff --git a/tools/nixery/build-image/build-image.nix b/tools/nixery/build-image/build-image.nix deleted file mode 100644 index 4393f2b859a6..000000000000 --- a/tools/nixery/build-image/build-image.nix +++ /dev/null @@ -1,173 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file contains a derivation that outputs structured information -# about the runtime dependencies of an image with a given set of -# packages. This is used by Nixery to determine the layer grouping and -# assemble each layer. -# -# In addition it creates and outputs a meta-layer with the symlink -# structure required for using the image together with the individual -# package layers. - -{ - # Description of the package set to be used (will be loaded by load-pkgs.nix) - srcType ? "nixpkgs", - srcArgs ? "nixos-19.03", - system ? "x86_64-linux", - importArgs ? { }, - # Path to load-pkgs.nix - loadPkgs ? ./load-pkgs.nix, - # Packages to install by name (which must refer to top-level attributes of - # nixpkgs). This is passed in as a JSON-array in string form. - packages ? "[]" -}: - -let - inherit (builtins) - foldl' - fromJSON - hasAttr - length - match - readFile - toFile - toJSON; - - # Package set to use for sourcing utilities - nativePkgs = import loadPkgs { inherit srcType srcArgs importArgs; }; - inherit (nativePkgs) coreutils jq openssl lib runCommand writeText symlinkJoin; - - # Package set to use for packages to be included in the image. This - # package set is imported with the system set to the target - # architecture. - pkgs = import loadPkgs { - inherit srcType srcArgs; - importArgs = importArgs // { - inherit system; - }; - }; - - # deepFetch traverses the top-level Nix package set to retrieve an item via a - # path specified in string form. - # - # For top-level items, the name of the key yields the result directly. Nested - # items are fetched by using dot-syntax, as in Nix itself. - # - # Due to a restriction of the registry API specification it is not possible to - # pass uppercase characters in an image name, however the Nix package set - # makes use of camelCasing repeatedly (for example for `haskellPackages`). - # - # To work around this, if no value is found on the top-level a second lookup - # is done on the package set using lowercase-names. This is not done for - # nested sets, as they often have keys that only differ in case. - # - # For example, `deepFetch pkgs "xorg.xev"` retrieves `pkgs.xorg.xev` and - # `deepFetch haskellpackages.stylish-haskell` retrieves - # `haskellPackages.stylish-haskell`. - deepFetch = with lib; s: n: - let path = splitString "." n; - err = { error = "not_found"; pkg = n; }; - # The most efficient way I've found to do a lookup against - # case-differing versions of an attribute is to first construct a - # mapping of all lowercased attribute names to their differently cased - # equivalents. - # - # This map is then used for a second lookup if the top-level - # (case-sensitive) one does not yield a result. - hasUpper = str: (match ".*[A-Z].*" str) != null; - allUpperKeys = filter hasUpper (attrNames s); - lowercased = listToAttrs (map (k: { - name = toLower k; - value = k; - }) allUpperKeys); - caseAmendedPath = map (v: if hasAttr v lowercased then lowercased."${v}" else v) path; - fetchLower = attrByPath caseAmendedPath err s; - in attrByPath path fetchLower s; - - # allContents contains all packages successfully retrieved by name - # from the package set, as well as any errors encountered while - # attempting to fetch a package. - # - # Accumulated error information is returned back to the server. - allContents = - # Folds over the results of 'deepFetch' on all requested packages to - # separate them into errors and content. This allows the program to - # terminate early and return only the errors if any are encountered. - let splitter = attrs: res: - if hasAttr "error" res - then attrs // { errors = attrs.errors ++ [ res ]; } - else attrs // { contents = attrs.contents ++ [ res ]; }; - init = { contents = []; errors = []; }; - fetched = (map (deepFetch pkgs) (fromJSON packages)); - in foldl' splitter init fetched; - - # Contains the export references graph of all retrieved packages, - # which has information about all runtime dependencies of the image. - # - # This is used by Nixery to group closures into image layers. - runtimeGraph = runCommand "runtime-graph.json" { - __structuredAttrs = true; - exportReferencesGraph.graph = allContents.contents; - PATH = "${coreutils}/bin"; - builder = toFile "builder" '' - . .attrs.sh - cp .attrs.json ''${outputs[out]} - ''; - } ""; - - # Create a symlink forest into all top-level store paths of the - # image contents. - contentsEnv = symlinkJoin { - name = "bulk-layers"; - paths = allContents.contents; - }; - - # Image layer that contains the symlink forest created above. This - # must be included in the image to ensure that the filesystem has a - # useful layout at runtime. - symlinkLayer = runCommand "symlink-layer.tar" {} '' - cp -r ${contentsEnv}/ ./layer - tar --transform='s|^\./||' -C layer --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 -cf $out . - ''; - - # Metadata about the symlink layer which is required for serving it. - # Two different hashes are computed for different usages (inclusion - # in manifest vs. content-checking in the layer cache). - symlinkLayerMeta = fromJSON (readFile (runCommand "symlink-layer-meta.json" { - buildInputs = [ coreutils jq openssl ]; - }'' - tarHash=$(sha256sum ${symlinkLayer} | cut -d ' ' -f1) - layerSize=$(stat --printf '%s' ${symlinkLayer}) - - jq -n -c --arg tarHash $tarHash --arg size $layerSize --arg path ${symlinkLayer} \ - '{ size: ($size | tonumber), tarHash: $tarHash, path: $path }' >> $out - '')); - - # Final output structure returned to Nixery if the build succeeded - buildOutput = { - runtimeGraph = fromJSON (readFile runtimeGraph); - symlinkLayer = symlinkLayerMeta; - }; - - # Output structure returned if errors occured during the build. Currently the - # only error type that is returned in a structured way is 'not_found'. - errorOutput = { - error = "not_found"; - pkgs = map (err: err.pkg) allContents.errors; - }; -in writeText "build-output.json" (if (length allContents.errors) == 0 - then toJSON buildOutput - else toJSON errorOutput -) diff --git a/tools/nixery/build-image/default.nix b/tools/nixery/build-image/default.nix deleted file mode 100644 index a61ac06bdd92..000000000000 --- a/tools/nixery/build-image/default.nix +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file builds a wrapper script called by Nixery to ask for the -# content information for a given image. -# -# The purpose of using a wrapper script is to ensure that the paths to -# all required Nix files are set correctly at runtime. - -{ pkgs ? import {} }: - -pkgs.writeShellScriptBin "nixery-build-image" '' - exec ${pkgs.nix}/bin/nix-build \ - --show-trace \ - --no-out-link "$@" \ - --argstr loadPkgs ${./load-pkgs.nix} \ - ${./build-image.nix} -'' diff --git a/tools/nixery/build-image/load-pkgs.nix b/tools/nixery/build-image/load-pkgs.nix deleted file mode 100644 index cceebfc14dae..000000000000 --- a/tools/nixery/build-image/load-pkgs.nix +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Load a Nix package set from one of the supported source types -# (nixpkgs, git, path). -{ srcType, srcArgs, importArgs ? { } }: - -with builtins; -let - # If a nixpkgs channel is requested, it is retrieved from Github (as - # a tarball) and imported. - fetchImportChannel = channel: - let - url = - "https://github.com/NixOS/nixpkgs-channels/archive/${channel}.tar.gz"; - in import (fetchTarball url) importArgs; - - # If a git repository is requested, it is retrieved via - # builtins.fetchGit which defaults to the git configuration of the - # outside environment. This means that user-configured SSH - # credentials etc. are going to work as expected. - fetchImportGit = spec: import (fetchGit spec) importArgs; - - # No special handling is used for paths, so users are expected to pass one - # that will work natively with Nix. - importPath = path: import (toPath path) importArgs; -in if srcType == "nixpkgs" then - fetchImportChannel srcArgs -else if srcType == "git" then - fetchImportGit (fromJSON srcArgs) -else if srcType == "path" then - importPath srcArgs -else - throw ("Invalid package set source specification: ${srcType} (${srcArgs})") diff --git a/tools/nixery/builder/archive.go b/tools/nixery/builder/archive.go new file mode 100644 index 000000000000..ff822e389a7d --- /dev/null +++ b/tools/nixery/builder/archive.go @@ -0,0 +1,113 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +package builder + +// This file implements logic for walking through a directory and creating a +// tarball of it. +// +// The tarball is written straight to the supplied reader, which makes it +// possible to create an image layer from the specified store paths, hash it and +// upload it in one reading pass. +import ( + "archive/tar" + "compress/gzip" + "crypto/sha256" + "fmt" + "io" + "os" + "path/filepath" +) + +// Create a new compressed tarball from each of the paths in the list +// and write it to the supplied writer. +// +// The uncompressed tarball is hashed because image manifests must +// contain both the hashes of compressed and uncompressed layers. +func packStorePaths(l *layer, w io.Writer) (string, error) { + shasum := sha256.New() + gz := gzip.NewWriter(w) + multi := io.MultiWriter(shasum, gz) + t := tar.NewWriter(multi) + + for _, path := range l.Contents { + err := filepath.Walk(path, tarStorePath(t)) + if err != nil { + return "", err + } + } + + if err := t.Close(); err != nil { + return "", err + } + + if err := gz.Close(); err != nil { + return "", err + } + + return fmt.Sprintf("sha256:%x", shasum.Sum([]byte{})), nil +} + +func tarStorePath(w *tar.Writer) filepath.WalkFunc { + return func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // If the entry is not a symlink or regular file, skip it. + if info.Mode()&os.ModeSymlink == 0 && !info.Mode().IsRegular() { + return nil + } + + // the symlink target is read if this entry is a symlink, as it + // is required when creating the file header + var link string + if info.Mode()&os.ModeSymlink != 0 { + link, err = os.Readlink(path) + if err != nil { + return err + } + } + + header, err := tar.FileInfoHeader(info, link) + if err != nil { + return err + } + + // The name retrieved from os.FileInfo only contains the file's + // basename, but the full path is required within the layer + // tarball. + header.Name = path + if err = w.WriteHeader(header); err != nil { + return err + } + + // At this point, return if no file content needs to be written + if !info.Mode().IsRegular() { + return nil + } + + f, err := os.Open(path) + if err != nil { + return err + } + + if _, err := io.Copy(w, f); err != nil { + return err + } + + f.Close() + + return nil + } +} diff --git a/tools/nixery/builder/builder.go b/tools/nixery/builder/builder.go new file mode 100644 index 000000000000..ceb112df90cf --- /dev/null +++ b/tools/nixery/builder/builder.go @@ -0,0 +1,521 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +// Package builder implements the logic for assembling container +// images. It shells out to Nix to retrieve all required Nix-packages +// and assemble the symlink layer and then creates the required +// tarballs in-process. +package builder + +import ( + "bufio" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "sort" + "strings" + + "github.com/google/nixery/config" + "github.com/google/nixery/manifest" + "github.com/google/nixery/storage" + log "github.com/sirupsen/logrus" +) + +// The maximum number of layers in an image is 125. To allow for +// extensibility, the actual number of layers Nixery is "allowed" to +// use up is set at a lower point. +const LayerBudget int = 94 + +// State holds the runtime state that is carried around in Nixery and +// passed to builder functions. +type State struct { + Storage storage.Backend + Cache *LocalCache + Cfg config.Config + Pop Popularity +} + +// Architecture represents the possible CPU architectures for which +// container images can be built. +// +// The default architecture is amd64, but support for ARM platforms is +// available within nixpkgs and can be toggled via meta-packages. +type Architecture struct { + // Name of the system tuple to pass to Nix + nixSystem string + + // Name of the architecture as used in the OCI manifests + imageArch string +} + +var amd64 = Architecture{"x86_64-linux", "amd64"} +var arm64 = Architecture{"aarch64-linux", "arm64"} + +// Image represents the information necessary for building a container image. +// This can be either a list of package names (corresponding to keys in the +// nixpkgs set) or a Nix expression that results in a *list* of derivations. +type Image struct { + Name string + Tag string + + // Names of packages to include in the image. These must correspond + // directly to top-level names of Nix packages in the nixpkgs tree. + Packages []string + + // Architecture for which to build the image. Nixery defaults + // this to amd64 if not specified via meta-packages. + Arch *Architecture +} + +// BuildResult represents the data returned from the server to the +// HTTP handlers. Error information is propagated straight from Nix +// for errors inside of the build that should be fed back to the +// client (such as missing packages). +type BuildResult struct { + Error string `json:"error"` + Pkgs []string `json:"pkgs"` + Manifest json.RawMessage `json:"manifest"` +} + +// ImageFromName parses an image name into the corresponding structure which can +// be used to invoke Nix. +// +// It will expand convenience names under the hood (see the `convenienceNames` +// function below) and append packages that are always included (cacert, iana-etc). +// +// Once assembled the image structure uses a sorted representation of +// the name. This is to avoid unnecessarily cache-busting images if +// only the order of requested packages has changed. +func ImageFromName(name string, tag string) Image { + pkgs := strings.Split(name, "/") + arch, expanded := metaPackages(pkgs) + expanded = append(expanded, "cacert", "iana-etc") + + sort.Strings(pkgs) + sort.Strings(expanded) + + return Image{ + Name: strings.Join(pkgs, "/"), + Tag: tag, + Packages: expanded, + Arch: arch, + } +} + +// ImageResult represents the output of calling the Nix derivation +// responsible for preparing an image. +type ImageResult struct { + // These fields are populated in case of an error + Error string `json:"error"` + Pkgs []string `json:"pkgs"` + + // These fields are populated in case of success + Graph runtimeGraph `json:"runtimeGraph"` + SymlinkLayer struct { + Size int `json:"size"` + TarHash string `json:"tarHash"` + Path string `json:"path"` + } `json:"symlinkLayer"` +} + +// metaPackages expands package names defined by Nixery which either +// include sets of packages or trigger certain image-building +// behaviour. +// +// Meta-packages must be specified as the first packages in an image +// name. +// +// Currently defined meta-packages are: +// +// * `shell`: Includes bash, coreutils and other common command-line tools +// * `arm64`: Causes Nixery to build images for the ARM64 architecture +func metaPackages(packages []string) (*Architecture, []string) { + arch := &amd64 + + var metapkgs []string + lastMeta := 0 + for idx, p := range packages { + if p == "shell" || p == "arm64" { + metapkgs = append(metapkgs, p) + lastMeta = idx + 1 + } else { + break + } + } + + // Chop off the meta-packages from the front of the package + // list + packages = packages[lastMeta:] + + for _, p := range metapkgs { + switch p { + case "shell": + packages = append(packages, "bashInteractive", "coreutils", "moreutils", "nano") + case "arm64": + arch = &arm64 + } + } + + return arch, packages +} + +// logNix logs each output line from Nix. It runs in a goroutine per +// output channel that should be live-logged. +func logNix(image, cmd string, r io.ReadCloser) { + scanner := bufio.NewScanner(r) + for scanner.Scan() { + log.WithFields(log.Fields{ + "image": image, + "cmd": cmd, + }).Info("[nix] " + scanner.Text()) + } +} + +func callNix(program, image string, args []string) ([]byte, error) { + cmd := exec.Command(program, args...) + + outpipe, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + + errpipe, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + go logNix(program, image, errpipe) + + if err = cmd.Start(); err != nil { + log.WithError(err).WithFields(log.Fields{ + "image": image, + "cmd": program, + }).Error("error invoking Nix") + + return nil, err + } + + log.WithFields(log.Fields{ + "cmd": program, + "image": image, + }).Info("invoked Nix build") + + stdout, _ := ioutil.ReadAll(outpipe) + + if err = cmd.Wait(); err != nil { + log.WithError(err).WithFields(log.Fields{ + "image": image, + "cmd": program, + "stdout": stdout, + }).Info("failed to invoke Nix") + + return nil, err + } + + resultFile := strings.TrimSpace(string(stdout)) + buildOutput, err := ioutil.ReadFile(resultFile) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "image": image, + "file": resultFile, + }).Info("failed to read Nix result file") + + return nil, err + } + + return buildOutput, nil +} + +// Call out to Nix and request metadata for the image to be built. All +// required store paths for the image will be realised, but layers +// will not yet be created from them. +// +// This function is only invoked if the manifest is not found in any +// cache. +func prepareImage(s *State, image *Image) (*ImageResult, error) { + packages, err := json.Marshal(image.Packages) + if err != nil { + return nil, err + } + + srcType, srcArgs := s.Cfg.Pkgs.Render(image.Tag) + + args := []string{ + "--timeout", s.Cfg.Timeout, + "--argstr", "packages", string(packages), + "--argstr", "srcType", srcType, + "--argstr", "srcArgs", srcArgs, + "--argstr", "system", image.Arch.nixSystem, + } + + output, err := callNix("nixery-prepare-image", image.Name, args) + if err != nil { + // granular error logging is performed in callNix already + return nil, err + } + + log.WithFields(log.Fields{ + "image": image.Name, + "tag": image.Tag, + }).Info("finished image preparation via Nix") + + var result ImageResult + err = json.Unmarshal(output, &result) + if err != nil { + return nil, err + } + + return &result, nil +} + +// Groups layers and checks whether they are present in the cache +// already, otherwise calls out to Nix to assemble layers. +// +// Newly built layers are uploaded to the bucket. Cache entries are +// added only after successful uploads, which guarantees that entries +// retrieved from the cache are present in the bucket. +func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageResult) ([]manifest.Entry, error) { + grouped := groupLayers(&result.Graph, &s.Pop, LayerBudget) + + var entries []manifest.Entry + + // Splits the layers into those which are already present in + // the cache, and those that are missing. + // + // Missing layers are built and uploaded to the storage + // bucket. + for _, l := range grouped { + if entry, cached := layerFromCache(ctx, s, l.Hash()); cached { + entries = append(entries, *entry) + } else { + lh := l.Hash() + + // While packing store paths, the SHA sum of + // the uncompressed layer is computed and + // written to `tarhash`. + // + // TODO(tazjin): Refactor this to make the + // flow of data cleaner. + var tarhash string + lw := func(w io.Writer) error { + var err error + tarhash, err = packStorePaths(&l, w) + return err + } + + entry, err := uploadHashLayer(ctx, s, lh, lw) + if err != nil { + return nil, err + } + entry.MergeRating = l.MergeRating + entry.TarHash = tarhash + + var pkgs []string + for _, p := range l.Contents { + pkgs = append(pkgs, packageFromPath(p)) + } + + log.WithFields(log.Fields{ + "layer": lh, + "packages": pkgs, + "tarhash": tarhash, + }).Info("created image layer") + + go cacheLayer(ctx, s, l.Hash(), *entry) + entries = append(entries, *entry) + } + } + + // Symlink layer (built in the first Nix build) needs to be + // included here manually: + slkey := result.SymlinkLayer.TarHash + entry, err := uploadHashLayer(ctx, s, slkey, func(w io.Writer) error { + f, err := os.Open(result.SymlinkLayer.Path) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "image": image.Name, + "tag": image.Tag, + "layer": slkey, + }).Error("failed to open symlink layer") + + return err + } + defer f.Close() + + gz := gzip.NewWriter(w) + _, err = io.Copy(gz, f) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "image": image.Name, + "tag": image.Tag, + "layer": slkey, + }).Error("failed to upload symlink layer") + + return err + } + + return gz.Close() + }) + + if err != nil { + return nil, err + } + + entry.TarHash = "sha256:" + result.SymlinkLayer.TarHash + go cacheLayer(ctx, s, slkey, *entry) + entries = append(entries, *entry) + + return entries, nil +} + +// layerWriter is the type for functions that can write a layer to the +// multiwriter used for uploading & hashing. +// +// This type exists to avoid duplication between the handling of +// symlink layers and store path layers. +type layerWriter func(w io.Writer) error + +// byteCounter is a special io.Writer that counts all bytes written to +// it and does nothing else. +// +// This is required because the ad-hoc writing of tarballs leaves no +// single place to count the final tarball size otherwise. +type byteCounter struct { + count int64 +} + +func (b *byteCounter) Write(p []byte) (n int, err error) { + b.count += int64(len(p)) + return len(p), nil +} + +// Upload a layer tarball to the storage bucket, while hashing it at +// the same time. The supplied function is expected to provide the +// layer data to the writer. +// +// The initial upload is performed in a 'staging' folder, as the +// SHA256-hash is not yet available when the upload is initiated. +// +// After a successful upload, the file is moved to its final location +// in the bucket and the build cache is populated. +// +// The return value is the layer's SHA256 hash, which is used in the +// image manifest. +func uploadHashLayer(ctx context.Context, s *State, key string, lw layerWriter) (*manifest.Entry, error) { + path := "staging/" + key + sha256sum, size, err := s.Storage.Persist(ctx, path, func(sw io.Writer) (string, int64, error) { + // Sets up a "multiwriter" that simultaneously runs both hash + // algorithms and uploads to the storage backend. + shasum := sha256.New() + counter := &byteCounter{} + multi := io.MultiWriter(sw, shasum, counter) + + err := lw(multi) + sha256sum := fmt.Sprintf("%x", shasum.Sum([]byte{})) + + return sha256sum, counter.count, err + }) + + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "layer": key, + "backend": s.Storage.Name(), + }).Error("failed to create and store layer") + + return nil, err + } + + // Hashes are now known and the object is in the bucket, what + // remains is to move it to the correct location and cache it. + err = s.Storage.Move(ctx, "staging/"+key, "layers/"+sha256sum) + if err != nil { + log.WithError(err).WithField("layer", key). + Error("failed to move layer from staging") + + return nil, err + } + + log.WithFields(log.Fields{ + "layer": key, + "sha256": sha256sum, + "size": size, + }).Info("created and persisted layer") + + entry := manifest.Entry{ + Digest: "sha256:" + sha256sum, + Size: size, + } + + return &entry, nil +} + +func BuildImage(ctx context.Context, s *State, image *Image) (*BuildResult, error) { + key := s.Cfg.Pkgs.CacheKey(image.Packages, image.Tag) + if key != "" { + if m, c := manifestFromCache(ctx, s, key); c { + return &BuildResult{ + Manifest: m, + }, nil + } + } + + imageResult, err := prepareImage(s, image) + if err != nil { + return nil, err + } + + if imageResult.Error != "" { + return &BuildResult{ + Error: imageResult.Error, + Pkgs: imageResult.Pkgs, + }, nil + } + + layers, err := prepareLayers(ctx, s, image, imageResult) + if err != nil { + return nil, err + } + + m, c := manifest.Manifest(image.Arch.imageArch, layers) + + lw := func(w io.Writer) error { + r := bytes.NewReader(c.Config) + _, err := io.Copy(w, r) + return err + } + + if _, err = uploadHashLayer(ctx, s, c.SHA256, lw); err != nil { + log.WithError(err).WithFields(log.Fields{ + "image": image.Name, + "tag": image.Tag, + }).Error("failed to upload config") + + return nil, err + } + + if key != "" { + go cacheManifest(ctx, s, key, m) + } + + result := BuildResult{ + Manifest: m, + } + return &result, nil +} diff --git a/tools/nixery/builder/builder_test.go b/tools/nixery/builder/builder_test.go new file mode 100644 index 000000000000..3fbe2ab40e23 --- /dev/null +++ b/tools/nixery/builder/builder_test.go @@ -0,0 +1,123 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +package builder + +import ( + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "testing" +) + +var ignoreArch = cmpopts.IgnoreFields(Image{}, "Arch") + +func TestImageFromNameSimple(t *testing.T) { + image := ImageFromName("hello", "latest") + expected := Image{ + Name: "hello", + Tag: "latest", + Packages: []string{ + "cacert", + "hello", + "iana-etc", + }, + } + + if diff := cmp.Diff(expected, image, ignoreArch); diff != "" { + t.Fatalf("Image(\"hello\", \"latest\") mismatch:\n%s", diff) + } +} + +func TestImageFromNameMultiple(t *testing.T) { + image := ImageFromName("hello/git/htop", "latest") + expected := Image{ + Name: "git/hello/htop", + Tag: "latest", + Packages: []string{ + "cacert", + "git", + "hello", + "htop", + "iana-etc", + }, + } + + if diff := cmp.Diff(expected, image, ignoreArch); diff != "" { + t.Fatalf("Image(\"hello/git/htop\", \"latest\") mismatch:\n%s", diff) + } +} + +func TestImageFromNameShell(t *testing.T) { + image := ImageFromName("shell", "latest") + expected := Image{ + Name: "shell", + Tag: "latest", + Packages: []string{ + "bashInteractive", + "cacert", + "coreutils", + "iana-etc", + "moreutils", + "nano", + }, + } + + if diff := cmp.Diff(expected, image, ignoreArch); diff != "" { + t.Fatalf("Image(\"shell\", \"latest\") mismatch:\n%s", diff) + } +} + +func TestImageFromNameShellMultiple(t *testing.T) { + image := ImageFromName("shell/htop", "latest") + expected := Image{ + Name: "htop/shell", + Tag: "latest", + Packages: []string{ + "bashInteractive", + "cacert", + "coreutils", + "htop", + "iana-etc", + "moreutils", + "nano", + }, + } + + if diff := cmp.Diff(expected, image, ignoreArch); diff != "" { + t.Fatalf("Image(\"shell/htop\", \"latest\") mismatch:\n%s", diff) + } +} + +func TestImageFromNameShellArm64(t *testing.T) { + image := ImageFromName("shell/arm64", "latest") + expected := Image{ + Name: "arm64/shell", + Tag: "latest", + Packages: []string{ + "bashInteractive", + "cacert", + "coreutils", + "iana-etc", + "moreutils", + "nano", + }, + } + + if diff := cmp.Diff(expected, image, ignoreArch); diff != "" { + t.Fatalf("Image(\"shell/arm64\", \"latest\") mismatch:\n%s", diff) + } + + if image.Arch.imageArch != "arm64" { + t.Fatal("Image(\"shell/arm64\"): Expected arch arm64") + } +} diff --git a/tools/nixery/builder/cache.go b/tools/nixery/builder/cache.go new file mode 100644 index 000000000000..a4ebe03e1c94 --- /dev/null +++ b/tools/nixery/builder/cache.go @@ -0,0 +1,236 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +package builder + +import ( + "bytes" + "context" + "encoding/json" + "io" + "io/ioutil" + "os" + "sync" + + "github.com/google/nixery/manifest" + log "github.com/sirupsen/logrus" +) + +// LocalCache implements the structure used for local caching of +// manifests and layer uploads. +type LocalCache struct { + // Manifest cache + mmtx sync.RWMutex + mdir string + + // Layer cache + lmtx sync.RWMutex + lcache map[string]manifest.Entry +} + +// Creates an in-memory cache and ensures that the local file path for +// manifest caching exists. +func NewCache() (LocalCache, error) { + path := os.TempDir() + "/nixery" + err := os.MkdirAll(path, 0755) + if err != nil { + return LocalCache{}, err + } + + return LocalCache{ + mdir: path + "/", + lcache: make(map[string]manifest.Entry), + }, nil +} + +// Retrieve a cached manifest if the build is cacheable and it exists. +func (c *LocalCache) manifestFromLocalCache(key string) (json.RawMessage, bool) { + c.mmtx.RLock() + defer c.mmtx.RUnlock() + + f, err := os.Open(c.mdir + key) + if err != nil { + // This is a debug log statement because failure to + // read the manifest key is currently expected if it + // is not cached. + log.WithError(err).WithField("manifest", key). + Debug("failed to read manifest from local cache") + + return nil, false + } + defer f.Close() + + m, err := ioutil.ReadAll(f) + if err != nil { + log.WithError(err).WithField("manifest", key). + Error("failed to read manifest from local cache") + + return nil, false + } + + return json.RawMessage(m), true +} + +// Adds the result of a manifest build to the local cache, if the +// manifest is considered cacheable. +// +// Manifests can be quite large and are cached on disk instead of in +// memory. +func (c *LocalCache) localCacheManifest(key string, m json.RawMessage) { + c.mmtx.Lock() + defer c.mmtx.Unlock() + + err := ioutil.WriteFile(c.mdir+key, []byte(m), 0644) + if err != nil { + log.WithError(err).WithField("manifest", key). + Error("failed to locally cache manifest") + } +} + +// Retrieve a layer build from the local cache. +func (c *LocalCache) layerFromLocalCache(key string) (*manifest.Entry, bool) { + c.lmtx.RLock() + e, ok := c.lcache[key] + c.lmtx.RUnlock() + + return &e, ok +} + +// Add a layer build result to the local cache. +func (c *LocalCache) localCacheLayer(key string, e manifest.Entry) { + c.lmtx.Lock() + c.lcache[key] = e + c.lmtx.Unlock() +} + +// Retrieve a manifest from the cache(s). First the local cache is +// checked, then the storage backend. +func manifestFromCache(ctx context.Context, s *State, key string) (json.RawMessage, bool) { + if m, cached := s.Cache.manifestFromLocalCache(key); cached { + return m, true + } + + r, err := s.Storage.Fetch(ctx, "manifests/"+key) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "manifest": key, + "backend": s.Storage.Name(), + }).Error("failed to fetch manifest from cache") + + return nil, false + } + defer r.Close() + + m, err := ioutil.ReadAll(r) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "manifest": key, + "backend": s.Storage.Name(), + }).Error("failed to read cached manifest from storage backend") + + return nil, false + } + + go s.Cache.localCacheManifest(key, m) + log.WithField("manifest", key).Info("retrieved manifest from GCS") + + return json.RawMessage(m), true +} + +// Add a manifest to the bucket & local caches +func cacheManifest(ctx context.Context, s *State, key string, m json.RawMessage) { + go s.Cache.localCacheManifest(key, m) + + path := "manifests/" + key + _, size, err := s.Storage.Persist(ctx, path, func(w io.Writer) (string, int64, error) { + size, err := io.Copy(w, bytes.NewReader([]byte(m))) + return "", size, err + }) + + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "manifest": key, + "backend": s.Storage.Name(), + }).Error("failed to cache manifest to storage backend") + + return + } + + log.WithFields(log.Fields{ + "manifest": key, + "size": size, + "backend": s.Storage.Name(), + }).Info("cached manifest to storage backend") +} + +// Retrieve a layer build from the cache, first checking the local +// cache followed by the bucket cache. +func layerFromCache(ctx context.Context, s *State, key string) (*manifest.Entry, bool) { + if entry, cached := s.Cache.layerFromLocalCache(key); cached { + return entry, true + } + + r, err := s.Storage.Fetch(ctx, "builds/"+key) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "layer": key, + "backend": s.Storage.Name(), + }).Debug("failed to retrieve cached layer from storage backend") + + return nil, false + } + defer r.Close() + + jb := bytes.NewBuffer([]byte{}) + _, err = io.Copy(jb, r) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "layer": key, + "backend": s.Storage.Name(), + }).Error("failed to read cached layer from storage backend") + + return nil, false + } + + var entry manifest.Entry + err = json.Unmarshal(jb.Bytes(), &entry) + if err != nil { + log.WithError(err).WithField("layer", key). + Error("failed to unmarshal cached layer") + + return nil, false + } + + go s.Cache.localCacheLayer(key, entry) + return &entry, true +} + +func cacheLayer(ctx context.Context, s *State, key string, entry manifest.Entry) { + s.Cache.localCacheLayer(key, entry) + + j, _ := json.Marshal(&entry) + path := "builds/" + key + _, _, err := s.Storage.Persist(ctx, path, func(w io.Writer) (string, int64, error) { + size, err := io.Copy(w, bytes.NewReader(j)) + return "", size, err + }) + + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "layer": key, + "backend": s.Storage.Name(), + }).Error("failed to cache layer") + } + + return +} diff --git a/tools/nixery/builder/layers.go b/tools/nixery/builder/layers.go new file mode 100644 index 000000000000..f769e43c5808 --- /dev/null +++ b/tools/nixery/builder/layers.go @@ -0,0 +1,364 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +// This package reads an export reference graph (i.e. a graph representing the +// runtime dependencies of a set of derivations) created by Nix and groups it in +// a way that is likely to match the grouping for other derivation sets with +// overlapping dependencies. +// +// This is used to determine which derivations to include in which layers of a +// container image. +// +// # Inputs +// +// * a graph of Nix runtime dependencies, generated via exportReferenceGraph +// * popularity values of each package in the Nix package set (in the form of a +// direct reference count) +// * a maximum number of layers to allocate for the image (the "layer budget") +// +// # Algorithm +// +// It works by first creating a (directed) dependency tree: +// +// img (root node) +// │ +// ├───> A ─────┐ +// │ v +// ├───> B ───> E +// │ ^ +// ├───> C ─────┘ +// │ │ +// │ v +// └───> D ───> F +// │ +// └────> G +// +// Each node (i.e. package) is then visited to determine how important +// it is to separate this node into its own layer, specifically: +// +// 1. Is the node within a certain threshold percentile of absolute +// popularity within all of nixpkgs? (e.g. `glibc`, `openssl`) +// +// 2. Is the node's runtime closure above a threshold size? (e.g. 100MB) +// +// In either case, a bit is flipped for this node representing each +// condition and an edge to it is inserted directly from the image +// root, if it does not already exist. +// +// For the rest of the example we assume 'G' is above the threshold +// size and 'E' is popular. +// +// This tree is then transformed into a dominator tree: +// +// img +// │ +// ├───> A +// ├───> B +// ├───> C +// ├───> E +// ├───> D ───> F +// └───> G +// +// Specifically this means that the paths to A, B, C, E, G, and D +// always pass through the root (i.e. are dominated by it), whilst F +// is dominated by D (all paths go through it). +// +// The top-level subtrees are considered as the initially selected +// layers. +// +// If the list of layers fits within the layer budget, it is returned. +// +// Otherwise, a merge rating is calculated for each layer. This is the +// product of the layer's total size and its root node's popularity. +// +// Layers are then merged in ascending order of merge ratings until +// they fit into the layer budget. +// +// # Threshold values +// +// Threshold values for the partitioning conditions mentioned above +// have not yet been determined, but we will make a good first guess +// based on gut feeling and proceed to measure their impact on cache +// hits/misses. +// +// # Example +// +// Using the logic described above as well as the example presented in +// the introduction, this program would create the following layer +// groupings (assuming no additional partitioning): +// +// Layer budget: 1 +// Layers: { A, B, C, D, E, F, G } +// +// Layer budget: 2 +// Layers: { G }, { A, B, C, D, E, F } +// +// Layer budget: 3 +// Layers: { G }, { E }, { A, B, C, D, F } +// +// Layer budget: 4 +// Layers: { G }, { E }, { D, F }, { A, B, C } +// +// ... +// +// Layer budget: 10 +// Layers: { E }, { D, F }, { A }, { B }, { C } +package builder + +import ( + "crypto/sha1" + "fmt" + "regexp" + "sort" + "strings" + + log "github.com/sirupsen/logrus" + "gonum.org/v1/gonum/graph/flow" + "gonum.org/v1/gonum/graph/simple" +) + +// runtimeGraph represents structured information from Nix about the runtime +// dependencies of a derivation. +// +// This is generated in Nix by using the exportReferencesGraph feature. +type runtimeGraph struct { + References struct { + Graph []string `json:"graph"` + } `json:"exportReferencesGraph"` + + Graph []struct { + Size uint64 `json:"closureSize"` + Path string `json:"path"` + Refs []string `json:"references"` + } `json:"graph"` +} + +// Popularity data for each Nix package that was calculated in advance. +// +// Popularity is a number from 1-100 that represents the +// popularity percentile in which this package resides inside +// of the nixpkgs tree. +type Popularity = map[string]int + +// Layer represents the data returned for each layer that Nix should +// build for the container image. +type layer struct { + Contents []string `json:"contents"` + MergeRating uint64 +} + +// Hash the contents of a layer to create a deterministic identifier that can be +// used for caching. +func (l *layer) Hash() string { + sum := sha1.Sum([]byte(strings.Join(l.Contents, ":"))) + return fmt.Sprintf("%x", sum) +} + +func (a layer) merge(b layer) layer { + a.Contents = append(a.Contents, b.Contents...) + a.MergeRating += b.MergeRating + return a +} + +// closure as pointed to by the graph nodes. +type closure struct { + GraphID int64 + Path string + Size uint64 + Refs []string + Popularity int +} + +func (c *closure) ID() int64 { + return c.GraphID +} + +var nixRegexp = regexp.MustCompile(`^/nix/store/[a-z0-9]+-`) + +// PackageFromPath returns the name of a Nix package based on its +// output store path. +func packageFromPath(path string) string { + return nixRegexp.ReplaceAllString(path, "") +} + +// DOTID provides a human-readable package name. The name stems from +// the dot format used by GraphViz, into which the dependency graph +// can be rendered. +func (c *closure) DOTID() string { + return packageFromPath(c.Path) +} + +// bigOrPopular checks whether this closure should be considered for +// separation into its own layer, even if it would otherwise only +// appear in a subtree of the dominator tree. +func (c *closure) bigOrPopular() bool { + const sizeThreshold = 100 * 1000000 // 100MB + + if c.Size > sizeThreshold { + return true + } + + // Threshold value is picked arbitrarily right now. The reason + // for this is that some packages (such as `cacert`) have very + // few direct dependencies, but are required by pretty much + // everything. + if c.Popularity >= 100 { + return true + } + + return false +} + +func insertEdges(graph *simple.DirectedGraph, cmap *map[string]*closure, node *closure) { + // Big or popular nodes get a separate edge from the top to + // flag them for their own layer. + if node.bigOrPopular() && !graph.HasEdgeFromTo(0, node.ID()) { + edge := graph.NewEdge(graph.Node(0), node) + graph.SetEdge(edge) + } + + for _, c := range node.Refs { + // Nix adds a self reference to each node, which + // should not be inserted. + if c != node.Path { + edge := graph.NewEdge(node, (*cmap)[c]) + graph.SetEdge(edge) + } + } +} + +// Create a graph structure from the references supplied by Nix. +func buildGraph(refs *runtimeGraph, pop *Popularity) *simple.DirectedGraph { + cmap := make(map[string]*closure) + graph := simple.NewDirectedGraph() + + // Insert all closures into the graph, as well as a fake root + // closure which serves as the top of the tree. + // + // A map from store paths to IDs is kept to actually insert + // edges below. + root := &closure{ + GraphID: 0, + Path: "image_root", + } + graph.AddNode(root) + + for idx, c := range refs.Graph { + node := &closure{ + GraphID: int64(idx + 1), // inc because of root node + Path: c.Path, + Size: c.Size, + Refs: c.Refs, + } + + // The packages `nss-cacert` and `iana-etc` are added + // by Nixery to *every single image* and should have a + // very high popularity. + // + // Other popularity values are populated from the data + // set assembled by Nixery's popcount. + id := node.DOTID() + if strings.HasPrefix(id, "nss-cacert") || strings.HasPrefix(id, "iana-etc") { + // glibc has ~300k references, these packages need *more* + node.Popularity = 500000 + } else if p, ok := (*pop)[id]; ok { + node.Popularity = p + } else { + node.Popularity = 1 + } + + graph.AddNode(node) + cmap[c.Path] = node + } + + // Insert the top-level closures with edges from the root + // node, then insert all edges for each closure. + for _, p := range refs.References.Graph { + edge := graph.NewEdge(root, cmap[p]) + graph.SetEdge(edge) + } + + for _, c := range cmap { + insertEdges(graph, &cmap, c) + } + + return graph +} + +// Extracts a subgraph starting at the specified root from the +// dominator tree. The subgraph is converted into a flat list of +// layers, each containing the store paths and merge rating. +func groupLayer(dt *flow.DominatorTree, root *closure) layer { + size := root.Size + contents := []string{root.Path} + children := dt.DominatedBy(root.ID()) + + // This iteration does not use 'range' because the list being + // iterated is modified during the iteration (yes, I'm sorry). + for i := 0; i < len(children); i++ { + child := children[i].(*closure) + size += child.Size + contents = append(contents, child.Path) + children = append(children, dt.DominatedBy(child.ID())...) + } + + // Contents are sorted to ensure that hashing is consistent + sort.Strings(contents) + + return layer{ + Contents: contents, + MergeRating: uint64(root.Popularity) * size, + } +} + +// Calculate the dominator tree of the entire package set and group +// each top-level subtree into a layer. +// +// Layers are merged together until they fit into the layer budget, +// based on their merge rating. +func dominate(budget int, graph *simple.DirectedGraph) []layer { + dt := flow.Dominators(graph.Node(0), graph) + + var layers []layer + for _, n := range dt.DominatedBy(dt.Root().ID()) { + layers = append(layers, groupLayer(&dt, n.(*closure))) + } + + sort.Slice(layers, func(i, j int) bool { + return layers[i].MergeRating < layers[j].MergeRating + }) + + if len(layers) > budget { + log.WithFields(log.Fields{ + "layers": len(layers), + "budget": budget, + }).Info("ideal image exceeds layer budget") + } + + for len(layers) > budget { + merged := layers[0].merge(layers[1]) + layers[1] = merged + layers = layers[1:] + } + + return layers +} + +// groupLayers applies the algorithm described above the its input and returns a +// list of layers, each consisting of a list of Nix store paths that it should +// contain. +func groupLayers(refs *runtimeGraph, pop *Popularity, budget int) []layer { + graph := buildGraph(refs, pop) + return dominate(budget, graph) +} diff --git a/tools/nixery/config/config.go b/tools/nixery/config/config.go new file mode 100644 index 000000000000..7ec102bd6cee --- /dev/null +++ b/tools/nixery/config/config.go @@ -0,0 +1,84 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +// Package config implements structures to store Nixery's configuration at +// runtime as well as the logic for instantiating this configuration from the +// environment. +package config + +import ( + "os" + + log "github.com/sirupsen/logrus" +) + +func getConfig(key, desc, def string) string { + value := os.Getenv(key) + if value == "" && def == "" { + log.WithFields(log.Fields{ + "option": key, + "description": desc, + }).Fatal("missing required configuration envvar") + } else if value == "" { + return def + } + + return value +} + +// Backend represents the possible storage backend types +type Backend int + +const ( + GCS = iota + FileSystem +) + +// Config holds the Nixery configuration options. +type Config struct { + Port string // Port on which to launch HTTP server + Pkgs PkgSource // Source for Nix package set + Timeout string // Timeout for a single Nix builder (seconds) + WebDir string // Directory with static web assets + PopUrl string // URL to the Nix package popularity count + Backend Backend // Storage backend to use for Nixery +} + +func FromEnv() (Config, error) { + pkgs, err := pkgSourceFromEnv() + if err != nil { + return Config{}, err + } + + var b Backend + switch os.Getenv("NIXERY_STORAGE_BACKEND") { + case "gcs": + b = GCS + case "filesystem": + b = FileSystem + default: + log.WithField("values", []string{ + "gcs", + }).Fatal("NIXERY_STORAGE_BUCKET must be set to a supported value") + } + + return Config{ + Port: getConfig("PORT", "HTTP port", ""), + Pkgs: pkgs, + Timeout: getConfig("NIX_TIMEOUT", "Nix builder timeout", "60"), + WebDir: getConfig("WEB_DIR", "Static web file dir", ""), + PopUrl: os.Getenv("NIX_POPULARITY_URL"), + Backend: b, + }, nil +} diff --git a/tools/nixery/config/pkgsource.go b/tools/nixery/config/pkgsource.go new file mode 100644 index 000000000000..95236c4b0d15 --- /dev/null +++ b/tools/nixery/config/pkgsource.go @@ -0,0 +1,159 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +package config + +import ( + "crypto/sha1" + "encoding/json" + "fmt" + "os" + "regexp" + "strings" + + log "github.com/sirupsen/logrus" +) + +// PkgSource represents the source from which the Nix package set used +// by Nixery is imported. Users configure the source by setting one of +// the supported environment variables. +type PkgSource interface { + // Convert the package source into the representation required + // for calling Nix. + Render(tag string) (string, string) + + // Create a key by which builds for this source and iamge + // combination can be cached. + // + // The empty string means that this value is not cacheable due + // to the package source being a moving target (such as a + // channel). + CacheKey(pkgs []string, tag string) string +} + +type GitSource struct { + repository string +} + +// Regex to determine whether a git reference is a commit hash or +// something else (branch/tag). +// +// Used to check whether a git reference is cacheable, and to pass the +// correct git structure to Nix. +// +// Note: If a user creates a branch or tag with the name of a commit +// and references it intentionally, this heuristic will fail. +var commitRegex = regexp.MustCompile(`^[0-9a-f]{40}$`) + +func (g *GitSource) Render(tag string) (string, string) { + args := map[string]string{ + "url": g.repository, + } + + // The 'git' source requires a tag to be present. If the user + // has not specified one, it is assumed that the default + // 'master' branch should be used. + if tag == "latest" || tag == "" { + tag = "master" + } + + if commitRegex.MatchString(tag) { + args["rev"] = tag + } else { + args["ref"] = tag + } + + j, _ := json.Marshal(args) + + return "git", string(j) +} + +func (g *GitSource) CacheKey(pkgs []string, tag string) string { + // Only full commit hashes can be used for caching, as + // everything else is potentially a moving target. + if !commitRegex.MatchString(tag) { + return "" + } + + unhashed := strings.Join(pkgs, "") + tag + hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed))) + + return hashed +} + +type NixChannel struct { + channel string +} + +func (n *NixChannel) Render(tag string) (string, string) { + return "nixpkgs", n.channel +} + +func (n *NixChannel) CacheKey(pkgs []string, tag string) string { + // Since Nix channels are downloaded from the nixpkgs-channels + // Github, users can specify full commit hashes as the + // "channel", in which case builds are cacheable. + if !commitRegex.MatchString(n.channel) { + return "" + } + + unhashed := strings.Join(pkgs, "") + n.channel + hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed))) + + return hashed +} + +type PkgsPath struct { + path string +} + +func (p *PkgsPath) Render(tag string) (string, string) { + return "path", p.path +} + +func (p *PkgsPath) CacheKey(pkgs []string, tag string) string { + // Path-based builds are not currently cacheable because we + // have no local hash of the package folder's state easily + // available. + return "" +} + +// Retrieve a package source from the environment. If no source is +// specified, the Nix code will default to a recent NixOS channel. +func pkgSourceFromEnv() (PkgSource, error) { + if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" { + log.WithField("channel", channel).Info("using Nix package set from Nix channel or commit") + + return &NixChannel{ + channel: channel, + }, nil + } + + if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" { + log.WithField("repo", git).Info("using NIx package set from git repository") + + return &GitSource{ + repository: git, + }, nil + } + + if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" { + log.WithField("path", path).Info("using Nix package set at local path") + + return &PkgsPath{ + path: path, + }, nil + } + + return nil, fmt.Errorf("no valid package source has been specified") +} diff --git a/tools/nixery/default.nix b/tools/nixery/default.nix index 44ac7313adc7..7454c14a8567 100644 --- a/tools/nixery/default.nix +++ b/tools/nixery/default.nix @@ -20,6 +20,8 @@ with pkgs; let + inherit (pkgs) buildGoPackage; + # Hash of all Nixery sources - this is used as the Nixery version in # builds to distinguish errors between deployed versions, see # server/logs.go for details. @@ -30,13 +32,41 @@ let # Go implementation of the Nixery server which implements the # container registry interface. # - # Users should use the nixery-bin derivation below instead. - nixery-server = callPackage ./server { - srcHash = nixery-src-hash; + # Users should use the nixery-bin derivation below instead as it + # provides the paths of files needed at runtime. + nixery-server = buildGoPackage rec { + name = "nixery-server"; + goDeps = ./go-deps.nix; + src = ./.; + + goPackagePath = "github.com/google/nixery"; + doCheck = true; + + # Simplify the Nix build instructions for Go to just the basics + # required to get Nixery up and running with the additional linker + # flags required. + outputs = [ "out" ]; + preConfigure = "bin=$out"; + buildPhase = '' + runHook preBuild + runHook renameImport + + export GOBIN="$out/bin" + go install -ldflags "-X main.version=$(cat ${nixery-src-hash})" ${goPackagePath} + ''; + + fixupPhase = '' + remove-references-to -t ${go} $out/bin/nixery + ''; + + checkPhase = '' + go vet ${goPackagePath} + go test ${goPackagePath} + ''; }; in rec { # Implementation of the Nix image building logic - nixery-build-image = import ./build-image { inherit pkgs; }; + nixery-prepare-image = import ./prepare-image { inherit pkgs; }; # Use mdBook to build a static asset page which Nixery can then # serve. This is primarily used for the public instance at @@ -50,8 +80,8 @@ in rec { # are installing Nixery directly. nixery-bin = writeShellScriptBin "nixery" '' export WEB_DIR="${nixery-book}" - export PATH="${nixery-build-image}/bin:$PATH" - exec ${nixery-server}/bin/server + export PATH="${nixery-prepare-image}/bin:$PATH" + exec ${nixery-server}/bin/nixery ''; nixery-popcount = callPackage ./popcount { }; @@ -104,7 +134,7 @@ in rec { gzip iana-etc nix - nixery-build-image + nixery-prepare-image nixery-launch-script openssh zlib diff --git a/tools/nixery/go-deps.nix b/tools/nixery/go-deps.nix new file mode 100644 index 000000000000..847b44dce63c --- /dev/null +++ b/tools/nixery/go-deps.nix @@ -0,0 +1,129 @@ +# This file was generated by https://github.com/kamilchm/go2nix v1.3.0 +[ + { + goPackagePath = "cloud.google.com/go"; + fetch = { + type = "git"; + url = "https://code.googlesource.com/gocloud"; + rev = "77f6a3a292a7dbf66a5329de0d06326f1906b450"; + sha256 = "1c9pkx782nbcp8jnl5lprcbzf97van789ky5qsncjgywjyymhigi"; + }; + } + { + goPackagePath = "github.com/golang/protobuf"; + fetch = { + type = "git"; + url = "https://github.com/golang/protobuf"; + rev = "6c65a5562fc06764971b7c5d05c76c75e84bdbf7"; + sha256 = "1k1wb4zr0qbwgpvz9q5ws9zhlal8hq7dmq62pwxxriksayl6hzym"; + }; + } + { + goPackagePath = "github.com/googleapis/gax-go"; + fetch = { + type = "git"; + url = "https://github.com/googleapis/gax-go"; + rev = "bd5b16380fd03dc758d11cef74ba2e3bc8b0e8c2"; + sha256 = "1lxawwngv6miaqd25s3ba0didfzylbwisd2nz7r4gmbmin6jsjrx"; + }; + } + { + goPackagePath = "github.com/hashicorp/golang-lru"; + fetch = { + type = "git"; + url = "https://github.com/hashicorp/golang-lru"; + rev = "59383c442f7d7b190497e9bb8fc17a48d06cd03f"; + sha256 = "0yzwl592aa32vfy73pl7wdc21855w17zssrp85ckw2nisky8rg9c"; + }; + } + { + goPackagePath = "go.opencensus.io"; + fetch = { + type = "git"; + url = "https://github.com/census-instrumentation/opencensus-go"; + rev = "b4a14686f0a98096416fe1b4cb848e384fb2b22b"; + sha256 = "1aidyp301v5ngwsnnc8v1s09vvbsnch1jc4vd615f7qv77r9s7dn"; + }; + } + { + goPackagePath = "golang.org/x/net"; + fetch = { + type = "git"; + url = "https://go.googlesource.com/net"; + rev = "da137c7871d730100384dbcf36e6f8fa493aef5b"; + sha256 = "1qsiyr3irmb6ii06hivm9p2c7wqyxczms1a9v1ss5698yjr3fg47"; + }; + } + { + goPackagePath = "golang.org/x/oauth2"; + fetch = { + type = "git"; + url = "https://go.googlesource.com/oauth2"; + rev = "0f29369cfe4552d0e4bcddc57cc75f4d7e672a33"; + sha256 = "06jwpvx0x2gjn2y959drbcir5kd7vg87k0r1216abk6rrdzzrzi2"; + }; + } + { + goPackagePath = "golang.org/x/sys"; + fetch = { + type = "git"; + url = "https://go.googlesource.com/sys"; + rev = "51ab0e2deafac1f46c46ad59cf0921be2f180c3d"; + sha256 = "0xdhpckbql3bsqkpc2k5b1cpnq3q1qjqjjq2j3p707rfwb8nm91a"; + }; + } + { + goPackagePath = "golang.org/x/text"; + fetch = { + type = "git"; + url = "https://go.googlesource.com/text"; + rev = "342b2e1fbaa52c93f31447ad2c6abc048c63e475"; + sha256 = "0flv9idw0jm5nm8lx25xqanbkqgfiym6619w575p7nrdh0riqwqh"; + }; + } + { + goPackagePath = "google.golang.org/api"; + fetch = { + type = "git"; + url = "https://code.googlesource.com/google-api-go-client"; + rev = "069bea57b1be6ad0671a49ea7a1128025a22b73f"; + sha256 = "19q2b610lkf3z3y9hn6rf11dd78xr9q4340mdyri7kbijlj2r44q"; + }; + } + { + goPackagePath = "google.golang.org/genproto"; + fetch = { + type = "git"; + url = "https://github.com/google/go-genproto"; + rev = "c506a9f9061087022822e8da603a52fc387115a8"; + sha256 = "03hh80aqi58dqi5ykj4shk3chwkzrgq2f3k6qs5qhgvmcy79y2py"; + }; + } + { + goPackagePath = "google.golang.org/grpc"; + fetch = { + type = "git"; + url = "https://github.com/grpc/grpc-go"; + rev = "977142214c45640483838b8672a43c46f89f90cb"; + sha256 = "05wig23l2sil3bfdv19gq62sya7hsabqj9l8pzr1sm57qsvj218d"; + }; + } + { + goPackagePath = "gonum.org/v1/gonum"; + fetch = { + type = "git"; + url = "https://github.com/gonum/gonum"; + rev = "ced62fe5104b907b6c16cb7e575c17b2e62ceddd"; + sha256 = "1b7q6haabnp53igpmvr6a2414yralhbrldixx4kbxxg1apy8jdjg"; + }; + } + { + goPackagePath = "github.com/sirupsen/logrus"; + fetch = { + type = "git"; + url = "https://github.com/sirupsen/logrus"; + rev = "de736cf91b921d56253b4010270681d33fdf7cb5"; + sha256 = "1qixss8m5xy7pzbf0qz2k3shjw0asklm9sj6zyczp7mryrari0aj"; + }; + } +] diff --git a/tools/nixery/logs/logs.go b/tools/nixery/logs/logs.go new file mode 100644 index 000000000000..4c755bc8ab0c --- /dev/null +++ b/tools/nixery/logs/logs.go @@ -0,0 +1,119 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +package logs + +// This file configures different log formatters via logrus. The +// standard formatter uses a structured JSON format that is compatible +// with Stackdriver Error Reporting. +// +// https://cloud.google.com/error-reporting/docs/formatting-error-messages + +import ( + "bytes" + "encoding/json" + log "github.com/sirupsen/logrus" +) + +type stackdriverFormatter struct{} + +type serviceContext struct { + Service string `json:"service"` + Version string `json:"version"` +} + +type reportLocation struct { + FilePath string `json:"filePath"` + LineNumber int `json:"lineNumber"` + FunctionName string `json:"functionName"` +} + +var nixeryContext = serviceContext{ + Service: "nixery", +} + +// isError determines whether an entry should be logged as an error +// (i.e. with attached `context`). +// +// This requires the caller information to be present on the log +// entry, as stacktraces are not available currently. +func isError(e *log.Entry) bool { + l := e.Level + return (l == log.ErrorLevel || l == log.FatalLevel || l == log.PanicLevel) && + e.HasCaller() +} + +// logSeverity formats the entry's severity into a format compatible +// with Stackdriver Logging. +// +// The two formats that are being mapped do not have an equivalent set +// of severities/levels, so the mapping is somewhat arbitrary for a +// handful of them. +// +// https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#LogSeverity +func logSeverity(l log.Level) string { + switch l { + case log.TraceLevel: + return "DEBUG" + case log.DebugLevel: + return "DEBUG" + case log.InfoLevel: + return "INFO" + case log.WarnLevel: + return "WARNING" + case log.ErrorLevel: + return "ERROR" + case log.FatalLevel: + return "CRITICAL" + case log.PanicLevel: + return "EMERGENCY" + default: + return "DEFAULT" + } +} + +func (f stackdriverFormatter) Format(e *log.Entry) ([]byte, error) { + msg := e.Data + msg["serviceContext"] = &nixeryContext + msg["message"] = &e.Message + msg["eventTime"] = &e.Time + msg["severity"] = logSeverity(e.Level) + + if e, ok := msg[log.ErrorKey]; ok { + if err, isError := e.(error); isError { + msg[log.ErrorKey] = err.Error() + } else { + delete(msg, log.ErrorKey) + } + } + + if isError(e) { + loc := reportLocation{ + FilePath: e.Caller.File, + LineNumber: e.Caller.Line, + FunctionName: e.Caller.Function, + } + msg["context"] = &loc + } + + b := new(bytes.Buffer) + err := json.NewEncoder(b).Encode(&msg) + + return b.Bytes(), err +} + +func Init(version string) { + nixeryContext.Version = version + log.SetReportCaller(true) + log.SetFormatter(stackdriverFormatter{}) +} diff --git a/tools/nixery/main.go b/tools/nixery/main.go new file mode 100644 index 000000000000..6cad93740978 --- /dev/null +++ b/tools/nixery/main.go @@ -0,0 +1,249 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +// The nixery server implements a container registry that transparently builds +// container images based on Nix derivations. +// +// The Nix derivation used for image creation is responsible for creating +// objects that are compatible with the registry API. The targeted registry +// protocol is currently Docker's. +// +// When an image is requested, the required contents are parsed out of the +// request and a Nix-build is initiated that eventually responds with the +// manifest as well as information linking each layer digest to a local +// filesystem path. +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "regexp" + + "github.com/google/nixery/builder" + "github.com/google/nixery/config" + "github.com/google/nixery/logs" + "github.com/google/nixery/storage" + log "github.com/sirupsen/logrus" +) + +// ManifestMediaType is the Content-Type used for the manifest itself. This +// corresponds to the "Image Manifest V2, Schema 2" described on this page: +// +// https://docs.docker.com/registry/spec/manifest-v2-2/ +const manifestMediaType string = "application/vnd.docker.distribution.manifest.v2+json" + +// This variable will be initialised during the build process and set +// to the hash of the entire Nixery source tree. +var version string = "devel" + +// Regexes matching the V2 Registry API routes. This only includes the +// routes required for serving images, since pushing and other such +// functionality is not available. +var ( + manifestRegex = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/manifests/([\w|\-|\.|\_]+)$`) + layerRegex = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/blobs/sha256:(\w+)$`) +) + +// Downloads the popularity information for the package set from the +// URL specified in Nixery's configuration. +func downloadPopularity(url string) (builder.Popularity, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("popularity download from '%s' returned status: %s\n", url, resp.Status) + } + + j, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var pop builder.Popularity + err = json.Unmarshal(j, &pop) + if err != nil { + return nil, err + } + + return pop, nil +} + +// Error format corresponding to the registry protocol V2 specification. This +// allows feeding back errors to clients in a way that can be presented to +// users. +type registryError struct { + Code string `json:"code"` + Message string `json:"message"` +} + +type registryErrors struct { + Errors []registryError `json:"errors"` +} + +func writeError(w http.ResponseWriter, status int, code, message string) { + err := registryErrors{ + Errors: []registryError{ + {code, message}, + }, + } + json, _ := json.Marshal(err) + + w.WriteHeader(status) + w.Header().Add("Content-Type", "application/json") + w.Write(json) +} + +type registryHandler struct { + state *builder.State +} + +func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Acknowledge that we speak V2 with an empty response + if r.RequestURI == "/v2/" { + return + } + + // Serve the manifest (straight from Nix) + manifestMatches := manifestRegex.FindStringSubmatch(r.RequestURI) + if len(manifestMatches) == 3 { + imageName := manifestMatches[1] + imageTag := manifestMatches[2] + + log.WithFields(log.Fields{ + "image": imageName, + "tag": imageTag, + }).Info("requesting image manifest") + + image := builder.ImageFromName(imageName, imageTag) + buildResult, err := builder.BuildImage(r.Context(), h.state, &image) + + if err != nil { + writeError(w, 500, "UNKNOWN", "image build failure") + + log.WithError(err).WithFields(log.Fields{ + "image": imageName, + "tag": imageTag, + }).Error("failed to build image manifest") + + return + } + + // Some error types have special handling, which is applied + // here. + if buildResult.Error == "not_found" { + s := fmt.Sprintf("Could not find Nix packages: %v", buildResult.Pkgs) + writeError(w, 404, "MANIFEST_UNKNOWN", s) + + log.WithFields(log.Fields{ + "image": imageName, + "tag": imageTag, + "packages": buildResult.Pkgs, + }).Warn("could not find Nix packages") + + return + } + + // This marshaling error is ignored because we know that this + // field represents valid JSON data. + manifest, _ := json.Marshal(buildResult.Manifest) + w.Header().Add("Content-Type", manifestMediaType) + w.Write(manifest) + return + } + + // Serve an image layer. For this we need to first ask Nix for + // the manifest, then proceed to extract the correct layer from + // it. + layerMatches := layerRegex.FindStringSubmatch(r.RequestURI) + if len(layerMatches) == 3 { + digest := layerMatches[2] + storage := h.state.Storage + err := storage.ServeLayer(digest, r, w) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "layer": digest, + "backend": storage.Name(), + }).Error("failed to serve layer from storage backend") + } + + return + } + + log.WithField("uri", r.RequestURI).Info("unsupported registry route") + + w.WriteHeader(404) +} + +func main() { + logs.Init(version) + cfg, err := config.FromEnv() + if err != nil { + log.WithError(err).Fatal("failed to load configuration") + } + + var s storage.Backend + + switch cfg.Backend { + case config.GCS: + s, err = storage.NewGCSBackend() + case config.FileSystem: + s, err = storage.NewFSBackend() + } + if err != nil { + log.WithError(err).Fatal("failed to initialise storage backend") + } + + log.WithField("backend", s.Name()).Info("initialised storage backend") + + cache, err := builder.NewCache() + if err != nil { + log.WithError(err).Fatal("failed to instantiate build cache") + } + + var pop builder.Popularity + if cfg.PopUrl != "" { + pop, err = downloadPopularity(cfg.PopUrl) + if err != nil { + log.WithError(err).WithField("popURL", cfg.PopUrl). + Fatal("failed to fetch popularity information") + } + } + + state := builder.State{ + Cache: &cache, + Cfg: cfg, + Pop: pop, + Storage: s, + } + + log.WithFields(log.Fields{ + "version": version, + "port": cfg.Port, + }).Info("starting Nixery") + + // All /v2/ requests belong to the registry handler. + http.Handle("/v2/", ®istryHandler{ + state: &state, + }) + + // All other roots are served by the static file server. + webDir := http.Dir(cfg.WebDir) + http.Handle("/", http.FileServer(webDir)) + + log.Fatal(http.ListenAndServe(":"+cfg.Port, nil)) +} diff --git a/tools/nixery/manifest/manifest.go b/tools/nixery/manifest/manifest.go new file mode 100644 index 000000000000..0d36826fb7e5 --- /dev/null +++ b/tools/nixery/manifest/manifest.go @@ -0,0 +1,141 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +// Package image implements logic for creating the image metadata +// (such as the image manifest and configuration). +package manifest + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "sort" +) + +const ( + // manifest constants + schemaVersion = 2 + + // media types + manifestType = "application/vnd.docker.distribution.manifest.v2+json" + layerType = "application/vnd.docker.image.rootfs.diff.tar.gzip" + configType = "application/vnd.docker.container.image.v1+json" + + // image config constants + os = "linux" + fsType = "layers" +) + +type Entry struct { + MediaType string `json:"mediaType,omitempty"` + Size int64 `json:"size"` + Digest string `json:"digest"` + + // These fields are internal to Nixery and not part of the + // serialised entry. + MergeRating uint64 `json:"-"` + TarHash string `json:",omitempty"` +} + +type manifest struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Config Entry `json:"config"` + Layers []Entry `json:"layers"` +} + +type imageConfig struct { + Architecture string `json:"architecture"` + OS string `json:"os"` + + RootFS struct { + FSType string `json:"type"` + DiffIDs []string `json:"diff_ids"` + } `json:"rootfs"` + + // sic! empty struct (rather than `null`) is required by the + // image metadata deserialiser in Kubernetes + Config struct{} `json:"config"` +} + +// ConfigLayer represents the configuration layer to be included in +// the manifest, containing its JSON-serialised content and SHA256 +// hash. +type ConfigLayer struct { + Config []byte + SHA256 string +} + +// imageConfig creates an image configuration with the values set to +// the constant defaults. +// +// Outside of this module the image configuration is treated as an +// opaque blob and it is thus returned as an already serialised byte +// array and its SHA256-hash. +func configLayer(arch string, hashes []string) ConfigLayer { + c := imageConfig{} + c.Architecture = arch + c.OS = os + c.RootFS.FSType = fsType + c.RootFS.DiffIDs = hashes + + j, _ := json.Marshal(c) + + return ConfigLayer{ + Config: j, + SHA256: fmt.Sprintf("%x", sha256.Sum256(j)), + } +} + +// Manifest creates an image manifest from the specified layer entries +// and returns its JSON-serialised form as well as the configuration +// layer. +// +// Callers do not need to set the media type for the layer entries. +func Manifest(arch string, layers []Entry) (json.RawMessage, ConfigLayer) { + // Sort layers by their merge rating, from highest to lowest. + // This makes it likely for a contiguous chain of shared image + // layers to appear at the beginning of a layer. + // + // Due to moby/moby#38446 Docker considers the order of layers + // when deciding which layers to download again. + sort.Slice(layers, func(i, j int) bool { + return layers[i].MergeRating > layers[j].MergeRating + }) + + hashes := make([]string, len(layers)) + for i, l := range layers { + hashes[i] = l.TarHash + l.MediaType = layerType + l.TarHash = "" + layers[i] = l + } + + c := configLayer(arch, hashes) + + m := manifest{ + SchemaVersion: schemaVersion, + MediaType: manifestType, + Config: Entry{ + MediaType: configType, + Size: int64(len(c.Config)), + Digest: "sha256:" + c.SHA256, + }, + Layers: layers, + } + + j, _ := json.Marshal(m) + + return json.RawMessage(j), c +} diff --git a/tools/nixery/popcount/popcount.go b/tools/nixery/popcount/popcount.go index b21cee2e0e7d..992a88e874d1 100644 --- a/tools/nixery/popcount/popcount.go +++ b/tools/nixery/popcount/popcount.go @@ -175,7 +175,7 @@ func fetchNarInfo(i *item) (string, error) { narinfo, err := ioutil.ReadAll(resp.Body) // best-effort write the file to the cache - ioutil.WriteFile("popcache/" + i.hash, narinfo, 0644) + ioutil.WriteFile("popcache/"+i.hash, narinfo, 0644) return string(narinfo), err } diff --git a/tools/nixery/prepare-image/default.nix b/tools/nixery/prepare-image/default.nix new file mode 100644 index 000000000000..60b208f522d5 --- /dev/null +++ b/tools/nixery/prepare-image/default.nix @@ -0,0 +1,29 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file builds a wrapper script called by Nixery to ask for the +# content information for a given image. +# +# The purpose of using a wrapper script is to ensure that the paths to +# all required Nix files are set correctly at runtime. + +{ pkgs ? import {} }: + +pkgs.writeShellScriptBin "nixery-prepare-image" '' + exec ${pkgs.nix}/bin/nix-build \ + --show-trace \ + --no-out-link "$@" \ + --argstr loadPkgs ${./load-pkgs.nix} \ + ${./prepare-image.nix} +'' diff --git a/tools/nixery/prepare-image/load-pkgs.nix b/tools/nixery/prepare-image/load-pkgs.nix new file mode 100644 index 000000000000..cceebfc14dae --- /dev/null +++ b/tools/nixery/prepare-image/load-pkgs.nix @@ -0,0 +1,45 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Load a Nix package set from one of the supported source types +# (nixpkgs, git, path). +{ srcType, srcArgs, importArgs ? { } }: + +with builtins; +let + # If a nixpkgs channel is requested, it is retrieved from Github (as + # a tarball) and imported. + fetchImportChannel = channel: + let + url = + "https://github.com/NixOS/nixpkgs-channels/archive/${channel}.tar.gz"; + in import (fetchTarball url) importArgs; + + # If a git repository is requested, it is retrieved via + # builtins.fetchGit which defaults to the git configuration of the + # outside environment. This means that user-configured SSH + # credentials etc. are going to work as expected. + fetchImportGit = spec: import (fetchGit spec) importArgs; + + # No special handling is used for paths, so users are expected to pass one + # that will work natively with Nix. + importPath = path: import (toPath path) importArgs; +in if srcType == "nixpkgs" then + fetchImportChannel srcArgs +else if srcType == "git" then + fetchImportGit (fromJSON srcArgs) +else if srcType == "path" then + importPath srcArgs +else + throw ("Invalid package set source specification: ${srcType} (${srcArgs})") diff --git a/tools/nixery/prepare-image/prepare-image.nix b/tools/nixery/prepare-image/prepare-image.nix new file mode 100644 index 000000000000..4393f2b859a6 --- /dev/null +++ b/tools/nixery/prepare-image/prepare-image.nix @@ -0,0 +1,173 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file contains a derivation that outputs structured information +# about the runtime dependencies of an image with a given set of +# packages. This is used by Nixery to determine the layer grouping and +# assemble each layer. +# +# In addition it creates and outputs a meta-layer with the symlink +# structure required for using the image together with the individual +# package layers. + +{ + # Description of the package set to be used (will be loaded by load-pkgs.nix) + srcType ? "nixpkgs", + srcArgs ? "nixos-19.03", + system ? "x86_64-linux", + importArgs ? { }, + # Path to load-pkgs.nix + loadPkgs ? ./load-pkgs.nix, + # Packages to install by name (which must refer to top-level attributes of + # nixpkgs). This is passed in as a JSON-array in string form. + packages ? "[]" +}: + +let + inherit (builtins) + foldl' + fromJSON + hasAttr + length + match + readFile + toFile + toJSON; + + # Package set to use for sourcing utilities + nativePkgs = import loadPkgs { inherit srcType srcArgs importArgs; }; + inherit (nativePkgs) coreutils jq openssl lib runCommand writeText symlinkJoin; + + # Package set to use for packages to be included in the image. This + # package set is imported with the system set to the target + # architecture. + pkgs = import loadPkgs { + inherit srcType srcArgs; + importArgs = importArgs // { + inherit system; + }; + }; + + # deepFetch traverses the top-level Nix package set to retrieve an item via a + # path specified in string form. + # + # For top-level items, the name of the key yields the result directly. Nested + # items are fetched by using dot-syntax, as in Nix itself. + # + # Due to a restriction of the registry API specification it is not possible to + # pass uppercase characters in an image name, however the Nix package set + # makes use of camelCasing repeatedly (for example for `haskellPackages`). + # + # To work around this, if no value is found on the top-level a second lookup + # is done on the package set using lowercase-names. This is not done for + # nested sets, as they often have keys that only differ in case. + # + # For example, `deepFetch pkgs "xorg.xev"` retrieves `pkgs.xorg.xev` and + # `deepFetch haskellpackages.stylish-haskell` retrieves + # `haskellPackages.stylish-haskell`. + deepFetch = with lib; s: n: + let path = splitString "." n; + err = { error = "not_found"; pkg = n; }; + # The most efficient way I've found to do a lookup against + # case-differing versions of an attribute is to first construct a + # mapping of all lowercased attribute names to their differently cased + # equivalents. + # + # This map is then used for a second lookup if the top-level + # (case-sensitive) one does not yield a result. + hasUpper = str: (match ".*[A-Z].*" str) != null; + allUpperKeys = filter hasUpper (attrNames s); + lowercased = listToAttrs (map (k: { + name = toLower k; + value = k; + }) allUpperKeys); + caseAmendedPath = map (v: if hasAttr v lowercased then lowercased."${v}" else v) path; + fetchLower = attrByPath caseAmendedPath err s; + in attrByPath path fetchLower s; + + # allContents contains all packages successfully retrieved by name + # from the package set, as well as any errors encountered while + # attempting to fetch a package. + # + # Accumulated error information is returned back to the server. + allContents = + # Folds over the results of 'deepFetch' on all requested packages to + # separate them into errors and content. This allows the program to + # terminate early and return only the errors if any are encountered. + let splitter = attrs: res: + if hasAttr "error" res + then attrs // { errors = attrs.errors ++ [ res ]; } + else attrs // { contents = attrs.contents ++ [ res ]; }; + init = { contents = []; errors = []; }; + fetched = (map (deepFetch pkgs) (fromJSON packages)); + in foldl' splitter init fetched; + + # Contains the export references graph of all retrieved packages, + # which has information about all runtime dependencies of the image. + # + # This is used by Nixery to group closures into image layers. + runtimeGraph = runCommand "runtime-graph.json" { + __structuredAttrs = true; + exportReferencesGraph.graph = allContents.contents; + PATH = "${coreutils}/bin"; + builder = toFile "builder" '' + . .attrs.sh + cp .attrs.json ''${outputs[out]} + ''; + } ""; + + # Create a symlink forest into all top-level store paths of the + # image contents. + contentsEnv = symlinkJoin { + name = "bulk-layers"; + paths = allContents.contents; + }; + + # Image layer that contains the symlink forest created above. This + # must be included in the image to ensure that the filesystem has a + # useful layout at runtime. + symlinkLayer = runCommand "symlink-layer.tar" {} '' + cp -r ${contentsEnv}/ ./layer + tar --transform='s|^\./||' -C layer --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 -cf $out . + ''; + + # Metadata about the symlink layer which is required for serving it. + # Two different hashes are computed for different usages (inclusion + # in manifest vs. content-checking in the layer cache). + symlinkLayerMeta = fromJSON (readFile (runCommand "symlink-layer-meta.json" { + buildInputs = [ coreutils jq openssl ]; + }'' + tarHash=$(sha256sum ${symlinkLayer} | cut -d ' ' -f1) + layerSize=$(stat --printf '%s' ${symlinkLayer}) + + jq -n -c --arg tarHash $tarHash --arg size $layerSize --arg path ${symlinkLayer} \ + '{ size: ($size | tonumber), tarHash: $tarHash, path: $path }' >> $out + '')); + + # Final output structure returned to Nixery if the build succeeded + buildOutput = { + runtimeGraph = fromJSON (readFile runtimeGraph); + symlinkLayer = symlinkLayerMeta; + }; + + # Output structure returned if errors occured during the build. Currently the + # only error type that is returned in a structured way is 'not_found'. + errorOutput = { + error = "not_found"; + pkgs = map (err: err.pkg) allContents.errors; + }; +in writeText "build-output.json" (if (length allContents.errors) == 0 + then toJSON buildOutput + else toJSON errorOutput +) diff --git a/tools/nixery/server/builder/archive.go b/tools/nixery/server/builder/archive.go deleted file mode 100644 index e0fb76d44bee..000000000000 --- a/tools/nixery/server/builder/archive.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. -package builder - -// This file implements logic for walking through a directory and creating a -// tarball of it. -// -// The tarball is written straight to the supplied reader, which makes it -// possible to create an image layer from the specified store paths, hash it and -// upload it in one reading pass. - -import ( - "archive/tar" - "compress/gzip" - "crypto/sha256" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/google/nixery/server/layers" -) - -// Create a new compressed tarball from each of the paths in the list -// and write it to the supplied writer. -// -// The uncompressed tarball is hashed because image manifests must -// contain both the hashes of compressed and uncompressed layers. -func packStorePaths(l *layers.Layer, w io.Writer) (string, error) { - shasum := sha256.New() - gz := gzip.NewWriter(w) - multi := io.MultiWriter(shasum, gz) - t := tar.NewWriter(multi) - - for _, path := range l.Contents { - err := filepath.Walk(path, tarStorePath(t)) - if err != nil { - return "", err - } - } - - if err := t.Close(); err != nil { - return "", err - } - - if err := gz.Close(); err != nil { - return "", err - } - - return fmt.Sprintf("sha256:%x", shasum.Sum([]byte{})), nil -} - -func tarStorePath(w *tar.Writer) filepath.WalkFunc { - return func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - // If the entry is not a symlink or regular file, skip it. - if info.Mode()&os.ModeSymlink == 0 && !info.Mode().IsRegular() { - return nil - } - - // the symlink target is read if this entry is a symlink, as it - // is required when creating the file header - var link string - if info.Mode()&os.ModeSymlink != 0 { - link, err = os.Readlink(path) - if err != nil { - return err - } - } - - header, err := tar.FileInfoHeader(info, link) - if err != nil { - return err - } - - // The name retrieved from os.FileInfo only contains the file's - // basename, but the full path is required within the layer - // tarball. - header.Name = path - if err = w.WriteHeader(header); err != nil { - return err - } - - // At this point, return if no file content needs to be written - if !info.Mode().IsRegular() { - return nil - } - - f, err := os.Open(path) - if err != nil { - return err - } - - if _, err := io.Copy(w, f); err != nil { - return err - } - - f.Close() - - return nil - } -} diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/server/builder/builder.go deleted file mode 100644 index da9dede1acd7..000000000000 --- a/tools/nixery/server/builder/builder.go +++ /dev/null @@ -1,521 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -// Package builder implements the code required to build images via Nix. Image -// build data is cached for up to 24 hours to avoid duplicated calls to Nix -// (which are costly even if no building is performed). -package builder - -import ( - "bufio" - "bytes" - "compress/gzip" - "context" - "crypto/sha256" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "os" - "os/exec" - "sort" - "strings" - - "github.com/google/nixery/server/config" - "github.com/google/nixery/server/layers" - "github.com/google/nixery/server/manifest" - "github.com/google/nixery/server/storage" - log "github.com/sirupsen/logrus" -) - -// The maximum number of layers in an image is 125. To allow for -// extensibility, the actual number of layers Nixery is "allowed" to -// use up is set at a lower point. -const LayerBudget int = 94 - -// State holds the runtime state that is carried around in Nixery and -// passed to builder functions. -type State struct { - Storage storage.Backend - Cache *LocalCache - Cfg config.Config - Pop layers.Popularity -} - -// Architecture represents the possible CPU architectures for which -// container images can be built. -// -// The default architecture is amd64, but support for ARM platforms is -// available within nixpkgs and can be toggled via meta-packages. -type Architecture struct { - // Name of the system tuple to pass to Nix - nixSystem string - - // Name of the architecture as used in the OCI manifests - imageArch string -} - -var amd64 = Architecture{"x86_64-linux", "amd64"} -var arm64 = Architecture{"aarch64-linux", "arm64"} - -// Image represents the information necessary for building a container image. -// This can be either a list of package names (corresponding to keys in the -// nixpkgs set) or a Nix expression that results in a *list* of derivations. -type Image struct { - Name string - Tag string - - // Names of packages to include in the image. These must correspond - // directly to top-level names of Nix packages in the nixpkgs tree. - Packages []string - - // Architecture for which to build the image. Nixery defaults - // this to amd64 if not specified via meta-packages. - Arch *Architecture -} - -// BuildResult represents the data returned from the server to the -// HTTP handlers. Error information is propagated straight from Nix -// for errors inside of the build that should be fed back to the -// client (such as missing packages). -type BuildResult struct { - Error string `json:"error"` - Pkgs []string `json:"pkgs"` - Manifest json.RawMessage `json:"manifest"` -} - -// ImageFromName parses an image name into the corresponding structure which can -// be used to invoke Nix. -// -// It will expand convenience names under the hood (see the `convenienceNames` -// function below) and append packages that are always included (cacert, iana-etc). -// -// Once assembled the image structure uses a sorted representation of -// the name. This is to avoid unnecessarily cache-busting images if -// only the order of requested packages has changed. -func ImageFromName(name string, tag string) Image { - pkgs := strings.Split(name, "/") - arch, expanded := metaPackages(pkgs) - expanded = append(expanded, "cacert", "iana-etc") - - sort.Strings(pkgs) - sort.Strings(expanded) - - return Image{ - Name: strings.Join(pkgs, "/"), - Tag: tag, - Packages: expanded, - Arch: arch, - } -} - -// ImageResult represents the output of calling the Nix derivation -// responsible for preparing an image. -type ImageResult struct { - // These fields are populated in case of an error - Error string `json:"error"` - Pkgs []string `json:"pkgs"` - - // These fields are populated in case of success - Graph layers.RuntimeGraph `json:"runtimeGraph"` - SymlinkLayer struct { - Size int `json:"size"` - TarHash string `json:"tarHash"` - Path string `json:"path"` - } `json:"symlinkLayer"` -} - -// metaPackages expands package names defined by Nixery which either -// include sets of packages or trigger certain image-building -// behaviour. -// -// Meta-packages must be specified as the first packages in an image -// name. -// -// Currently defined meta-packages are: -// -// * `shell`: Includes bash, coreutils and other common command-line tools -// * `arm64`: Causes Nixery to build images for the ARM64 architecture -func metaPackages(packages []string) (*Architecture, []string) { - arch := &amd64 - - var metapkgs []string - lastMeta := 0 - for idx, p := range packages { - if p == "shell" || p == "arm64" { - metapkgs = append(metapkgs, p) - lastMeta = idx + 1 - } else { - break - } - } - - // Chop off the meta-packages from the front of the package - // list - packages = packages[lastMeta:] - - for _, p := range metapkgs { - switch p { - case "shell": - packages = append(packages, "bashInteractive", "coreutils", "moreutils", "nano") - case "arm64": - arch = &arm64 - } - } - - return arch, packages -} - -// logNix logs each output line from Nix. It runs in a goroutine per -// output channel that should be live-logged. -func logNix(image, cmd string, r io.ReadCloser) { - scanner := bufio.NewScanner(r) - for scanner.Scan() { - log.WithFields(log.Fields{ - "image": image, - "cmd": cmd, - }).Info("[nix] " + scanner.Text()) - } -} - -func callNix(program, image string, args []string) ([]byte, error) { - cmd := exec.Command(program, args...) - - outpipe, err := cmd.StdoutPipe() - if err != nil { - return nil, err - } - - errpipe, err := cmd.StderrPipe() - if err != nil { - return nil, err - } - go logNix(program, image, errpipe) - - if err = cmd.Start(); err != nil { - log.WithError(err).WithFields(log.Fields{ - "image": image, - "cmd": program, - }).Error("error invoking Nix") - - return nil, err - } - - log.WithFields(log.Fields{ - "cmd": program, - "image": image, - }).Info("invoked Nix build") - - stdout, _ := ioutil.ReadAll(outpipe) - - if err = cmd.Wait(); err != nil { - log.WithError(err).WithFields(log.Fields{ - "image": image, - "cmd": program, - "stdout": stdout, - }).Info("failed to invoke Nix") - - return nil, err - } - - resultFile := strings.TrimSpace(string(stdout)) - buildOutput, err := ioutil.ReadFile(resultFile) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "image": image, - "file": resultFile, - }).Info("failed to read Nix result file") - - return nil, err - } - - return buildOutput, nil -} - -// Call out to Nix and request metadata for the image to be built. All -// required store paths for the image will be realised, but layers -// will not yet be created from them. -// -// This function is only invoked if the manifest is not found in any -// cache. -func prepareImage(s *State, image *Image) (*ImageResult, error) { - packages, err := json.Marshal(image.Packages) - if err != nil { - return nil, err - } - - srcType, srcArgs := s.Cfg.Pkgs.Render(image.Tag) - - args := []string{ - "--timeout", s.Cfg.Timeout, - "--argstr", "packages", string(packages), - "--argstr", "srcType", srcType, - "--argstr", "srcArgs", srcArgs, - "--argstr", "system", image.Arch.nixSystem, - } - - output, err := callNix("nixery-build-image", image.Name, args) - if err != nil { - // granular error logging is performed in callNix already - return nil, err - } - - log.WithFields(log.Fields{ - "image": image.Name, - "tag": image.Tag, - }).Info("finished image preparation via Nix") - - var result ImageResult - err = json.Unmarshal(output, &result) - if err != nil { - return nil, err - } - - return &result, nil -} - -// Groups layers and checks whether they are present in the cache -// already, otherwise calls out to Nix to assemble layers. -// -// Newly built layers are uploaded to the bucket. Cache entries are -// added only after successful uploads, which guarantees that entries -// retrieved from the cache are present in the bucket. -func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageResult) ([]manifest.Entry, error) { - grouped := layers.Group(&result.Graph, &s.Pop, LayerBudget) - - var entries []manifest.Entry - - // Splits the layers into those which are already present in - // the cache, and those that are missing. - // - // Missing layers are built and uploaded to the storage - // bucket. - for _, l := range grouped { - if entry, cached := layerFromCache(ctx, s, l.Hash()); cached { - entries = append(entries, *entry) - } else { - lh := l.Hash() - - // While packing store paths, the SHA sum of - // the uncompressed layer is computed and - // written to `tarhash`. - // - // TODO(tazjin): Refactor this to make the - // flow of data cleaner. - var tarhash string - lw := func(w io.Writer) error { - var err error - tarhash, err = packStorePaths(&l, w) - return err - } - - entry, err := uploadHashLayer(ctx, s, lh, lw) - if err != nil { - return nil, err - } - entry.MergeRating = l.MergeRating - entry.TarHash = tarhash - - var pkgs []string - for _, p := range l.Contents { - pkgs = append(pkgs, layers.PackageFromPath(p)) - } - - log.WithFields(log.Fields{ - "layer": lh, - "packages": pkgs, - "tarhash": tarhash, - }).Info("created image layer") - - go cacheLayer(ctx, s, l.Hash(), *entry) - entries = append(entries, *entry) - } - } - - // Symlink layer (built in the first Nix build) needs to be - // included here manually: - slkey := result.SymlinkLayer.TarHash - entry, err := uploadHashLayer(ctx, s, slkey, func(w io.Writer) error { - f, err := os.Open(result.SymlinkLayer.Path) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "image": image.Name, - "tag": image.Tag, - "layer": slkey, - }).Error("failed to open symlink layer") - - return err - } - defer f.Close() - - gz := gzip.NewWriter(w) - _, err = io.Copy(gz, f) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "image": image.Name, - "tag": image.Tag, - "layer": slkey, - }).Error("failed to upload symlink layer") - - return err - } - - return gz.Close() - }) - - if err != nil { - return nil, err - } - - entry.TarHash = "sha256:" + result.SymlinkLayer.TarHash - go cacheLayer(ctx, s, slkey, *entry) - entries = append(entries, *entry) - - return entries, nil -} - -// layerWriter is the type for functions that can write a layer to the -// multiwriter used for uploading & hashing. -// -// This type exists to avoid duplication between the handling of -// symlink layers and store path layers. -type layerWriter func(w io.Writer) error - -// byteCounter is a special io.Writer that counts all bytes written to -// it and does nothing else. -// -// This is required because the ad-hoc writing of tarballs leaves no -// single place to count the final tarball size otherwise. -type byteCounter struct { - count int64 -} - -func (b *byteCounter) Write(p []byte) (n int, err error) { - b.count += int64(len(p)) - return len(p), nil -} - -// Upload a layer tarball to the storage bucket, while hashing it at -// the same time. The supplied function is expected to provide the -// layer data to the writer. -// -// The initial upload is performed in a 'staging' folder, as the -// SHA256-hash is not yet available when the upload is initiated. -// -// After a successful upload, the file is moved to its final location -// in the bucket and the build cache is populated. -// -// The return value is the layer's SHA256 hash, which is used in the -// image manifest. -func uploadHashLayer(ctx context.Context, s *State, key string, lw layerWriter) (*manifest.Entry, error) { - path := "staging/" + key - sha256sum, size, err := s.Storage.Persist(ctx, path, func(sw io.Writer) (string, int64, error) { - // Sets up a "multiwriter" that simultaneously runs both hash - // algorithms and uploads to the storage backend. - shasum := sha256.New() - counter := &byteCounter{} - multi := io.MultiWriter(sw, shasum, counter) - - err := lw(multi) - sha256sum := fmt.Sprintf("%x", shasum.Sum([]byte{})) - - return sha256sum, counter.count, err - }) - - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "layer": key, - "backend": s.Storage.Name(), - }).Error("failed to create and store layer") - - return nil, err - } - - // Hashes are now known and the object is in the bucket, what - // remains is to move it to the correct location and cache it. - err = s.Storage.Move(ctx, "staging/"+key, "layers/"+sha256sum) - if err != nil { - log.WithError(err).WithField("layer", key). - Error("failed to move layer from staging") - - return nil, err - } - - log.WithFields(log.Fields{ - "layer": key, - "sha256": sha256sum, - "size": size, - }).Info("created and persisted layer") - - entry := manifest.Entry{ - Digest: "sha256:" + sha256sum, - Size: size, - } - - return &entry, nil -} - -func BuildImage(ctx context.Context, s *State, image *Image) (*BuildResult, error) { - key := s.Cfg.Pkgs.CacheKey(image.Packages, image.Tag) - if key != "" { - if m, c := manifestFromCache(ctx, s, key); c { - return &BuildResult{ - Manifest: m, - }, nil - } - } - - imageResult, err := prepareImage(s, image) - if err != nil { - return nil, err - } - - if imageResult.Error != "" { - return &BuildResult{ - Error: imageResult.Error, - Pkgs: imageResult.Pkgs, - }, nil - } - - layers, err := prepareLayers(ctx, s, image, imageResult) - if err != nil { - return nil, err - } - - m, c := manifest.Manifest(image.Arch.imageArch, layers) - - lw := func(w io.Writer) error { - r := bytes.NewReader(c.Config) - _, err := io.Copy(w, r) - return err - } - - if _, err = uploadHashLayer(ctx, s, c.SHA256, lw); err != nil { - log.WithError(err).WithFields(log.Fields{ - "image": image.Name, - "tag": image.Tag, - }).Error("failed to upload config") - - return nil, err - } - - if key != "" { - go cacheManifest(ctx, s, key, m) - } - - result := BuildResult{ - Manifest: m, - } - return &result, nil -} diff --git a/tools/nixery/server/builder/builder_test.go b/tools/nixery/server/builder/builder_test.go deleted file mode 100644 index 3fbe2ab40e23..000000000000 --- a/tools/nixery/server/builder/builder_test.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. -package builder - -import ( - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "testing" -) - -var ignoreArch = cmpopts.IgnoreFields(Image{}, "Arch") - -func TestImageFromNameSimple(t *testing.T) { - image := ImageFromName("hello", "latest") - expected := Image{ - Name: "hello", - Tag: "latest", - Packages: []string{ - "cacert", - "hello", - "iana-etc", - }, - } - - if diff := cmp.Diff(expected, image, ignoreArch); diff != "" { - t.Fatalf("Image(\"hello\", \"latest\") mismatch:\n%s", diff) - } -} - -func TestImageFromNameMultiple(t *testing.T) { - image := ImageFromName("hello/git/htop", "latest") - expected := Image{ - Name: "git/hello/htop", - Tag: "latest", - Packages: []string{ - "cacert", - "git", - "hello", - "htop", - "iana-etc", - }, - } - - if diff := cmp.Diff(expected, image, ignoreArch); diff != "" { - t.Fatalf("Image(\"hello/git/htop\", \"latest\") mismatch:\n%s", diff) - } -} - -func TestImageFromNameShell(t *testing.T) { - image := ImageFromName("shell", "latest") - expected := Image{ - Name: "shell", - Tag: "latest", - Packages: []string{ - "bashInteractive", - "cacert", - "coreutils", - "iana-etc", - "moreutils", - "nano", - }, - } - - if diff := cmp.Diff(expected, image, ignoreArch); diff != "" { - t.Fatalf("Image(\"shell\", \"latest\") mismatch:\n%s", diff) - } -} - -func TestImageFromNameShellMultiple(t *testing.T) { - image := ImageFromName("shell/htop", "latest") - expected := Image{ - Name: "htop/shell", - Tag: "latest", - Packages: []string{ - "bashInteractive", - "cacert", - "coreutils", - "htop", - "iana-etc", - "moreutils", - "nano", - }, - } - - if diff := cmp.Diff(expected, image, ignoreArch); diff != "" { - t.Fatalf("Image(\"shell/htop\", \"latest\") mismatch:\n%s", diff) - } -} - -func TestImageFromNameShellArm64(t *testing.T) { - image := ImageFromName("shell/arm64", "latest") - expected := Image{ - Name: "arm64/shell", - Tag: "latest", - Packages: []string{ - "bashInteractive", - "cacert", - "coreutils", - "iana-etc", - "moreutils", - "nano", - }, - } - - if diff := cmp.Diff(expected, image, ignoreArch); diff != "" { - t.Fatalf("Image(\"shell/arm64\", \"latest\") mismatch:\n%s", diff) - } - - if image.Arch.imageArch != "arm64" { - t.Fatal("Image(\"shell/arm64\"): Expected arch arm64") - } -} diff --git a/tools/nixery/server/builder/cache.go b/tools/nixery/server/builder/cache.go deleted file mode 100644 index 82bd90927cd0..000000000000 --- a/tools/nixery/server/builder/cache.go +++ /dev/null @@ -1,236 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. -package builder - -import ( - "bytes" - "context" - "encoding/json" - "io" - "io/ioutil" - "os" - "sync" - - "github.com/google/nixery/server/manifest" - log "github.com/sirupsen/logrus" -) - -// LocalCache implements the structure used for local caching of -// manifests and layer uploads. -type LocalCache struct { - // Manifest cache - mmtx sync.RWMutex - mdir string - - // Layer cache - lmtx sync.RWMutex - lcache map[string]manifest.Entry -} - -// Creates an in-memory cache and ensures that the local file path for -// manifest caching exists. -func NewCache() (LocalCache, error) { - path := os.TempDir() + "/nixery" - err := os.MkdirAll(path, 0755) - if err != nil { - return LocalCache{}, err - } - - return LocalCache{ - mdir: path + "/", - lcache: make(map[string]manifest.Entry), - }, nil -} - -// Retrieve a cached manifest if the build is cacheable and it exists. -func (c *LocalCache) manifestFromLocalCache(key string) (json.RawMessage, bool) { - c.mmtx.RLock() - defer c.mmtx.RUnlock() - - f, err := os.Open(c.mdir + key) - if err != nil { - // This is a debug log statement because failure to - // read the manifest key is currently expected if it - // is not cached. - log.WithError(err).WithField("manifest", key). - Debug("failed to read manifest from local cache") - - return nil, false - } - defer f.Close() - - m, err := ioutil.ReadAll(f) - if err != nil { - log.WithError(err).WithField("manifest", key). - Error("failed to read manifest from local cache") - - return nil, false - } - - return json.RawMessage(m), true -} - -// Adds the result of a manifest build to the local cache, if the -// manifest is considered cacheable. -// -// Manifests can be quite large and are cached on disk instead of in -// memory. -func (c *LocalCache) localCacheManifest(key string, m json.RawMessage) { - c.mmtx.Lock() - defer c.mmtx.Unlock() - - err := ioutil.WriteFile(c.mdir+key, []byte(m), 0644) - if err != nil { - log.WithError(err).WithField("manifest", key). - Error("failed to locally cache manifest") - } -} - -// Retrieve a layer build from the local cache. -func (c *LocalCache) layerFromLocalCache(key string) (*manifest.Entry, bool) { - c.lmtx.RLock() - e, ok := c.lcache[key] - c.lmtx.RUnlock() - - return &e, ok -} - -// Add a layer build result to the local cache. -func (c *LocalCache) localCacheLayer(key string, e manifest.Entry) { - c.lmtx.Lock() - c.lcache[key] = e - c.lmtx.Unlock() -} - -// Retrieve a manifest from the cache(s). First the local cache is -// checked, then the storage backend. -func manifestFromCache(ctx context.Context, s *State, key string) (json.RawMessage, bool) { - if m, cached := s.Cache.manifestFromLocalCache(key); cached { - return m, true - } - - r, err := s.Storage.Fetch(ctx, "manifests/"+key) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "manifest": key, - "backend": s.Storage.Name(), - }).Error("failed to fetch manifest from cache") - - return nil, false - } - defer r.Close() - - m, err := ioutil.ReadAll(r) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "manifest": key, - "backend": s.Storage.Name(), - }).Error("failed to read cached manifest from storage backend") - - return nil, false - } - - go s.Cache.localCacheManifest(key, m) - log.WithField("manifest", key).Info("retrieved manifest from GCS") - - return json.RawMessage(m), true -} - -// Add a manifest to the bucket & local caches -func cacheManifest(ctx context.Context, s *State, key string, m json.RawMessage) { - go s.Cache.localCacheManifest(key, m) - - path := "manifests/" + key - _, size, err := s.Storage.Persist(ctx, path, func(w io.Writer) (string, int64, error) { - size, err := io.Copy(w, bytes.NewReader([]byte(m))) - return "", size, err - }) - - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "manifest": key, - "backend": s.Storage.Name(), - }).Error("failed to cache manifest to storage backend") - - return - } - - log.WithFields(log.Fields{ - "manifest": key, - "size": size, - "backend": s.Storage.Name(), - }).Info("cached manifest to storage backend") -} - -// Retrieve a layer build from the cache, first checking the local -// cache followed by the bucket cache. -func layerFromCache(ctx context.Context, s *State, key string) (*manifest.Entry, bool) { - if entry, cached := s.Cache.layerFromLocalCache(key); cached { - return entry, true - } - - r, err := s.Storage.Fetch(ctx, "builds/"+key) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "layer": key, - "backend": s.Storage.Name(), - }).Debug("failed to retrieve cached layer from storage backend") - - return nil, false - } - defer r.Close() - - jb := bytes.NewBuffer([]byte{}) - _, err = io.Copy(jb, r) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "layer": key, - "backend": s.Storage.Name(), - }).Error("failed to read cached layer from storage backend") - - return nil, false - } - - var entry manifest.Entry - err = json.Unmarshal(jb.Bytes(), &entry) - if err != nil { - log.WithError(err).WithField("layer", key). - Error("failed to unmarshal cached layer") - - return nil, false - } - - go s.Cache.localCacheLayer(key, entry) - return &entry, true -} - -func cacheLayer(ctx context.Context, s *State, key string, entry manifest.Entry) { - s.Cache.localCacheLayer(key, entry) - - j, _ := json.Marshal(&entry) - path := "builds/" + key - _, _, err := s.Storage.Persist(ctx, path, func(w io.Writer) (string, int64, error) { - size, err := io.Copy(w, bytes.NewReader(j)) - return "", size, err - }) - - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "layer": key, - "backend": s.Storage.Name(), - }).Error("failed to cache layer") - } - - return -} diff --git a/tools/nixery/server/config/config.go b/tools/nixery/server/config/config.go deleted file mode 100644 index 7ec102bd6cee..000000000000 --- a/tools/nixery/server/config/config.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -// Package config implements structures to store Nixery's configuration at -// runtime as well as the logic for instantiating this configuration from the -// environment. -package config - -import ( - "os" - - log "github.com/sirupsen/logrus" -) - -func getConfig(key, desc, def string) string { - value := os.Getenv(key) - if value == "" && def == "" { - log.WithFields(log.Fields{ - "option": key, - "description": desc, - }).Fatal("missing required configuration envvar") - } else if value == "" { - return def - } - - return value -} - -// Backend represents the possible storage backend types -type Backend int - -const ( - GCS = iota - FileSystem -) - -// Config holds the Nixery configuration options. -type Config struct { - Port string // Port on which to launch HTTP server - Pkgs PkgSource // Source for Nix package set - Timeout string // Timeout for a single Nix builder (seconds) - WebDir string // Directory with static web assets - PopUrl string // URL to the Nix package popularity count - Backend Backend // Storage backend to use for Nixery -} - -func FromEnv() (Config, error) { - pkgs, err := pkgSourceFromEnv() - if err != nil { - return Config{}, err - } - - var b Backend - switch os.Getenv("NIXERY_STORAGE_BACKEND") { - case "gcs": - b = GCS - case "filesystem": - b = FileSystem - default: - log.WithField("values", []string{ - "gcs", - }).Fatal("NIXERY_STORAGE_BUCKET must be set to a supported value") - } - - return Config{ - Port: getConfig("PORT", "HTTP port", ""), - Pkgs: pkgs, - Timeout: getConfig("NIX_TIMEOUT", "Nix builder timeout", "60"), - WebDir: getConfig("WEB_DIR", "Static web file dir", ""), - PopUrl: os.Getenv("NIX_POPULARITY_URL"), - Backend: b, - }, nil -} diff --git a/tools/nixery/server/config/pkgsource.go b/tools/nixery/server/config/pkgsource.go deleted file mode 100644 index 95236c4b0d15..000000000000 --- a/tools/nixery/server/config/pkgsource.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. -package config - -import ( - "crypto/sha1" - "encoding/json" - "fmt" - "os" - "regexp" - "strings" - - log "github.com/sirupsen/logrus" -) - -// PkgSource represents the source from which the Nix package set used -// by Nixery is imported. Users configure the source by setting one of -// the supported environment variables. -type PkgSource interface { - // Convert the package source into the representation required - // for calling Nix. - Render(tag string) (string, string) - - // Create a key by which builds for this source and iamge - // combination can be cached. - // - // The empty string means that this value is not cacheable due - // to the package source being a moving target (such as a - // channel). - CacheKey(pkgs []string, tag string) string -} - -type GitSource struct { - repository string -} - -// Regex to determine whether a git reference is a commit hash or -// something else (branch/tag). -// -// Used to check whether a git reference is cacheable, and to pass the -// correct git structure to Nix. -// -// Note: If a user creates a branch or tag with the name of a commit -// and references it intentionally, this heuristic will fail. -var commitRegex = regexp.MustCompile(`^[0-9a-f]{40}$`) - -func (g *GitSource) Render(tag string) (string, string) { - args := map[string]string{ - "url": g.repository, - } - - // The 'git' source requires a tag to be present. If the user - // has not specified one, it is assumed that the default - // 'master' branch should be used. - if tag == "latest" || tag == "" { - tag = "master" - } - - if commitRegex.MatchString(tag) { - args["rev"] = tag - } else { - args["ref"] = tag - } - - j, _ := json.Marshal(args) - - return "git", string(j) -} - -func (g *GitSource) CacheKey(pkgs []string, tag string) string { - // Only full commit hashes can be used for caching, as - // everything else is potentially a moving target. - if !commitRegex.MatchString(tag) { - return "" - } - - unhashed := strings.Join(pkgs, "") + tag - hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed))) - - return hashed -} - -type NixChannel struct { - channel string -} - -func (n *NixChannel) Render(tag string) (string, string) { - return "nixpkgs", n.channel -} - -func (n *NixChannel) CacheKey(pkgs []string, tag string) string { - // Since Nix channels are downloaded from the nixpkgs-channels - // Github, users can specify full commit hashes as the - // "channel", in which case builds are cacheable. - if !commitRegex.MatchString(n.channel) { - return "" - } - - unhashed := strings.Join(pkgs, "") + n.channel - hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed))) - - return hashed -} - -type PkgsPath struct { - path string -} - -func (p *PkgsPath) Render(tag string) (string, string) { - return "path", p.path -} - -func (p *PkgsPath) CacheKey(pkgs []string, tag string) string { - // Path-based builds are not currently cacheable because we - // have no local hash of the package folder's state easily - // available. - return "" -} - -// Retrieve a package source from the environment. If no source is -// specified, the Nix code will default to a recent NixOS channel. -func pkgSourceFromEnv() (PkgSource, error) { - if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" { - log.WithField("channel", channel).Info("using Nix package set from Nix channel or commit") - - return &NixChannel{ - channel: channel, - }, nil - } - - if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" { - log.WithField("repo", git).Info("using NIx package set from git repository") - - return &GitSource{ - repository: git, - }, nil - } - - if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" { - log.WithField("path", path).Info("using Nix package set at local path") - - return &PkgsPath{ - path: path, - }, nil - } - - return nil, fmt.Errorf("no valid package source has been specified") -} diff --git a/tools/nixery/server/default.nix b/tools/nixery/server/default.nix deleted file mode 100644 index d497f106b02e..000000000000 --- a/tools/nixery/server/default.nix +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{ buildGoPackage, go, lib, srcHash }: - -buildGoPackage rec { - name = "nixery-server"; - goDeps = ./go-deps.nix; - src = ./.; - - goPackagePath = "github.com/google/nixery/server"; - doCheck = true; - - # The following phase configurations work around the overengineered - # Nix build configuration for Go. - # - # All I want this to do is produce a binary in the standard Nix - # output path, so pretty much all the phases except for the initial - # configuration of the "dependency forest" in $GOPATH have been - # overridden. - # - # This is necessary because the upstream builder does wonky things - # with the build arguments to the compiler, but I need to set some - # complex flags myself - - outputs = [ "out" ]; - preConfigure = "bin=$out"; - buildPhase = '' - runHook preBuild - runHook renameImport - - export GOBIN="$out/bin" - go install -ldflags "-X main.version=$(cat ${srcHash})" ${goPackagePath} - ''; - - fixupPhase = '' - remove-references-to -t ${go} $out/bin/server - ''; - - checkPhase = '' - go vet ${goPackagePath} - go test ${goPackagePath} - ''; - - meta = { - description = "Container image builder serving Nix-backed images"; - homepage = "https://github.com/google/nixery"; - license = lib.licenses.asl20; - maintainers = [ lib.maintainers.tazjin ]; - }; -} diff --git a/tools/nixery/server/go-deps.nix b/tools/nixery/server/go-deps.nix deleted file mode 100644 index 847b44dce63c..000000000000 --- a/tools/nixery/server/go-deps.nix +++ /dev/null @@ -1,129 +0,0 @@ -# This file was generated by https://github.com/kamilchm/go2nix v1.3.0 -[ - { - goPackagePath = "cloud.google.com/go"; - fetch = { - type = "git"; - url = "https://code.googlesource.com/gocloud"; - rev = "77f6a3a292a7dbf66a5329de0d06326f1906b450"; - sha256 = "1c9pkx782nbcp8jnl5lprcbzf97van789ky5qsncjgywjyymhigi"; - }; - } - { - goPackagePath = "github.com/golang/protobuf"; - fetch = { - type = "git"; - url = "https://github.com/golang/protobuf"; - rev = "6c65a5562fc06764971b7c5d05c76c75e84bdbf7"; - sha256 = "1k1wb4zr0qbwgpvz9q5ws9zhlal8hq7dmq62pwxxriksayl6hzym"; - }; - } - { - goPackagePath = "github.com/googleapis/gax-go"; - fetch = { - type = "git"; - url = "https://github.com/googleapis/gax-go"; - rev = "bd5b16380fd03dc758d11cef74ba2e3bc8b0e8c2"; - sha256 = "1lxawwngv6miaqd25s3ba0didfzylbwisd2nz7r4gmbmin6jsjrx"; - }; - } - { - goPackagePath = "github.com/hashicorp/golang-lru"; - fetch = { - type = "git"; - url = "https://github.com/hashicorp/golang-lru"; - rev = "59383c442f7d7b190497e9bb8fc17a48d06cd03f"; - sha256 = "0yzwl592aa32vfy73pl7wdc21855w17zssrp85ckw2nisky8rg9c"; - }; - } - { - goPackagePath = "go.opencensus.io"; - fetch = { - type = "git"; - url = "https://github.com/census-instrumentation/opencensus-go"; - rev = "b4a14686f0a98096416fe1b4cb848e384fb2b22b"; - sha256 = "1aidyp301v5ngwsnnc8v1s09vvbsnch1jc4vd615f7qv77r9s7dn"; - }; - } - { - goPackagePath = "golang.org/x/net"; - fetch = { - type = "git"; - url = "https://go.googlesource.com/net"; - rev = "da137c7871d730100384dbcf36e6f8fa493aef5b"; - sha256 = "1qsiyr3irmb6ii06hivm9p2c7wqyxczms1a9v1ss5698yjr3fg47"; - }; - } - { - goPackagePath = "golang.org/x/oauth2"; - fetch = { - type = "git"; - url = "https://go.googlesource.com/oauth2"; - rev = "0f29369cfe4552d0e4bcddc57cc75f4d7e672a33"; - sha256 = "06jwpvx0x2gjn2y959drbcir5kd7vg87k0r1216abk6rrdzzrzi2"; - }; - } - { - goPackagePath = "golang.org/x/sys"; - fetch = { - type = "git"; - url = "https://go.googlesource.com/sys"; - rev = "51ab0e2deafac1f46c46ad59cf0921be2f180c3d"; - sha256 = "0xdhpckbql3bsqkpc2k5b1cpnq3q1qjqjjq2j3p707rfwb8nm91a"; - }; - } - { - goPackagePath = "golang.org/x/text"; - fetch = { - type = "git"; - url = "https://go.googlesource.com/text"; - rev = "342b2e1fbaa52c93f31447ad2c6abc048c63e475"; - sha256 = "0flv9idw0jm5nm8lx25xqanbkqgfiym6619w575p7nrdh0riqwqh"; - }; - } - { - goPackagePath = "google.golang.org/api"; - fetch = { - type = "git"; - url = "https://code.googlesource.com/google-api-go-client"; - rev = "069bea57b1be6ad0671a49ea7a1128025a22b73f"; - sha256 = "19q2b610lkf3z3y9hn6rf11dd78xr9q4340mdyri7kbijlj2r44q"; - }; - } - { - goPackagePath = "google.golang.org/genproto"; - fetch = { - type = "git"; - url = "https://github.com/google/go-genproto"; - rev = "c506a9f9061087022822e8da603a52fc387115a8"; - sha256 = "03hh80aqi58dqi5ykj4shk3chwkzrgq2f3k6qs5qhgvmcy79y2py"; - }; - } - { - goPackagePath = "google.golang.org/grpc"; - fetch = { - type = "git"; - url = "https://github.com/grpc/grpc-go"; - rev = "977142214c45640483838b8672a43c46f89f90cb"; - sha256 = "05wig23l2sil3bfdv19gq62sya7hsabqj9l8pzr1sm57qsvj218d"; - }; - } - { - goPackagePath = "gonum.org/v1/gonum"; - fetch = { - type = "git"; - url = "https://github.com/gonum/gonum"; - rev = "ced62fe5104b907b6c16cb7e575c17b2e62ceddd"; - sha256 = "1b7q6haabnp53igpmvr6a2414yralhbrldixx4kbxxg1apy8jdjg"; - }; - } - { - goPackagePath = "github.com/sirupsen/logrus"; - fetch = { - type = "git"; - url = "https://github.com/sirupsen/logrus"; - rev = "de736cf91b921d56253b4010270681d33fdf7cb5"; - sha256 = "1qixss8m5xy7pzbf0qz2k3shjw0asklm9sj6zyczp7mryrari0aj"; - }; - } -] diff --git a/tools/nixery/server/layers/grouping.go b/tools/nixery/server/layers/grouping.go deleted file mode 100644 index 3902c8a4ef26..000000000000 --- a/tools/nixery/server/layers/grouping.go +++ /dev/null @@ -1,361 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -// This package reads an export reference graph (i.e. a graph representing the -// runtime dependencies of a set of derivations) created by Nix and groups it in -// a way that is likely to match the grouping for other derivation sets with -// overlapping dependencies. -// -// This is used to determine which derivations to include in which layers of a -// container image. -// -// # Inputs -// -// * a graph of Nix runtime dependencies, generated via exportReferenceGraph -// * popularity values of each package in the Nix package set (in the form of a -// direct reference count) -// * a maximum number of layers to allocate for the image (the "layer budget") -// -// # Algorithm -// -// It works by first creating a (directed) dependency tree: -// -// img (root node) -// │ -// ├───> A ─────┐ -// │ v -// ├───> B ───> E -// │ ^ -// ├───> C ─────┘ -// │ │ -// │ v -// └───> D ───> F -// │ -// └────> G -// -// Each node (i.e. package) is then visited to determine how important -// it is to separate this node into its own layer, specifically: -// -// 1. Is the node within a certain threshold percentile of absolute -// popularity within all of nixpkgs? (e.g. `glibc`, `openssl`) -// -// 2. Is the node's runtime closure above a threshold size? (e.g. 100MB) -// -// In either case, a bit is flipped for this node representing each -// condition and an edge to it is inserted directly from the image -// root, if it does not already exist. -// -// For the rest of the example we assume 'G' is above the threshold -// size and 'E' is popular. -// -// This tree is then transformed into a dominator tree: -// -// img -// │ -// ├───> A -// ├───> B -// ├───> C -// ├───> E -// ├───> D ───> F -// └───> G -// -// Specifically this means that the paths to A, B, C, E, G, and D -// always pass through the root (i.e. are dominated by it), whilst F -// is dominated by D (all paths go through it). -// -// The top-level subtrees are considered as the initially selected -// layers. -// -// If the list of layers fits within the layer budget, it is returned. -// -// Otherwise, a merge rating is calculated for each layer. This is the -// product of the layer's total size and its root node's popularity. -// -// Layers are then merged in ascending order of merge ratings until -// they fit into the layer budget. -// -// # Threshold values -// -// Threshold values for the partitioning conditions mentioned above -// have not yet been determined, but we will make a good first guess -// based on gut feeling and proceed to measure their impact on cache -// hits/misses. -// -// # Example -// -// Using the logic described above as well as the example presented in -// the introduction, this program would create the following layer -// groupings (assuming no additional partitioning): -// -// Layer budget: 1 -// Layers: { A, B, C, D, E, F, G } -// -// Layer budget: 2 -// Layers: { G }, { A, B, C, D, E, F } -// -// Layer budget: 3 -// Layers: { G }, { E }, { A, B, C, D, F } -// -// Layer budget: 4 -// Layers: { G }, { E }, { D, F }, { A, B, C } -// -// ... -// -// Layer budget: 10 -// Layers: { E }, { D, F }, { A }, { B }, { C } -package layers - -import ( - "crypto/sha1" - "fmt" - "regexp" - "sort" - "strings" - - log "github.com/sirupsen/logrus" - "gonum.org/v1/gonum/graph/flow" - "gonum.org/v1/gonum/graph/simple" -) - -// RuntimeGraph represents structured information from Nix about the runtime -// dependencies of a derivation. -// -// This is generated in Nix by using the exportReferencesGraph feature. -type RuntimeGraph struct { - References struct { - Graph []string `json:"graph"` - } `json:"exportReferencesGraph"` - - Graph []struct { - Size uint64 `json:"closureSize"` - Path string `json:"path"` - Refs []string `json:"references"` - } `json:"graph"` -} - -// Popularity data for each Nix package that was calculated in advance. -// -// Popularity is a number from 1-100 that represents the -// popularity percentile in which this package resides inside -// of the nixpkgs tree. -type Popularity = map[string]int - -// Layer represents the data returned for each layer that Nix should -// build for the container image. -type Layer struct { - Contents []string `json:"contents"` - MergeRating uint64 -} - -// Hash the contents of a layer to create a deterministic identifier that can be -// used for caching. -func (l *Layer) Hash() string { - sum := sha1.Sum([]byte(strings.Join(l.Contents, ":"))) - return fmt.Sprintf("%x", sum) -} - -func (a Layer) merge(b Layer) Layer { - a.Contents = append(a.Contents, b.Contents...) - a.MergeRating += b.MergeRating - return a -} - -// closure as pointed to by the graph nodes. -type closure struct { - GraphID int64 - Path string - Size uint64 - Refs []string - Popularity int -} - -func (c *closure) ID() int64 { - return c.GraphID -} - -var nixRegexp = regexp.MustCompile(`^/nix/store/[a-z0-9]+-`) - -// PackageFromPath returns the name of a Nix package based on its -// output store path. -func PackageFromPath(path string) string { - return nixRegexp.ReplaceAllString(path, "") -} - -func (c *closure) DOTID() string { - return PackageFromPath(c.Path) -} - -// bigOrPopular checks whether this closure should be considered for -// separation into its own layer, even if it would otherwise only -// appear in a subtree of the dominator tree. -func (c *closure) bigOrPopular() bool { - const sizeThreshold = 100 * 1000000 // 100MB - - if c.Size > sizeThreshold { - return true - } - - // Threshold value is picked arbitrarily right now. The reason - // for this is that some packages (such as `cacert`) have very - // few direct dependencies, but are required by pretty much - // everything. - if c.Popularity >= 100 { - return true - } - - return false -} - -func insertEdges(graph *simple.DirectedGraph, cmap *map[string]*closure, node *closure) { - // Big or popular nodes get a separate edge from the top to - // flag them for their own layer. - if node.bigOrPopular() && !graph.HasEdgeFromTo(0, node.ID()) { - edge := graph.NewEdge(graph.Node(0), node) - graph.SetEdge(edge) - } - - for _, c := range node.Refs { - // Nix adds a self reference to each node, which - // should not be inserted. - if c != node.Path { - edge := graph.NewEdge(node, (*cmap)[c]) - graph.SetEdge(edge) - } - } -} - -// Create a graph structure from the references supplied by Nix. -func buildGraph(refs *RuntimeGraph, pop *Popularity) *simple.DirectedGraph { - cmap := make(map[string]*closure) - graph := simple.NewDirectedGraph() - - // Insert all closures into the graph, as well as a fake root - // closure which serves as the top of the tree. - // - // A map from store paths to IDs is kept to actually insert - // edges below. - root := &closure{ - GraphID: 0, - Path: "image_root", - } - graph.AddNode(root) - - for idx, c := range refs.Graph { - node := &closure{ - GraphID: int64(idx + 1), // inc because of root node - Path: c.Path, - Size: c.Size, - Refs: c.Refs, - } - - // The packages `nss-cacert` and `iana-etc` are added - // by Nixery to *every single image* and should have a - // very high popularity. - // - // Other popularity values are populated from the data - // set assembled by Nixery's popcount. - id := node.DOTID() - if strings.HasPrefix(id, "nss-cacert") || strings.HasPrefix(id, "iana-etc") { - // glibc has ~300k references, these packages need *more* - node.Popularity = 500000 - } else if p, ok := (*pop)[id]; ok { - node.Popularity = p - } else { - node.Popularity = 1 - } - - graph.AddNode(node) - cmap[c.Path] = node - } - - // Insert the top-level closures with edges from the root - // node, then insert all edges for each closure. - for _, p := range refs.References.Graph { - edge := graph.NewEdge(root, cmap[p]) - graph.SetEdge(edge) - } - - for _, c := range cmap { - insertEdges(graph, &cmap, c) - } - - return graph -} - -// Extracts a subgraph starting at the specified root from the -// dominator tree. The subgraph is converted into a flat list of -// layers, each containing the store paths and merge rating. -func groupLayer(dt *flow.DominatorTree, root *closure) Layer { - size := root.Size - contents := []string{root.Path} - children := dt.DominatedBy(root.ID()) - - // This iteration does not use 'range' because the list being - // iterated is modified during the iteration (yes, I'm sorry). - for i := 0; i < len(children); i++ { - child := children[i].(*closure) - size += child.Size - contents = append(contents, child.Path) - children = append(children, dt.DominatedBy(child.ID())...) - } - - // Contents are sorted to ensure that hashing is consistent - sort.Strings(contents) - - return Layer{ - Contents: contents, - MergeRating: uint64(root.Popularity) * size, - } -} - -// Calculate the dominator tree of the entire package set and group -// each top-level subtree into a layer. -// -// Layers are merged together until they fit into the layer budget, -// based on their merge rating. -func dominate(budget int, graph *simple.DirectedGraph) []Layer { - dt := flow.Dominators(graph.Node(0), graph) - - var layers []Layer - for _, n := range dt.DominatedBy(dt.Root().ID()) { - layers = append(layers, groupLayer(&dt, n.(*closure))) - } - - sort.Slice(layers, func(i, j int) bool { - return layers[i].MergeRating < layers[j].MergeRating - }) - - if len(layers) > budget { - log.WithFields(log.Fields{ - "layers": len(layers), - "budget": budget, - }).Info("ideal image exceeds layer budget") - } - - for len(layers) > budget { - merged := layers[0].merge(layers[1]) - layers[1] = merged - layers = layers[1:] - } - - return layers -} - -// GroupLayers applies the algorithm described above the its input and returns a -// list of layers, each consisting of a list of Nix store paths that it should -// contain. -func Group(refs *RuntimeGraph, pop *Popularity, budget int) []Layer { - graph := buildGraph(refs, pop) - return dominate(budget, graph) -} diff --git a/tools/nixery/server/logs.go b/tools/nixery/server/logs.go deleted file mode 100644 index 3179402e2e1f..000000000000 --- a/tools/nixery/server/logs.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. -package main - -// This file configures different log formatters via logrus. The -// standard formatter uses a structured JSON format that is compatible -// with Stackdriver Error Reporting. -// -// https://cloud.google.com/error-reporting/docs/formatting-error-messages - -import ( - "bytes" - "encoding/json" - log "github.com/sirupsen/logrus" -) - -type stackdriverFormatter struct{} - -type serviceContext struct { - Service string `json:"service"` - Version string `json:"version"` -} - -type reportLocation struct { - FilePath string `json:"filePath"` - LineNumber int `json:"lineNumber"` - FunctionName string `json:"functionName"` -} - -var nixeryContext = serviceContext{ - Service: "nixery", -} - -// isError determines whether an entry should be logged as an error -// (i.e. with attached `context`). -// -// This requires the caller information to be present on the log -// entry, as stacktraces are not available currently. -func isError(e *log.Entry) bool { - l := e.Level - return (l == log.ErrorLevel || l == log.FatalLevel || l == log.PanicLevel) && - e.HasCaller() -} - -// logSeverity formats the entry's severity into a format compatible -// with Stackdriver Logging. -// -// The two formats that are being mapped do not have an equivalent set -// of severities/levels, so the mapping is somewhat arbitrary for a -// handful of them. -// -// https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#LogSeverity -func logSeverity(l log.Level) string { - switch l { - case log.TraceLevel: - return "DEBUG" - case log.DebugLevel: - return "DEBUG" - case log.InfoLevel: - return "INFO" - case log.WarnLevel: - return "WARNING" - case log.ErrorLevel: - return "ERROR" - case log.FatalLevel: - return "CRITICAL" - case log.PanicLevel: - return "EMERGENCY" - default: - return "DEFAULT" - } -} - -func (f stackdriverFormatter) Format(e *log.Entry) ([]byte, error) { - msg := e.Data - msg["serviceContext"] = &nixeryContext - msg["message"] = &e.Message - msg["eventTime"] = &e.Time - msg["severity"] = logSeverity(e.Level) - - if e, ok := msg[log.ErrorKey]; ok { - if err, isError := e.(error); isError { - msg[log.ErrorKey] = err.Error() - } else { - delete(msg, log.ErrorKey) - } - } - - if isError(e) { - loc := reportLocation{ - FilePath: e.Caller.File, - LineNumber: e.Caller.Line, - FunctionName: e.Caller.Function, - } - msg["context"] = &loc - } - - b := new(bytes.Buffer) - err := json.NewEncoder(b).Encode(&msg) - - return b.Bytes(), err -} - -func init() { - nixeryContext.Version = version - log.SetReportCaller(true) - log.SetFormatter(stackdriverFormatter{}) -} diff --git a/tools/nixery/server/main.go b/tools/nixery/server/main.go deleted file mode 100644 index 6ae0730906dc..000000000000 --- a/tools/nixery/server/main.go +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -// The nixery server implements a container registry that transparently builds -// container images based on Nix derivations. -// -// The Nix derivation used for image creation is responsible for creating -// objects that are compatible with the registry API. The targeted registry -// protocol is currently Docker's. -// -// When an image is requested, the required contents are parsed out of the -// request and a Nix-build is initiated that eventually responds with the -// manifest as well as information linking each layer digest to a local -// filesystem path. -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "regexp" - - "github.com/google/nixery/server/builder" - "github.com/google/nixery/server/config" - "github.com/google/nixery/server/layers" - "github.com/google/nixery/server/storage" - log "github.com/sirupsen/logrus" -) - -// ManifestMediaType is the Content-Type used for the manifest itself. This -// corresponds to the "Image Manifest V2, Schema 2" described on this page: -// -// https://docs.docker.com/registry/spec/manifest-v2-2/ -const manifestMediaType string = "application/vnd.docker.distribution.manifest.v2+json" - -// This variable will be initialised during the build process and set -// to the hash of the entire Nixery source tree. -var version string = "devel" - -// Regexes matching the V2 Registry API routes. This only includes the -// routes required for serving images, since pushing and other such -// functionality is not available. -var ( - manifestRegex = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/manifests/([\w|\-|\.|\_]+)$`) - layerRegex = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/blobs/sha256:(\w+)$`) -) - -// Downloads the popularity information for the package set from the -// URL specified in Nixery's configuration. -func downloadPopularity(url string) (layers.Popularity, error) { - resp, err := http.Get(url) - if err != nil { - return nil, err - } - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("popularity download from '%s' returned status: %s\n", url, resp.Status) - } - - j, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - var pop layers.Popularity - err = json.Unmarshal(j, &pop) - if err != nil { - return nil, err - } - - return pop, nil -} - -// Error format corresponding to the registry protocol V2 specification. This -// allows feeding back errors to clients in a way that can be presented to -// users. -type registryError struct { - Code string `json:"code"` - Message string `json:"message"` -} - -type registryErrors struct { - Errors []registryError `json:"errors"` -} - -func writeError(w http.ResponseWriter, status int, code, message string) { - err := registryErrors{ - Errors: []registryError{ - {code, message}, - }, - } - json, _ := json.Marshal(err) - - w.WriteHeader(status) - w.Header().Add("Content-Type", "application/json") - w.Write(json) -} - -type registryHandler struct { - state *builder.State -} - -func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - // Acknowledge that we speak V2 with an empty response - if r.RequestURI == "/v2/" { - return - } - - // Serve the manifest (straight from Nix) - manifestMatches := manifestRegex.FindStringSubmatch(r.RequestURI) - if len(manifestMatches) == 3 { - imageName := manifestMatches[1] - imageTag := manifestMatches[2] - - log.WithFields(log.Fields{ - "image": imageName, - "tag": imageTag, - }).Info("requesting image manifest") - - image := builder.ImageFromName(imageName, imageTag) - buildResult, err := builder.BuildImage(r.Context(), h.state, &image) - - if err != nil { - writeError(w, 500, "UNKNOWN", "image build failure") - - log.WithError(err).WithFields(log.Fields{ - "image": imageName, - "tag": imageTag, - }).Error("failed to build image manifest") - - return - } - - // Some error types have special handling, which is applied - // here. - if buildResult.Error == "not_found" { - s := fmt.Sprintf("Could not find Nix packages: %v", buildResult.Pkgs) - writeError(w, 404, "MANIFEST_UNKNOWN", s) - - log.WithFields(log.Fields{ - "image": imageName, - "tag": imageTag, - "packages": buildResult.Pkgs, - }).Warn("could not find Nix packages") - - return - } - - // This marshaling error is ignored because we know that this - // field represents valid JSON data. - manifest, _ := json.Marshal(buildResult.Manifest) - w.Header().Add("Content-Type", manifestMediaType) - w.Write(manifest) - return - } - - // Serve an image layer. For this we need to first ask Nix for - // the manifest, then proceed to extract the correct layer from - // it. - layerMatches := layerRegex.FindStringSubmatch(r.RequestURI) - if len(layerMatches) == 3 { - digest := layerMatches[2] - storage := h.state.Storage - err := storage.ServeLayer(digest, r, w) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "layer": digest, - "backend": storage.Name(), - }).Error("failed to serve layer from storage backend") - } - - return - } - - log.WithField("uri", r.RequestURI).Info("unsupported registry route") - - w.WriteHeader(404) -} - -func main() { - cfg, err := config.FromEnv() - if err != nil { - log.WithError(err).Fatal("failed to load configuration") - } - - var s storage.Backend - - switch cfg.Backend { - case config.GCS: - s, err = storage.NewGCSBackend() - case config.FileSystem: - s, err = storage.NewFSBackend() - } - if err != nil { - log.WithError(err).Fatal("failed to initialise storage backend") - } - - log.WithField("backend", s.Name()).Info("initialised storage backend") - - cache, err := builder.NewCache() - if err != nil { - log.WithError(err).Fatal("failed to instantiate build cache") - } - - var pop layers.Popularity - if cfg.PopUrl != "" { - pop, err = downloadPopularity(cfg.PopUrl) - if err != nil { - log.WithError(err).WithField("popURL", cfg.PopUrl). - Fatal("failed to fetch popularity information") - } - } - - state := builder.State{ - Cache: &cache, - Cfg: cfg, - Pop: pop, - Storage: s, - } - - log.WithFields(log.Fields{ - "version": version, - "port": cfg.Port, - }).Info("starting Nixery") - - // All /v2/ requests belong to the registry handler. - http.Handle("/v2/", ®istryHandler{ - state: &state, - }) - - // All other roots are served by the static file server. - webDir := http.Dir(cfg.WebDir) - http.Handle("/", http.FileServer(webDir)) - - log.Fatal(http.ListenAndServe(":"+cfg.Port, nil)) -} diff --git a/tools/nixery/server/manifest/manifest.go b/tools/nixery/server/manifest/manifest.go deleted file mode 100644 index 0d36826fb7e5..000000000000 --- a/tools/nixery/server/manifest/manifest.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -// Package image implements logic for creating the image metadata -// (such as the image manifest and configuration). -package manifest - -import ( - "crypto/sha256" - "encoding/json" - "fmt" - "sort" -) - -const ( - // manifest constants - schemaVersion = 2 - - // media types - manifestType = "application/vnd.docker.distribution.manifest.v2+json" - layerType = "application/vnd.docker.image.rootfs.diff.tar.gzip" - configType = "application/vnd.docker.container.image.v1+json" - - // image config constants - os = "linux" - fsType = "layers" -) - -type Entry struct { - MediaType string `json:"mediaType,omitempty"` - Size int64 `json:"size"` - Digest string `json:"digest"` - - // These fields are internal to Nixery and not part of the - // serialised entry. - MergeRating uint64 `json:"-"` - TarHash string `json:",omitempty"` -} - -type manifest struct { - SchemaVersion int `json:"schemaVersion"` - MediaType string `json:"mediaType"` - Config Entry `json:"config"` - Layers []Entry `json:"layers"` -} - -type imageConfig struct { - Architecture string `json:"architecture"` - OS string `json:"os"` - - RootFS struct { - FSType string `json:"type"` - DiffIDs []string `json:"diff_ids"` - } `json:"rootfs"` - - // sic! empty struct (rather than `null`) is required by the - // image metadata deserialiser in Kubernetes - Config struct{} `json:"config"` -} - -// ConfigLayer represents the configuration layer to be included in -// the manifest, containing its JSON-serialised content and SHA256 -// hash. -type ConfigLayer struct { - Config []byte - SHA256 string -} - -// imageConfig creates an image configuration with the values set to -// the constant defaults. -// -// Outside of this module the image configuration is treated as an -// opaque blob and it is thus returned as an already serialised byte -// array and its SHA256-hash. -func configLayer(arch string, hashes []string) ConfigLayer { - c := imageConfig{} - c.Architecture = arch - c.OS = os - c.RootFS.FSType = fsType - c.RootFS.DiffIDs = hashes - - j, _ := json.Marshal(c) - - return ConfigLayer{ - Config: j, - SHA256: fmt.Sprintf("%x", sha256.Sum256(j)), - } -} - -// Manifest creates an image manifest from the specified layer entries -// and returns its JSON-serialised form as well as the configuration -// layer. -// -// Callers do not need to set the media type for the layer entries. -func Manifest(arch string, layers []Entry) (json.RawMessage, ConfigLayer) { - // Sort layers by their merge rating, from highest to lowest. - // This makes it likely for a contiguous chain of shared image - // layers to appear at the beginning of a layer. - // - // Due to moby/moby#38446 Docker considers the order of layers - // when deciding which layers to download again. - sort.Slice(layers, func(i, j int) bool { - return layers[i].MergeRating > layers[j].MergeRating - }) - - hashes := make([]string, len(layers)) - for i, l := range layers { - hashes[i] = l.TarHash - l.MediaType = layerType - l.TarHash = "" - layers[i] = l - } - - c := configLayer(arch, hashes) - - m := manifest{ - SchemaVersion: schemaVersion, - MediaType: manifestType, - Config: Entry{ - MediaType: configType, - Size: int64(len(c.Config)), - Digest: "sha256:" + c.SHA256, - }, - Layers: layers, - } - - j, _ := json.Marshal(m) - - return json.RawMessage(j), c -} diff --git a/tools/nixery/server/storage/filesystem.go b/tools/nixery/server/storage/filesystem.go deleted file mode 100644 index cdbc31c5e046..000000000000 --- a/tools/nixery/server/storage/filesystem.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -// Filesystem storage backend for Nixery. -package storage - -import ( - "context" - "fmt" - "io" - "net/http" - "os" - "path" - - log "github.com/sirupsen/logrus" -) - -type FSBackend struct { - path string -} - -func NewFSBackend() (*FSBackend, error) { - p := os.Getenv("STORAGE_PATH") - if p == "" { - return nil, fmt.Errorf("STORAGE_PATH must be set for filesystem storage") - } - - p = path.Clean(p) - err := os.MkdirAll(p, 0755) - if err != nil { - return nil, fmt.Errorf("failed to create storage dir: %s", err) - } - - return &FSBackend{p}, nil -} - -func (b *FSBackend) Name() string { - return fmt.Sprintf("Filesystem (%s)", b.path) -} - -func (b *FSBackend) Persist(ctx context.Context, key string, f Persister) (string, int64, error) { - full := path.Join(b.path, key) - dir := path.Dir(full) - err := os.MkdirAll(dir, 0755) - if err != nil { - log.WithError(err).WithField("path", dir).Error("failed to create storage directory") - return "", 0, err - } - - file, err := os.OpenFile(full, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - log.WithError(err).WithField("file", full).Error("failed to write file") - return "", 0, err - } - defer file.Close() - - return f(file) -} - -func (b *FSBackend) Fetch(ctx context.Context, key string) (io.ReadCloser, error) { - full := path.Join(b.path, key) - return os.Open(full) -} - -func (b *FSBackend) Move(ctx context.Context, old, new string) error { - newpath := path.Join(b.path, new) - err := os.MkdirAll(path.Dir(newpath), 0755) - if err != nil { - return err - } - - return os.Rename(path.Join(b.path, old), newpath) -} - -func (b *FSBackend) ServeLayer(digest string, r *http.Request, w http.ResponseWriter) error { - p := path.Join(b.path, "layers", digest) - - log.WithFields(log.Fields{ - "layer": digest, - "path": p, - }).Info("serving layer from filesystem") - - http.ServeFile(w, r, p) - return nil -} diff --git a/tools/nixery/server/storage/gcs.go b/tools/nixery/server/storage/gcs.go deleted file mode 100644 index c247cca62140..000000000000 --- a/tools/nixery/server/storage/gcs.go +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -// Google Cloud Storage backend for Nixery. -package storage - -import ( - "context" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "os" - "time" - - "cloud.google.com/go/storage" - log "github.com/sirupsen/logrus" - "golang.org/x/oauth2/google" -) - -// HTTP client to use for direct calls to APIs that are not part of the SDK -var client = &http.Client{} - -// API scope needed for renaming objects in GCS -const gcsScope = "https://www.googleapis.com/auth/devstorage.read_write" - -type GCSBackend struct { - bucket string - handle *storage.BucketHandle - signing *storage.SignedURLOptions -} - -// Constructs a new GCS bucket backend based on the configured -// environment variables. -func NewGCSBackend() (*GCSBackend, error) { - bucket := os.Getenv("GCS_BUCKET") - if bucket == "" { - return nil, fmt.Errorf("GCS_BUCKET must be configured for GCS usage") - } - - ctx := context.Background() - client, err := storage.NewClient(ctx) - if err != nil { - log.WithError(err).Fatal("failed to set up Cloud Storage client") - } - - handle := client.Bucket(bucket) - - if _, err := handle.Attrs(ctx); err != nil { - log.WithError(err).WithField("bucket", bucket).Error("could not access configured bucket") - return nil, err - } - - signing, err := signingOptsFromEnv() - if err != nil { - log.WithError(err).Error("failed to configure GCS bucket signing") - return nil, err - } - - return &GCSBackend{ - bucket: bucket, - handle: handle, - signing: signing, - }, nil -} - -func (b *GCSBackend) Name() string { - return "Google Cloud Storage (" + b.bucket + ")" -} - -func (b *GCSBackend) Persist(ctx context.Context, path string, f Persister) (string, int64, error) { - obj := b.handle.Object(path) - w := obj.NewWriter(ctx) - - hash, size, err := f(w) - if err != nil { - log.WithError(err).WithField("path", path).Error("failed to upload to GCS") - return hash, size, err - } - - return hash, size, w.Close() -} - -func (b *GCSBackend) Fetch(ctx context.Context, path string) (io.ReadCloser, error) { - obj := b.handle.Object(path) - - // Probe whether the file exists before trying to fetch it - _, err := obj.Attrs(ctx) - if err != nil { - return nil, err - } - - return obj.NewReader(ctx) -} - -// renameObject renames an object in the specified Cloud Storage -// bucket. -// -// The Go API for Cloud Storage does not support renaming objects, but -// the HTTP API does. The code below makes the relevant call manually. -func (b *GCSBackend) Move(ctx context.Context, old, new string) error { - creds, err := google.FindDefaultCredentials(ctx, gcsScope) - if err != nil { - return err - } - - token, err := creds.TokenSource.Token() - if err != nil { - return err - } - - // as per https://cloud.google.com/storage/docs/renaming-copying-moving-objects#rename - url := fmt.Sprintf( - "https://www.googleapis.com/storage/v1/b/%s/o/%s/rewriteTo/b/%s/o/%s", - url.PathEscape(b.bucket), url.PathEscape(old), - url.PathEscape(b.bucket), url.PathEscape(new), - ) - - req, err := http.NewRequest("POST", url, nil) - req.Header.Add("Authorization", "Bearer "+token.AccessToken) - _, err = client.Do(req) - if err != nil { - return err - } - - // It seems that 'rewriteTo' copies objects instead of - // renaming/moving them, hence a deletion call afterwards is - // required. - if err = b.handle.Object(old).Delete(ctx); err != nil { - log.WithError(err).WithFields(log.Fields{ - "new": new, - "old": old, - }).Warn("failed to delete renamed object") - - // this error should not break renaming and is not returned - } - - return nil -} - -func (b *GCSBackend) ServeLayer(digest string, r *http.Request, w http.ResponseWriter) error { - url, err := b.constructLayerUrl(digest) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "layer": digest, - "bucket": b.bucket, - }).Error("failed to sign GCS URL") - - return err - } - - log.WithField("layer", digest).Info("redirecting layer request to GCS bucket") - - w.Header().Set("Location", url) - w.WriteHeader(303) - return nil -} - -// Configure GCS URL signing in the presence of a service account key -// (toggled if the user has set GOOGLE_APPLICATION_CREDENTIALS). -func signingOptsFromEnv() (*storage.SignedURLOptions, error) { - path := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") - if path == "" { - // No credentials configured -> no URL signing - return nil, nil - } - - key, err := ioutil.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read service account key: %s", err) - } - - conf, err := google.JWTConfigFromJSON(key) - if err != nil { - return nil, fmt.Errorf("failed to parse service account key: %s", err) - } - - log.WithField("account", conf.Email).Info("GCS URL signing enabled") - - return &storage.SignedURLOptions{ - Scheme: storage.SigningSchemeV4, - GoogleAccessID: conf.Email, - PrivateKey: conf.PrivateKey, - Method: "GET", - }, nil -} - -// layerRedirect constructs the public URL of the layer object in the Cloud -// Storage bucket, signs it and redirects the user there. -// -// Signing the URL allows unauthenticated clients to retrieve objects from the -// bucket. -// -// The Docker client is known to follow redirects, but this might not be true -// for all other registry clients. -func (b *GCSBackend) constructLayerUrl(digest string) (string, error) { - log.WithField("layer", digest).Info("redirecting layer request to bucket") - object := "layers/" + digest - - if b.signing != nil { - opts := *b.signing - opts.Expires = time.Now().Add(5 * time.Minute) - return storage.SignedURL(b.bucket, object, &opts) - } else { - return ("https://storage.googleapis.com/" + b.bucket + "/" + object), nil - } -} diff --git a/tools/nixery/server/storage/storage.go b/tools/nixery/server/storage/storage.go deleted file mode 100644 index c97b5e4facc6..000000000000 --- a/tools/nixery/server/storage/storage.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -// Package storage implements an interface that can be implemented by -// storage backends, such as Google Cloud Storage or the local -// filesystem. -package storage - -import ( - "context" - "io" - "net/http" -) - -type Persister = func(io.Writer) (string, int64, error) - -type Backend interface { - // Name returns the name of the storage backend, for use in - // log messages and such. - Name() string - - // Persist provides a user-supplied function with a writer - // that stores data in the storage backend. - // - // It needs to return the SHA256 hash of the data written as - // well as the total number of bytes, as those are required - // for the image manifest. - Persist(context.Context, string, Persister) (string, int64, error) - - // Fetch retrieves data from the storage backend. - Fetch(ctx context.Context, path string) (io.ReadCloser, error) - - // Move renames a path inside the storage backend. This is - // used for staging uploads while calculating their hashes. - Move(ctx context.Context, old, new string) error - - // Serve provides a handler function to serve HTTP requests - // for layers in the storage backend. - ServeLayer(digest string, r *http.Request, w http.ResponseWriter) error -} diff --git a/tools/nixery/shell.nix b/tools/nixery/shell.nix index 93cd1f4cec62..b37caa83ade3 100644 --- a/tools/nixery/shell.nix +++ b/tools/nixery/shell.nix @@ -20,5 +20,5 @@ let nixery = import ./default.nix { inherit pkgs; }; in pkgs.stdenv.mkDerivation { name = "nixery-dev-shell"; - buildInputs = with pkgs; [ jq nixery.nixery-build-image ]; + buildInputs = with pkgs; [ jq nixery.nixery-prepare-image ]; } diff --git a/tools/nixery/storage/filesystem.go b/tools/nixery/storage/filesystem.go new file mode 100644 index 000000000000..cdbc31c5e046 --- /dev/null +++ b/tools/nixery/storage/filesystem.go @@ -0,0 +1,96 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +// Filesystem storage backend for Nixery. +package storage + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path" + + log "github.com/sirupsen/logrus" +) + +type FSBackend struct { + path string +} + +func NewFSBackend() (*FSBackend, error) { + p := os.Getenv("STORAGE_PATH") + if p == "" { + return nil, fmt.Errorf("STORAGE_PATH must be set for filesystem storage") + } + + p = path.Clean(p) + err := os.MkdirAll(p, 0755) + if err != nil { + return nil, fmt.Errorf("failed to create storage dir: %s", err) + } + + return &FSBackend{p}, nil +} + +func (b *FSBackend) Name() string { + return fmt.Sprintf("Filesystem (%s)", b.path) +} + +func (b *FSBackend) Persist(ctx context.Context, key string, f Persister) (string, int64, error) { + full := path.Join(b.path, key) + dir := path.Dir(full) + err := os.MkdirAll(dir, 0755) + if err != nil { + log.WithError(err).WithField("path", dir).Error("failed to create storage directory") + return "", 0, err + } + + file, err := os.OpenFile(full, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + log.WithError(err).WithField("file", full).Error("failed to write file") + return "", 0, err + } + defer file.Close() + + return f(file) +} + +func (b *FSBackend) Fetch(ctx context.Context, key string) (io.ReadCloser, error) { + full := path.Join(b.path, key) + return os.Open(full) +} + +func (b *FSBackend) Move(ctx context.Context, old, new string) error { + newpath := path.Join(b.path, new) + err := os.MkdirAll(path.Dir(newpath), 0755) + if err != nil { + return err + } + + return os.Rename(path.Join(b.path, old), newpath) +} + +func (b *FSBackend) ServeLayer(digest string, r *http.Request, w http.ResponseWriter) error { + p := path.Join(b.path, "layers", digest) + + log.WithFields(log.Fields{ + "layer": digest, + "path": p, + }).Info("serving layer from filesystem") + + http.ServeFile(w, r, p) + return nil +} diff --git a/tools/nixery/storage/gcs.go b/tools/nixery/storage/gcs.go new file mode 100644 index 000000000000..c247cca62140 --- /dev/null +++ b/tools/nixery/storage/gcs.go @@ -0,0 +1,219 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +// Google Cloud Storage backend for Nixery. +package storage + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "time" + + "cloud.google.com/go/storage" + log "github.com/sirupsen/logrus" + "golang.org/x/oauth2/google" +) + +// HTTP client to use for direct calls to APIs that are not part of the SDK +var client = &http.Client{} + +// API scope needed for renaming objects in GCS +const gcsScope = "https://www.googleapis.com/auth/devstorage.read_write" + +type GCSBackend struct { + bucket string + handle *storage.BucketHandle + signing *storage.SignedURLOptions +} + +// Constructs a new GCS bucket backend based on the configured +// environment variables. +func NewGCSBackend() (*GCSBackend, error) { + bucket := os.Getenv("GCS_BUCKET") + if bucket == "" { + return nil, fmt.Errorf("GCS_BUCKET must be configured for GCS usage") + } + + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + log.WithError(err).Fatal("failed to set up Cloud Storage client") + } + + handle := client.Bucket(bucket) + + if _, err := handle.Attrs(ctx); err != nil { + log.WithError(err).WithField("bucket", bucket).Error("could not access configured bucket") + return nil, err + } + + signing, err := signingOptsFromEnv() + if err != nil { + log.WithError(err).Error("failed to configure GCS bucket signing") + return nil, err + } + + return &GCSBackend{ + bucket: bucket, + handle: handle, + signing: signing, + }, nil +} + +func (b *GCSBackend) Name() string { + return "Google Cloud Storage (" + b.bucket + ")" +} + +func (b *GCSBackend) Persist(ctx context.Context, path string, f Persister) (string, int64, error) { + obj := b.handle.Object(path) + w := obj.NewWriter(ctx) + + hash, size, err := f(w) + if err != nil { + log.WithError(err).WithField("path", path).Error("failed to upload to GCS") + return hash, size, err + } + + return hash, size, w.Close() +} + +func (b *GCSBackend) Fetch(ctx context.Context, path string) (io.ReadCloser, error) { + obj := b.handle.Object(path) + + // Probe whether the file exists before trying to fetch it + _, err := obj.Attrs(ctx) + if err != nil { + return nil, err + } + + return obj.NewReader(ctx) +} + +// renameObject renames an object in the specified Cloud Storage +// bucket. +// +// The Go API for Cloud Storage does not support renaming objects, but +// the HTTP API does. The code below makes the relevant call manually. +func (b *GCSBackend) Move(ctx context.Context, old, new string) error { + creds, err := google.FindDefaultCredentials(ctx, gcsScope) + if err != nil { + return err + } + + token, err := creds.TokenSource.Token() + if err != nil { + return err + } + + // as per https://cloud.google.com/storage/docs/renaming-copying-moving-objects#rename + url := fmt.Sprintf( + "https://www.googleapis.com/storage/v1/b/%s/o/%s/rewriteTo/b/%s/o/%s", + url.PathEscape(b.bucket), url.PathEscape(old), + url.PathEscape(b.bucket), url.PathEscape(new), + ) + + req, err := http.NewRequest("POST", url, nil) + req.Header.Add("Authorization", "Bearer "+token.AccessToken) + _, err = client.Do(req) + if err != nil { + return err + } + + // It seems that 'rewriteTo' copies objects instead of + // renaming/moving them, hence a deletion call afterwards is + // required. + if err = b.handle.Object(old).Delete(ctx); err != nil { + log.WithError(err).WithFields(log.Fields{ + "new": new, + "old": old, + }).Warn("failed to delete renamed object") + + // this error should not break renaming and is not returned + } + + return nil +} + +func (b *GCSBackend) ServeLayer(digest string, r *http.Request, w http.ResponseWriter) error { + url, err := b.constructLayerUrl(digest) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "layer": digest, + "bucket": b.bucket, + }).Error("failed to sign GCS URL") + + return err + } + + log.WithField("layer", digest).Info("redirecting layer request to GCS bucket") + + w.Header().Set("Location", url) + w.WriteHeader(303) + return nil +} + +// Configure GCS URL signing in the presence of a service account key +// (toggled if the user has set GOOGLE_APPLICATION_CREDENTIALS). +func signingOptsFromEnv() (*storage.SignedURLOptions, error) { + path := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") + if path == "" { + // No credentials configured -> no URL signing + return nil, nil + } + + key, err := ioutil.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read service account key: %s", err) + } + + conf, err := google.JWTConfigFromJSON(key) + if err != nil { + return nil, fmt.Errorf("failed to parse service account key: %s", err) + } + + log.WithField("account", conf.Email).Info("GCS URL signing enabled") + + return &storage.SignedURLOptions{ + Scheme: storage.SigningSchemeV4, + GoogleAccessID: conf.Email, + PrivateKey: conf.PrivateKey, + Method: "GET", + }, nil +} + +// layerRedirect constructs the public URL of the layer object in the Cloud +// Storage bucket, signs it and redirects the user there. +// +// Signing the URL allows unauthenticated clients to retrieve objects from the +// bucket. +// +// The Docker client is known to follow redirects, but this might not be true +// for all other registry clients. +func (b *GCSBackend) constructLayerUrl(digest string) (string, error) { + log.WithField("layer", digest).Info("redirecting layer request to bucket") + object := "layers/" + digest + + if b.signing != nil { + opts := *b.signing + opts.Expires = time.Now().Add(5 * time.Minute) + return storage.SignedURL(b.bucket, object, &opts) + } else { + return ("https://storage.googleapis.com/" + b.bucket + "/" + object), nil + } +} diff --git a/tools/nixery/storage/storage.go b/tools/nixery/storage/storage.go new file mode 100644 index 000000000000..c97b5e4facc6 --- /dev/null +++ b/tools/nixery/storage/storage.go @@ -0,0 +1,51 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +// Package storage implements an interface that can be implemented by +// storage backends, such as Google Cloud Storage or the local +// filesystem. +package storage + +import ( + "context" + "io" + "net/http" +) + +type Persister = func(io.Writer) (string, int64, error) + +type Backend interface { + // Name returns the name of the storage backend, for use in + // log messages and such. + Name() string + + // Persist provides a user-supplied function with a writer + // that stores data in the storage backend. + // + // It needs to return the SHA256 hash of the data written as + // well as the total number of bytes, as those are required + // for the image manifest. + Persist(context.Context, string, Persister) (string, int64, error) + + // Fetch retrieves data from the storage backend. + Fetch(ctx context.Context, path string) (io.ReadCloser, error) + + // Move renames a path inside the storage backend. This is + // used for staging uploads while calculating their hashes. + Move(ctx context.Context, old, new string) error + + // Serve provides a handler function to serve HTTP requests + // for layers in the storage backend. + ServeLayer(digest string, r *http.Request, w http.ResponseWriter) error +} -- cgit 1.4.1