feat(tools/nixery): Absorb Nixery into depot r/3978

This absorbs a josh-filtered Nix subtree into depot, at
//tools/nixery.

This subtree was created through `josh-filter ':prefix=tools/nixery'`,
which allows a filter on tools/nixery to yield the same commit hashes
as the original Nixery repository (allowing for history continuity).

Change-Id: Icc1a99bf1248226b91f437b0a90361d36fb0d327

3 files changed, 261 insertions, 0 deletions

diff --git a/tools/nixery/prepare-image/default.nix b/tools/nixery/prepare-image/default.nix
new file mode 100644
index 000000000000..60b208f522d5
--- /dev/null
+++ b/tools/nixery/prepare-image/default.nix
@@ -0,0 +1,29 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file builds a wrapper script called by Nixery to ask for the
+# content information for a given image.
+#
+# The purpose of using a wrapper script is to ensure that the paths to
+# all required Nix files are set correctly at runtime.
+
+{ pkgs ? import <nixpkgs> {} }:
+
+pkgs.writeShellScriptBin "nixery-prepare-image" ''
+  exec ${pkgs.nix}/bin/nix-build \
+    --show-trace \
+    --no-out-link "$@" \
+    --argstr loadPkgs ${./load-pkgs.nix} \
+    ${./prepare-image.nix}
+''
diff --git a/tools/nixery/prepare-image/load-pkgs.nix b/tools/nixery/prepare-image/load-pkgs.nix
new file mode 100644
index 000000000000..4a89dcde3a14
--- /dev/null
+++ b/tools/nixery/prepare-image/load-pkgs.nix
@@ -0,0 +1,45 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Load a Nix package set from one of the supported source types
+# (nixpkgs, git, path).
+{ srcType, srcArgs, importArgs ? { } }:
+
+with builtins;
+let
+  # If a nixpkgs channel is requested, it is retrieved from Github (as
+  # a tarball) and imported.
+  fetchImportChannel = channel:
+    let
+      url =
+        "https://github.com/NixOS/nixpkgs/archive/${channel}.tar.gz";
+    in import (fetchTarball url) importArgs;
+
+  # If a git repository is requested, it is retrieved via
+  # builtins.fetchGit which defaults to the git configuration of the
+  # outside environment. This means that user-configured SSH
+  # credentials etc. are going to work as expected.
+  fetchImportGit = spec: import (fetchGit spec) importArgs;
+
+  # No special handling is used for paths, so users are expected to pass one
+  # that will work natively with Nix.
+  importPath = path: import (toPath path) importArgs;
+in if srcType == "nixpkgs" then
+  fetchImportChannel srcArgs
+else if srcType == "git" then
+  fetchImportGit (fromJSON srcArgs)
+else if srcType == "path" then
+  importPath srcArgs
+else
+  throw ("Invalid package set source specification: ${srcType} (${srcArgs})")
diff --git a/tools/nixery/prepare-image/prepare-image.nix b/tools/nixery/prepare-image/prepare-image.nix
new file mode 100644
index 000000000000..acd1430548b2
--- /dev/null
+++ b/tools/nixery/prepare-image/prepare-image.nix
@@ -0,0 +1,187 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file contains a derivation that outputs structured information
+# about the runtime dependencies of an image with a given set of
+# packages. This is used by Nixery to determine the layer grouping and
+# assemble each layer.
+#
+# In addition it creates and outputs a meta-layer with the symlink
+# structure required for using the image together with the individual
+# package layers.
+
+{
+  # Description of the package set to be used (will be loaded by load-pkgs.nix)
+  srcType ? "nixpkgs",
+  srcArgs ? "nixos-20.09",
+  system ? "x86_64-linux",
+  importArgs ? { },
+  # Path to load-pkgs.nix
+  loadPkgs ? ./load-pkgs.nix,
+  # Packages to install by name (which must refer to top-level attributes of
+  # nixpkgs). This is passed in as a JSON-array in string form.
+  packages ? "[]"
+}:
+
+let
+  inherit (builtins)
+    foldl'
+    fromJSON
+    hasAttr
+    length
+    match
+    readFile
+    toFile
+    toJSON;
+
+  # Package set to use for sourcing utilities
+  nativePkgs = import loadPkgs { inherit srcType srcArgs importArgs; };
+  inherit (nativePkgs) coreutils jq openssl lib runCommand writeText symlinkJoin;
+
+  # Package set to use for packages to be included in the image. This
+  # package set is imported with the system set to the target
+  # architecture.
+  pkgs = import loadPkgs {
+    inherit srcType srcArgs;
+    importArgs = importArgs // {
+      inherit system;
+    };
+  };
+
+  # deepFetch traverses the top-level Nix package set to retrieve an item via a
+  # path specified in string form.
+  #
+  # For top-level items, the name of the key yields the result directly. Nested
+  # items are fetched by using dot-syntax, as in Nix itself.
+  #
+  # Due to a restriction of the registry API specification it is not possible to
+  # pass uppercase characters in an image name, however the Nix package set
+  # makes use of camelCasing repeatedly (for example for `haskellPackages`).
+  #
+  # To work around this, if no value is found on the top-level a second lookup
+  # is done on the package set using lowercase-names. This is not done for
+  # nested sets, as they often have keys that only differ in case.
+  #
+  # For example, `deepFetch pkgs "xorg.xev"` retrieves `pkgs.xorg.xev` and
+  # `deepFetch haskellpackages.stylish-haskell` retrieves
+  # `haskellPackages.stylish-haskell`.
+  deepFetch = with lib; s: n:
+    let path = splitString "." n;
+        err = { error = "not_found"; pkg = n; };
+        # The most efficient way I've found to do a lookup against
+        # case-differing versions of an attribute is to first construct a
+        # mapping of all lowercased attribute names to their differently cased
+        # equivalents.
+        #
+        # This map is then used for a second lookup if the top-level
+        # (case-sensitive) one does not yield a result.
+        hasUpper = str: (match ".*[A-Z].*" str) != null;
+        allUpperKeys = filter hasUpper (attrNames s);
+        lowercased = listToAttrs (map (k: {
+          name = toLower k;
+          value = k;
+          }) allUpperKeys);
+        caseAmendedPath = map (v: if hasAttr v lowercased then lowercased."${v}" else v) path;
+        fetchLower = attrByPath caseAmendedPath err s;
+    in attrByPath path fetchLower s;
+
+  # allContents contains all packages successfully retrieved by name
+  # from the package set, as well as any errors encountered while
+  # attempting to fetch a package.
+  #
+  # Accumulated error information is returned back to the server.
+  allContents =
+    # Folds over the results of 'deepFetch' on all requested packages to
+    # separate them into errors and content. This allows the program to
+    # terminate early and return only the errors if any are encountered.
+    let splitter = attrs: res:
+          if hasAttr "error" res
+          then attrs // { errors = attrs.errors ++ [ res ]; }
+          else attrs // { contents = attrs.contents ++ [ res ]; };
+        init = { contents = []; errors = []; };
+        fetched = (map (deepFetch pkgs) (fromJSON packages));
+    in foldl' splitter init fetched;
+
+  # Contains the export references graph of all retrieved packages,
+  # which has information about all runtime dependencies of the image.
+  #
+  # This is used by Nixery to group closures into image layers.
+  runtimeGraph = runCommand "runtime-graph.json" {
+    __structuredAttrs = true;
+    exportReferencesGraph.graph = allContents.contents;
+    PATH = "${coreutils}/bin";
+    builder = toFile "builder" ''
+      . .attrs.sh
+      cp .attrs.json ''${outputs[out]}
+    '';
+  } "";
+
+  # Create a symlink forest into all top-level store paths of the
+  # image contents.
+  contentsEnv = symlinkJoin {
+    name = "bulk-layers";
+    paths = allContents.contents;
+
+    # Provide a few essentials that many programs expect:
+    # - a /tmp directory,
+    # - a /usr/bin/env for shell scripts that require it.
+    #
+    # Note that in images that do not actually contain `coreutils`,
+    # /usr/bin/env will be a dangling symlink.
+    #
+    # TODO(tazjin): Don't link /usr/bin/env if coreutils is not included.
+    postBuild = ''
+      mkdir -p $out/tmp
+      mkdir -p $out/usr/bin
+      ln -s ${coreutils}/bin/env $out/usr/bin/env
+    '';
+  };
+
+  # Image layer that contains the symlink forest created above. This
+  # must be included in the image to ensure that the filesystem has a
+  # useful layout at runtime.
+  symlinkLayer = runCommand "symlink-layer.tar" {} ''
+    cp -r ${contentsEnv}/ ./layer
+    tar --transform='s|^\./||' -C layer --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 -cf $out .
+  '';
+
+  # Metadata about the symlink layer which is required for serving it.
+  # Two different hashes are computed for different usages (inclusion
+  # in manifest vs. content-checking in the layer cache).
+  symlinkLayerMeta = fromJSON (readFile (runCommand "symlink-layer-meta.json" {
+    buildInputs = [ coreutils jq openssl ];
+  }''
+    tarHash=$(sha256sum ${symlinkLayer} | cut -d ' ' -f1)
+    layerSize=$(stat --printf '%s' ${symlinkLayer})
+
+    jq -n -c --arg tarHash $tarHash --arg size $layerSize --arg path ${symlinkLayer} \
+      '{ size: ($size | tonumber), tarHash: $tarHash, path: $path }' >> $out
+  ''));
+
+  # Final output structure returned to Nixery if the build succeeded
+  buildOutput = {
+    runtimeGraph = fromJSON (readFile runtimeGraph);
+    symlinkLayer = symlinkLayerMeta;
+  };
+
+  # Output structure returned if errors occured during the build. Currently the
+  # only error type that is returned in a structured way is 'not_found'.
+  errorOutput = {
+    error = "not_found";
+    pkgs = map (err: err.pkg) allContents.errors;
+  };
+in writeText "build-output.json" (if (length allContents.errors) == 0
+  then toJSON buildOutput
+  else toJSON errorOutput
+)