feat(build-image): Introduce a terrifying hack to build group-layers

The issue is described in detail in a comment in
`build-image/default.nix`, please read it.

3 files changed, 86 insertions, 13 deletions

diff --git a/tools/nixery/build-image/build-image.nix b/tools/nixery/build-image/build-image.nix
index 37156905fa38..e5b195a63d11 100644
--- a/tools/nixery/build-image/build-image.nix
+++ b/tools/nixery/build-image/build-image.nix
@@ -22,6 +22,8 @@
   name,
   # Image tag, the Nix's output hash will be used if null
   tag ? null,
+  # Tool used to determine layer grouping
+  groupLayers,
   # Files to put on the image (a nix store path or list of paths).
   contents ? [],
   # Packages to install by name (which must refer to top-level attributes of
@@ -48,7 +50,8 @@
   # '!' was chosen as the separator because `builtins.split` does not
   # support regex escapes and there are few other candidates. It
   # doesn't matter much because this is invoked by the server.
-  pkgSource ? "nixpkgs!nixos-19.03"
+  pkgSource ? "nixpkgs!nixos-19.03",
+  ...
 }:
 
 let
@@ -165,6 +168,25 @@ let
     paths = allContents.contents;
   };
 
+  # Before actually creating any image layers, the store paths that need to be
+  # included in the image must be sorted into the layers that they should go
+  # into.
+  #
+  # How contents are allocated to each layer is decided by the `group-layers.go`
+  # program. The mechanism used is described at the top of the program's source
+  # code, or alternatively in the layering design document:
+  #
+  #   https://storage.googleapis.com/nixdoc/nixery-layers.html
+  #
+  # To invoke the program, a graph of all runtime references is created via
+  # Nix's exportReferencesGraph feature - the resulting layers are read back
+  # into Nix using import-from-derivation.
+  groupedLayers = runCommand "grouped-layers.json" {
+    buildInputs = [ groupLayers ];
+  } ''
+    group-layers --fnorg
+  '';
+
   # The image build infrastructure expects to be outputting a slightly different
   # format than the one we serve over the registry protocol. To work around its
   # expectations we need to provide an empty JSON file that it can write some
@@ -287,6 +309,6 @@ let
     pkgs = map (err: err.pkg) allContents.errors;
   };
 in writeText "manifest-output.json" (if (length allContents.errors) == 0
-  then toJSON manifestOutput
+  then toJSON groupedLayers # manifestOutput
   else toJSON errorOutput
 )
diff --git a/tools/nixery/build-image/default.nix b/tools/nixery/build-image/default.nix
index 4962e07deee9..cff403995884 100644
--- a/tools/nixery/build-image/default.nix
+++ b/tools/nixery/build-image/default.nix
@@ -16,25 +16,76 @@
 # moves the files needed to call the Nix builds at runtime in the
 # correct locations.
 
-{ buildGoPackage, lib, nix, writeShellScriptBin }:
+{ pkgs ? import <nixpkgs> { }, self ? ./.
 
-let
-  group-layers = buildGoPackage {
+  # Because of the insanity occuring below, this function must mirror
+  # all arguments of build-image.nix.
+, tag ? null, name ? null, packages ? null, maxLayers ? null, pkgSource ? null
+}@args:
+
+with pkgs; rec {
+  groupLayers = buildGoPackage {
     name = "group-layers";
     goDeps = ./go-deps.nix;
-    src = ./.;
-
     goPackagePath = "github.com/google/nixery/group-layers";
 
+    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+    #                    WARNING: HERE BE DRAGONS!                    #
+    #                                                                 #
+    # The hack below is used to break evaluation purity. The issue is #
+    # that Nixery's build instructions (the default.nix in the folder #
+    # above this one) must build a program that can invoke Nix at     #
+    # runtime, with a derivation that needs a program tracked in this #
+    # source tree (`group-layers`).                                   #
+    #                                                                 #
+    # Simply installing that program in the $PATH of Nixery does not  #
+    # work, because the runtime Nix builds use their own isolated     #
+    # environment.                                                    #
+    #                                                                 #
+    # I first attempted to naively copy the sources into the Nix      #
+    # store, so that Nixery could build `group-layers` when it starts #
+    # up - however those sources are not available to a nested Nix    #
+    # build because they're not part of the context of the nested     #
+    # invocation.                                                     #
+    #                                                                 #
+    # Nix has several primitives under `builtins.` that can break     #
+    # evaluation purity, these (namely readDir and readFile) are used #
+    # below to break out of the isolated environment and reconstruct  #
+    # the source tree for `group-layers`.                             #
+    #                                                                 #
+    # There might be a better way to do this, but I don't know what   #
+    # it is.                                                          #
+    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+    src = runCommand "group-layers-srcs" { } ''
+      mkdir -p $out
+      ${with builtins;
+      let
+        files =
+          (attrNames (lib.filterAttrs (_: t: t != "symlink") (readDir self)));
+        commands =
+          map (f: "cp ${toFile f (readFile "${self}/${f}")} $out/${f}") files;
+      in lib.concatStringsSep "\n" commands}
+    '';
+
     meta = {
-      description = "Tool to group a set of packages into container image layers";
+      description =
+        "Tool to group a set of packages into container image layers";
       license = lib.licenses.asl20;
       maintainers = [ lib.maintainers.tazjin ];
     };
   };
 
+  buildImage = import ./build-image.nix
+    ({ inherit groupLayers; } // (lib.filterAttrs (_: v: v != null) args));
+
   # Wrapper script which is called by the Nixery server to trigger an
-  # actual image build.
-in writeShellScriptBin "nixery-build-image" ''
-  exec ${nix}/bin/nix-build --show-trace --no-out-link "$@" ${./build-image.nix}
-''
+  # actual image build. This exists to avoid having to specify the
+  # location of build-image.nix at runtime.
+  wrapper = writeShellScriptBin "nixery-build-image" ''
+    exec ${nix}/bin/nix-build \
+      --show-trace \
+      --no-out-link "$@" \
+      --argstr self "${./.}" \
+      -A buildImage ${./.}
+  '';
+}
diff --git a/tools/nixery/default.nix b/tools/nixery/default.nix
index 7d201869dc90..926ab0d19f6b 100644
--- a/tools/nixery/default.nix
+++ b/tools/nixery/default.nix
@@ -26,7 +26,7 @@ rec {
   nixery-server = callPackage ./server {};
 
   # Implementation of the image building & layering logic
-  nixery-build-image = callPackage ./build-image {};
+  nixery-build-image = (import ./build-image { inherit pkgs; }).wrapper;
 
   # Use mdBook to build a static asset page which Nixery can then
   # serve. This is primarily used for the public instance at