diff options
Diffstat (limited to 'tools/nixery')
| -rw-r--r-- | tools/nixery/build-image/build-image.nix | 26 | ||||
| -rw-r--r-- | tools/nixery/build-image/default.nix | 71 | ||||
| -rw-r--r-- | tools/nixery/default.nix | 2 |
3 files changed, 86 insertions, 13 deletions
diff --git a/tools/nixery/build-image/build-image.nix b/tools/nixery/build-image/build-image.nix index 37156905fa38..e5b195a63d11 100644 --- a/tools/nixery/build-image/build-image.nix +++ b/tools/nixery/build-image/build-image.nix @@ -22,6 +22,8 @@ name, # Image tag, the Nix's output hash will be used if null tag ? null, + # Tool used to determine layer grouping + groupLayers, # Files to put on the image (a nix store path or list of paths). contents ? [], # Packages to install by name (which must refer to top-level attributes of @@ -48,7 +50,8 @@ # '!' was chosen as the separator because `builtins.split` does not # support regex escapes and there are few other candidates. It # doesn't matter much because this is invoked by the server. - pkgSource ? "nixpkgs!nixos-19.03" + pkgSource ? "nixpkgs!nixos-19.03", + ... }: let @@ -165,6 +168,25 @@ let paths = allContents.contents; }; + # Before actually creating any image layers, the store paths that need to be + # included in the image must be sorted into the layers that they should go + # into. + # + # How contents are allocated to each layer is decided by the `group-layers.go` + # program. The mechanism used is described at the top of the program's source + # code, or alternatively in the layering design document: + # + # https://storage.googleapis.com/nixdoc/nixery-layers.html + # + # To invoke the program, a graph of all runtime references is created via + # Nix's exportReferencesGraph feature - the resulting layers are read back + # into Nix using import-from-derivation. + groupedLayers = runCommand "grouped-layers.json" { + buildInputs = [ groupLayers ]; + } '' + group-layers --fnorg + ''; + # The image build infrastructure expects to be outputting a slightly different # format than the one we serve over the registry protocol. To work around its # expectations we need to provide an empty JSON file that it can write some @@ -287,6 +309,6 @@ let pkgs = map (err: err.pkg) allContents.errors; }; in writeText "manifest-output.json" (if (length allContents.errors) == 0 - then toJSON manifestOutput + then toJSON groupedLayers # manifestOutput else toJSON errorOutput ) diff --git a/tools/nixery/build-image/default.nix b/tools/nixery/build-image/default.nix index 4962e07deee9..cff403995884 100644 --- a/tools/nixery/build-image/default.nix +++ b/tools/nixery/build-image/default.nix @@ -16,25 +16,76 @@ # moves the files needed to call the Nix builds at runtime in the # correct locations. -{ buildGoPackage, lib, nix, writeShellScriptBin }: +{ pkgs ? import <nixpkgs> { }, self ? ./. -let - group-layers = buildGoPackage { + # Because of the insanity occuring below, this function must mirror + # all arguments of build-image.nix. +, tag ? null, name ? null, packages ? null, maxLayers ? null, pkgSource ? null +}@args: + +with pkgs; rec { + groupLayers = buildGoPackage { name = "group-layers"; goDeps = ./go-deps.nix; - src = ./.; - goPackagePath = "github.com/google/nixery/group-layers"; + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + # WARNING: HERE BE DRAGONS! # + # # + # The hack below is used to break evaluation purity. The issue is # + # that Nixery's build instructions (the default.nix in the folder # + # above this one) must build a program that can invoke Nix at # + # runtime, with a derivation that needs a program tracked in this # + # source tree (`group-layers`). # + # # + # Simply installing that program in the $PATH of Nixery does not # + # work, because the runtime Nix builds use their own isolated # + # environment. # + # # + # I first attempted to naively copy the sources into the Nix # + # store, so that Nixery could build `group-layers` when it starts # + # up - however those sources are not available to a nested Nix # + # build because they're not part of the context of the nested # + # invocation. # + # # + # Nix has several primitives under `builtins.` that can break # + # evaluation purity, these (namely readDir and readFile) are used # + # below to break out of the isolated environment and reconstruct # + # the source tree for `group-layers`. # + # # + # There might be a better way to do this, but I don't know what # + # it is. # + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + src = runCommand "group-layers-srcs" { } '' + mkdir -p $out + ${with builtins; + let + files = + (attrNames (lib.filterAttrs (_: t: t != "symlink") (readDir self))); + commands = + map (f: "cp ${toFile f (readFile "${self}/${f}")} $out/${f}") files; + in lib.concatStringsSep "\n" commands} + ''; + meta = { - description = "Tool to group a set of packages into container image layers"; + description = + "Tool to group a set of packages into container image layers"; license = lib.licenses.asl20; maintainers = [ lib.maintainers.tazjin ]; }; }; + buildImage = import ./build-image.nix + ({ inherit groupLayers; } // (lib.filterAttrs (_: v: v != null) args)); + # Wrapper script which is called by the Nixery server to trigger an - # actual image build. -in writeShellScriptBin "nixery-build-image" '' - exec ${nix}/bin/nix-build --show-trace --no-out-link "$@" ${./build-image.nix} -'' + # actual image build. This exists to avoid having to specify the + # location of build-image.nix at runtime. + wrapper = writeShellScriptBin "nixery-build-image" '' + exec ${nix}/bin/nix-build \ + --show-trace \ + --no-out-link "$@" \ + --argstr self "${./.}" \ + -A buildImage ${./.} + ''; +} diff --git a/tools/nixery/default.nix b/tools/nixery/default.nix index 7d201869dc90..926ab0d19f6b 100644 --- a/tools/nixery/default.nix +++ b/tools/nixery/default.nix @@ -26,7 +26,7 @@ rec { nixery-server = callPackage ./server {}; # Implementation of the image building & layering logic - nixery-build-image = callPackage ./build-image {}; + nixery-build-image = (import ./build-image { inherit pkgs; }).wrapper; # Use mdBook to build a static asset page which Nixery can then # serve. This is primarily used for the public instance at |