21 files changed, 83 insertions, 114 deletions

diff --git a/tools/nixery/server/builder/archive.go b/tools/nixery/builder/archive.go
index e0fb76d44bee..ff822e389a7d 100644
--- a/tools/nixery/server/builder/archive.go
+++ b/tools/nixery/builder/archive.go
@@ -19,7 +19,6 @@ package builder
 // The tarball is written straight to the supplied reader, which makes it
 // possible to create an image layer from the specified store paths, hash it and
 // upload it in one reading pass.
-
 import (
 	"archive/tar"
 	"compress/gzip"
@@ -28,8 +27,6 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-
-	"github.com/google/nixery/server/layers"
 )
 
 // Create a new compressed tarball from each of the paths in the list
@@ -37,7 +34,7 @@ import (
 //
 // The uncompressed tarball is hashed because image manifests must
 // contain both the hashes of compressed and uncompressed layers.
-func packStorePaths(l *layers.Layer, w io.Writer) (string, error) {
+func packStorePaths(l *layer, w io.Writer) (string, error) {
 	shasum := sha256.New()
 	gz := gzip.NewWriter(w)
 	multi := io.MultiWriter(shasum, gz)
diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/builder/builder.go
index da9dede1acd7..ceb112df90cf 100644
--- a/tools/nixery/server/builder/builder.go
+++ b/tools/nixery/builder/builder.go
@@ -12,9 +12,10 @@
 // License for the specific language governing permissions and limitations under
 // the License.
 
-// Package builder implements the code required to build images via Nix. Image
-// build data is cached for up to 24 hours to avoid duplicated calls to Nix
-// (which are costly even if no building is performed).
+// Package builder implements the logic for assembling container
+// images. It shells out to Nix to retrieve all required Nix-packages
+// and assemble the symlink layer and then creates the required
+// tarballs in-process.
 package builder
 
 import (
@@ -32,10 +33,9 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/google/nixery/server/config"
-	"github.com/google/nixery/server/layers"
-	"github.com/google/nixery/server/manifest"
-	"github.com/google/nixery/server/storage"
+	"github.com/google/nixery/config"
+	"github.com/google/nixery/manifest"
+	"github.com/google/nixery/storage"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -50,7 +50,7 @@ type State struct {
 	Storage storage.Backend
 	Cache   *LocalCache
 	Cfg     config.Config
-	Pop     layers.Popularity
+	Pop     Popularity
 }
 
 // Architecture represents the possible CPU architectures for which
@@ -128,7 +128,7 @@ type ImageResult struct {
 	Pkgs  []string `json:"pkgs"`
 
 	// These fields are populated in case of success
-	Graph        layers.RuntimeGraph `json:"runtimeGraph"`
+	Graph        runtimeGraph `json:"runtimeGraph"`
 	SymlinkLayer struct {
 		Size    int    `json:"size"`
 		TarHash string `json:"tarHash"`
@@ -265,7 +265,7 @@ func prepareImage(s *State, image *Image) (*ImageResult, error) {
 		"--argstr", "system", image.Arch.nixSystem,
 	}
 
-	output, err := callNix("nixery-build-image", image.Name, args)
+	output, err := callNix("nixery-prepare-image", image.Name, args)
 	if err != nil {
 		// granular error logging is performed in callNix already
 		return nil, err
@@ -292,7 +292,7 @@ func prepareImage(s *State, image *Image) (*ImageResult, error) {
 // added only after successful uploads, which guarantees that entries
 // retrieved from the cache are present in the bucket.
 func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageResult) ([]manifest.Entry, error) {
-	grouped := layers.Group(&result.Graph, &s.Pop, LayerBudget)
+	grouped := groupLayers(&result.Graph, &s.Pop, LayerBudget)
 
 	var entries []manifest.Entry
 
@@ -329,7 +329,7 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes
 
 			var pkgs []string
 			for _, p := range l.Contents {
-				pkgs = append(pkgs, layers.PackageFromPath(p))
+				pkgs = append(pkgs, packageFromPath(p))
 			}
 
 			log.WithFields(log.Fields{
diff --git a/tools/nixery/server/builder/builder_test.go b/tools/nixery/builder/builder_test.go
index 3fbe2ab40e23..3fbe2ab40e23 100644
--- a/tools/nixery/server/builder/builder_test.go
+++ b/tools/nixery/builder/builder_test.go
diff --git a/tools/nixery/server/builder/cache.go b/tools/nixery/builder/cache.go
index 82bd90927cd0..a4ebe03e1c94 100644
--- a/tools/nixery/server/builder/cache.go
+++ b/tools/nixery/builder/cache.go
@@ -22,7 +22,7 @@ import (
 	"os"
 	"sync"
 
-	"github.com/google/nixery/server/manifest"
+	"github.com/google/nixery/manifest"
 	log "github.com/sirupsen/logrus"
 )
 
diff --git a/tools/nixery/server/layers/grouping.go b/tools/nixery/builder/layers.go
index 3902c8a4ef26..f769e43c5808 100644
--- a/tools/nixery/server/layers/grouping.go
+++ b/tools/nixery/builder/layers.go
@@ -114,7 +114,7 @@
 //
 // Layer budget: 10
 // Layers: { E }, { D, F }, { A }, { B }, { C }
-package layers
+package builder
 
 import (
 	"crypto/sha1"
@@ -128,11 +128,11 @@ import (
 	"gonum.org/v1/gonum/graph/simple"
 )
 
-// RuntimeGraph represents structured information from Nix about the runtime
+// runtimeGraph represents structured information from Nix about the runtime
 // dependencies of a derivation.
 //
 // This is generated in Nix by using the exportReferencesGraph feature.
-type RuntimeGraph struct {
+type runtimeGraph struct {
 	References struct {
 		Graph []string `json:"graph"`
 	} `json:"exportReferencesGraph"`
@@ -153,19 +153,19 @@ type Popularity = map[string]int
 
 // Layer represents the data returned for each layer that Nix should
 // build for the container image.
-type Layer struct {
+type layer struct {
 	Contents    []string `json:"contents"`
 	MergeRating uint64
 }
 
 // Hash the contents of a layer to create a deterministic identifier that can be
 // used for caching.
-func (l *Layer) Hash() string {
+func (l *layer) Hash() string {
 	sum := sha1.Sum([]byte(strings.Join(l.Contents, ":")))
 	return fmt.Sprintf("%x", sum)
 }
 
-func (a Layer) merge(b Layer) Layer {
+func (a layer) merge(b layer) layer {
 	a.Contents = append(a.Contents, b.Contents...)
 	a.MergeRating += b.MergeRating
 	return a
@@ -188,12 +188,15 @@ var nixRegexp = regexp.MustCompile(`^/nix/store/[a-z0-9]+-`)
 
 // PackageFromPath returns the name of a Nix package based on its
 // output store path.
-func PackageFromPath(path string) string {
+func packageFromPath(path string) string {
 	return nixRegexp.ReplaceAllString(path, "")
 }
 
+// DOTID provides a human-readable package name. The name stems from
+// the dot format used by GraphViz, into which the dependency graph
+// can be rendered.
 func (c *closure) DOTID() string {
-	return PackageFromPath(c.Path)
+	return packageFromPath(c.Path)
 }
 
 // bigOrPopular checks whether this closure should be considered for
@@ -236,7 +239,7 @@ func insertEdges(graph *simple.DirectedGraph, cmap *map[string]*closure, node *c
 }
 
 // Create a graph structure from the references supplied by Nix.
-func buildGraph(refs *RuntimeGraph, pop *Popularity) *simple.DirectedGraph {
+func buildGraph(refs *runtimeGraph, pop *Popularity) *simple.DirectedGraph {
 	cmap := make(map[string]*closure)
 	graph := simple.NewDirectedGraph()
 
@@ -296,7 +299,7 @@ func buildGraph(refs *RuntimeGraph, pop *Popularity) *simple.DirectedGraph {
 // Extracts a subgraph starting at the specified root from the
 // dominator tree. The subgraph is converted into a flat list of
 // layers, each containing the store paths and merge rating.
-func groupLayer(dt *flow.DominatorTree, root *closure) Layer {
+func groupLayer(dt *flow.DominatorTree, root *closure) layer {
 	size := root.Size
 	contents := []string{root.Path}
 	children := dt.DominatedBy(root.ID())
@@ -313,7 +316,7 @@ func groupLayer(dt *flow.DominatorTree, root *closure) Layer {
 	// Contents are sorted to ensure that hashing is consistent
 	sort.Strings(contents)
 
-	return Layer{
+	return layer{
 		Contents:    contents,
 		MergeRating: uint64(root.Popularity) * size,
 	}
@@ -324,10 +327,10 @@ func groupLayer(dt *flow.DominatorTree, root *closure) Layer {
 //
 // Layers are merged together until they fit into the layer budget,
 // based on their merge rating.
-func dominate(budget int, graph *simple.DirectedGraph) []Layer {
+func dominate(budget int, graph *simple.DirectedGraph) []layer {
 	dt := flow.Dominators(graph.Node(0), graph)
 
-	var layers []Layer
+	var layers []layer
 	for _, n := range dt.DominatedBy(dt.Root().ID()) {
 		layers = append(layers, groupLayer(&dt, n.(*closure)))
 	}
@@ -352,10 +355,10 @@ func dominate(budget int, graph *simple.DirectedGraph) []Layer {
 	return layers
 }
 
-// GroupLayers applies the algorithm described above the its input and returns a
+// groupLayers applies the algorithm described above the its input and returns a
 // list of layers, each consisting of a list of Nix store paths that it should
 // contain.
-func Group(refs *RuntimeGraph, pop *Popularity, budget int) []Layer {
+func groupLayers(refs *runtimeGraph, pop *Popularity, budget int) []layer {
 	graph := buildGraph(refs, pop)
 	return dominate(budget, graph)
 }
diff --git a/tools/nixery/server/config/config.go b/tools/nixery/config/config.go
index 7ec102bd6cee..7ec102bd6cee 100644
--- a/tools/nixery/server/config/config.go
+++ b/tools/nixery/config/config.go
diff --git a/tools/nixery/server/config/pkgsource.go b/tools/nixery/config/pkgsource.go
index 95236c4b0d15..95236c4b0d15 100644
--- a/tools/nixery/server/config/pkgsource.go
+++ b/tools/nixery/config/pkgsource.go
diff --git a/tools/nixery/default.nix b/tools/nixery/default.nix
index 44ac7313adc7..7454c14a8567 100644
--- a/tools/nixery/default.nix
+++ b/tools/nixery/default.nix
@@ -20,6 +20,8 @@
 with pkgs;
 
 let
+  inherit (pkgs) buildGoPackage;
+
   # Hash of all Nixery sources - this is used as the Nixery version in
   # builds to distinguish errors between deployed versions, see
   # server/logs.go for details.
@@ -30,13 +32,41 @@ let
   # Go implementation of the Nixery server which implements the
   # container registry interface.
   #
-  # Users should use the nixery-bin derivation below instead.
-  nixery-server = callPackage ./server {
-    srcHash = nixery-src-hash;
+  # Users should use the nixery-bin derivation below instead as it
+  # provides the paths of files needed at runtime.
+  nixery-server = buildGoPackage rec {
+    name = "nixery-server";
+    goDeps = ./go-deps.nix;
+    src = ./.;
+
+    goPackagePath = "github.com/google/nixery";
+    doCheck = true;
+
+    # Simplify the Nix build instructions for Go to just the basics
+    # required to get Nixery up and running with the additional linker
+    # flags required.
+    outputs = [ "out" ];
+    preConfigure = "bin=$out";
+    buildPhase = ''
+      runHook preBuild
+      runHook renameImport
+
+      export GOBIN="$out/bin"
+      go install -ldflags "-X main.version=$(cat ${nixery-src-hash})" ${goPackagePath}
+    '';
+
+    fixupPhase = ''
+      remove-references-to -t ${go} $out/bin/nixery
+    '';
+
+    checkPhase = ''
+      go vet ${goPackagePath}
+      go test ${goPackagePath}
+    '';
   };
 in rec {
   # Implementation of the Nix image building logic
-  nixery-build-image = import ./build-image { inherit pkgs; };
+  nixery-prepare-image = import ./prepare-image { inherit pkgs; };
 
   # Use mdBook to build a static asset page which Nixery can then
   # serve. This is primarily used for the public instance at
@@ -50,8 +80,8 @@ in rec {
   # are installing Nixery directly.
   nixery-bin = writeShellScriptBin "nixery" ''
     export WEB_DIR="${nixery-book}"
-    export PATH="${nixery-build-image}/bin:$PATH"
-    exec ${nixery-server}/bin/server
+    export PATH="${nixery-prepare-image}/bin:$PATH"
+    exec ${nixery-server}/bin/nixery
   '';
 
   nixery-popcount = callPackage ./popcount { };
@@ -104,7 +134,7 @@ in rec {
       gzip
       iana-etc
       nix
-      nixery-build-image
+      nixery-prepare-image
       nixery-launch-script
       openssh
       zlib
diff --git a/tools/nixery/server/go-deps.nix b/tools/nixery/go-deps.nix
index 847b44dce63c..847b44dce63c 100644
--- a/tools/nixery/server/go-deps.nix
+++ b/tools/nixery/go-deps.nix
diff --git a/tools/nixery/server/logs.go b/tools/nixery/logs/logs.go
index 3179402e2e1f..4c755bc8ab0c 100644
--- a/tools/nixery/server/logs.go
+++ b/tools/nixery/logs/logs.go
@@ -11,7 +11,7 @@
 // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 // License for the specific language governing permissions and limitations under
 // the License.
-package main
+package logs
 
 // This file configures different log formatters via logrus. The
 // standard formatter uses a structured JSON format that is compatible
@@ -112,7 +112,7 @@ func (f stackdriverFormatter) Format(e *log.Entry) ([]byte, error) {
 	return b.Bytes(), err
 }
 
-func init() {
+func Init(version string) {
 	nixeryContext.Version = version
 	log.SetReportCaller(true)
 	log.SetFormatter(stackdriverFormatter{})
diff --git a/tools/nixery/server/main.go b/tools/nixery/main.go
index 6ae0730906dc..6cad93740978 100644
--- a/tools/nixery/server/main.go
+++ b/tools/nixery/main.go
@@ -32,10 +32,10 @@ import (
 	"net/http"
 	"regexp"
 
-	"github.com/google/nixery/server/builder"
-	"github.com/google/nixery/server/config"
-	"github.com/google/nixery/server/layers"
-	"github.com/google/nixery/server/storage"
+	"github.com/google/nixery/builder"
+	"github.com/google/nixery/config"
+	"github.com/google/nixery/logs"
+	"github.com/google/nixery/storage"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -59,7 +59,7 @@ var (
 
 // Downloads the popularity information for the package set from the
 // URL specified in Nixery's configuration.
-func downloadPopularity(url string) (layers.Popularity, error) {
+func downloadPopularity(url string) (builder.Popularity, error) {
 	resp, err := http.Get(url)
 	if err != nil {
 		return nil, err
@@ -74,7 +74,7 @@ func downloadPopularity(url string) (layers.Popularity, error) {
 		return nil, err
 	}
 
-	var pop layers.Popularity
+	var pop builder.Popularity
 	err = json.Unmarshal(j, &pop)
 	if err != nil {
 		return nil, err
@@ -190,6 +190,7 @@ func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 }
 
 func main() {
+	logs.Init(version)
 	cfg, err := config.FromEnv()
 	if err != nil {
 		log.WithError(err).Fatal("failed to load configuration")
@@ -214,7 +215,7 @@ func main() {
 		log.WithError(err).Fatal("failed to instantiate build cache")
 	}
 
-	var pop layers.Popularity
+	var pop builder.Popularity
 	if cfg.PopUrl != "" {
 		pop, err = downloadPopularity(cfg.PopUrl)
 		if err != nil {
diff --git a/tools/nixery/server/manifest/manifest.go b/tools/nixery/manifest/manifest.go
index 0d36826fb7e5..0d36826fb7e5 100644
--- a/tools/nixery/server/manifest/manifest.go
+++ b/tools/nixery/manifest/manifest.go
diff --git a/tools/nixery/popcount/popcount.go b/tools/nixery/popcount/popcount.go
index b21cee2e0e7d..992a88e874d1 100644
--- a/tools/nixery/popcount/popcount.go
+++ b/tools/nixery/popcount/popcount.go
@@ -175,7 +175,7 @@ func fetchNarInfo(i *item) (string, error) {
 	narinfo, err := ioutil.ReadAll(resp.Body)
 
 	// best-effort write the file to the cache
-	ioutil.WriteFile("popcache/" + i.hash, narinfo, 0644)
+	ioutil.WriteFile("popcache/"+i.hash, narinfo, 0644)
 
 	return string(narinfo), err
 }
diff --git a/tools/nixery/build-image/default.nix b/tools/nixery/prepare-image/default.nix
index a61ac06bdd92..60b208f522d5 100644
--- a/tools/nixery/build-image/default.nix
+++ b/tools/nixery/prepare-image/default.nix
@@ -20,10 +20,10 @@
 
 { pkgs ? import <nixpkgs> {} }:
 
-pkgs.writeShellScriptBin "nixery-build-image" ''
+pkgs.writeShellScriptBin "nixery-prepare-image" ''
   exec ${pkgs.nix}/bin/nix-build \
     --show-trace \
     --no-out-link "$@" \
     --argstr loadPkgs ${./load-pkgs.nix} \
-    ${./build-image.nix}
+    ${./prepare-image.nix}
 ''
diff --git a/tools/nixery/build-image/load-pkgs.nix b/tools/nixery/prepare-image/load-pkgs.nix
index cceebfc14dae..cceebfc14dae 100644
--- a/tools/nixery/build-image/load-pkgs.nix
+++ b/tools/nixery/prepare-image/load-pkgs.nix
diff --git a/tools/nixery/build-image/build-image.nix b/tools/nixery/prepare-image/prepare-image.nix
index 4393f2b859a6..4393f2b859a6 100644
--- a/tools/nixery/build-image/build-image.nix
+++ b/tools/nixery/prepare-image/prepare-image.nix
diff --git a/tools/nixery/server/default.nix b/tools/nixery/server/default.nix
deleted file mode 100644
index d497f106b02e..000000000000
--- a/tools/nixery/server/default.nix
+++ /dev/null
@@ -1,62 +0,0 @@
-# Copyright 2019 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{ buildGoPackage, go, lib, srcHash }:
-
-buildGoPackage rec {
-  name = "nixery-server";
-  goDeps = ./go-deps.nix;
-  src = ./.;
-
-  goPackagePath = "github.com/google/nixery/server";
-  doCheck = true;
-
-  # The following phase configurations work around the overengineered
-  # Nix build configuration for Go.
-  #
-  # All I want this to do is produce a binary in the standard Nix
-  # output path, so pretty much all the phases except for the initial
-  # configuration of the "dependency forest" in $GOPATH have been
-  # overridden.
-  #
-  # This is necessary because the upstream builder does wonky things
-  # with the build arguments to the compiler, but I need to set some
-  # complex flags myself
-
-  outputs = [ "out" ];
-  preConfigure = "bin=$out";
-  buildPhase = ''
-    runHook preBuild
-    runHook renameImport
-
-    export GOBIN="$out/bin"
-    go install -ldflags "-X main.version=$(cat ${srcHash})" ${goPackagePath}
-  '';
-
-  fixupPhase = ''
-    remove-references-to -t ${go} $out/bin/server
-  '';
-
-  checkPhase = ''
-    go vet ${goPackagePath}
-    go test ${goPackagePath}
-  '';
-
-  meta = {
-    description = "Container image builder serving Nix-backed images";
-    homepage = "https://github.com/google/nixery";
-    license = lib.licenses.asl20;
-    maintainers = [ lib.maintainers.tazjin ];
-  };
-}
diff --git a/tools/nixery/shell.nix b/tools/nixery/shell.nix
index 93cd1f4cec62..b37caa83ade3 100644
--- a/tools/nixery/shell.nix
+++ b/tools/nixery/shell.nix
@@ -20,5 +20,5 @@ let nixery = import ./default.nix { inherit pkgs; };
 in pkgs.stdenv.mkDerivation {
   name = "nixery-dev-shell";
 
-  buildInputs = with pkgs; [ jq nixery.nixery-build-image ];
+  buildInputs = with pkgs; [ jq nixery.nixery-prepare-image ];
 }
diff --git a/tools/nixery/server/storage/filesystem.go b/tools/nixery/storage/filesystem.go
index cdbc31c5e046..cdbc31c5e046 100644
--- a/tools/nixery/server/storage/filesystem.go
+++ b/tools/nixery/storage/filesystem.go
diff --git a/tools/nixery/server/storage/gcs.go b/tools/nixery/storage/gcs.go
index c247cca62140..c247cca62140 100644
--- a/tools/nixery/server/storage/gcs.go
+++ b/tools/nixery/storage/gcs.go
diff --git a/tools/nixery/server/storage/storage.go b/tools/nixery/storage/storage.go
index c97b5e4facc6..c97b5e4facc6 100644
--- a/tools/nixery/server/storage/storage.go
+++ b/tools/nixery/storage/storage.go