diff options
author | Vincent Ambo <tazjin@google.com> | 2019-11-11T21·07+0000 |
---|---|---|
committer | Vincent Ambo <github@tazj.in> | 2019-11-27T14·12+0000 |
commit | 2b82f1b71a50b8b1473421cce0eec1a0d7ddc360 (patch) | |
tree | d42dcfd823f63ac77d3517b3c3154619f87f2cd2 | |
parent | df88da126a5c0dc97aa0fadaf1baf069b80ce251 (diff) |
refactor: Reshuffle file structure for better code layout
This gets rid of the package called "server" and instead moves everything into the project root, such that Go actually builds us a binary called `nixery`. This is the first step towards factoring out CLI-based functionality for Nixery.
-rw-r--r-- | tools/nixery/builder/archive.go (renamed from tools/nixery/server/builder/archive.go) | 5 | ||||
-rw-r--r-- | tools/nixery/builder/builder.go (renamed from tools/nixery/server/builder/builder.go) | 24 | ||||
-rw-r--r-- | tools/nixery/builder/builder_test.go (renamed from tools/nixery/server/builder/builder_test.go) | 0 | ||||
-rw-r--r-- | tools/nixery/builder/cache.go (renamed from tools/nixery/server/builder/cache.go) | 2 | ||||
-rw-r--r-- | tools/nixery/builder/layers.go (renamed from tools/nixery/server/layers/grouping.go) | 33 | ||||
-rw-r--r-- | tools/nixery/config/config.go (renamed from tools/nixery/server/config/config.go) | 0 | ||||
-rw-r--r-- | tools/nixery/config/pkgsource.go (renamed from tools/nixery/server/config/pkgsource.go) | 0 | ||||
-rw-r--r-- | tools/nixery/default.nix | 44 | ||||
-rw-r--r-- | tools/nixery/go-deps.nix (renamed from tools/nixery/server/go-deps.nix) | 0 | ||||
-rw-r--r-- | tools/nixery/logs/logs.go (renamed from tools/nixery/server/logs.go) | 4 | ||||
-rw-r--r-- | tools/nixery/main.go (renamed from tools/nixery/server/main.go) | 15 | ||||
-rw-r--r-- | tools/nixery/manifest/manifest.go (renamed from tools/nixery/server/manifest/manifest.go) | 0 | ||||
-rw-r--r-- | tools/nixery/popcount/popcount.go | 2 | ||||
-rw-r--r-- | tools/nixery/prepare-image/default.nix (renamed from tools/nixery/build-image/default.nix) | 4 | ||||
-rw-r--r-- | tools/nixery/prepare-image/load-pkgs.nix (renamed from tools/nixery/build-image/load-pkgs.nix) | 0 | ||||
-rw-r--r-- | tools/nixery/prepare-image/prepare-image.nix (renamed from tools/nixery/build-image/build-image.nix) | 0 | ||||
-rw-r--r-- | tools/nixery/server/default.nix | 62 | ||||
-rw-r--r-- | tools/nixery/shell.nix | 2 | ||||
-rw-r--r-- | tools/nixery/storage/filesystem.go (renamed from tools/nixery/server/storage/filesystem.go) | 0 | ||||
-rw-r--r-- | tools/nixery/storage/gcs.go (renamed from tools/nixery/server/storage/gcs.go) | 0 | ||||
-rw-r--r-- | tools/nixery/storage/storage.go (renamed from tools/nixery/server/storage/storage.go) | 0 |
21 files changed, 83 insertions, 114 deletions
diff --git a/tools/nixery/server/builder/archive.go b/tools/nixery/builder/archive.go index e0fb76d44bee..ff822e389a7d 100644 --- a/tools/nixery/server/builder/archive.go +++ b/tools/nixery/builder/archive.go @@ -19,7 +19,6 @@ package builder // The tarball is written straight to the supplied reader, which makes it // possible to create an image layer from the specified store paths, hash it and // upload it in one reading pass. - import ( "archive/tar" "compress/gzip" @@ -28,8 +27,6 @@ import ( "io" "os" "path/filepath" - - "github.com/google/nixery/server/layers" ) // Create a new compressed tarball from each of the paths in the list @@ -37,7 +34,7 @@ import ( // // The uncompressed tarball is hashed because image manifests must // contain both the hashes of compressed and uncompressed layers. -func packStorePaths(l *layers.Layer, w io.Writer) (string, error) { +func packStorePaths(l *layer, w io.Writer) (string, error) { shasum := sha256.New() gz := gzip.NewWriter(w) multi := io.MultiWriter(shasum, gz) diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/builder/builder.go index da9dede1acd7..ceb112df90cf 100644 --- a/tools/nixery/server/builder/builder.go +++ b/tools/nixery/builder/builder.go @@ -12,9 +12,10 @@ // License for the specific language governing permissions and limitations under // the License. -// Package builder implements the code required to build images via Nix. Image -// build data is cached for up to 24 hours to avoid duplicated calls to Nix -// (which are costly even if no building is performed). +// Package builder implements the logic for assembling container +// images. It shells out to Nix to retrieve all required Nix-packages +// and assemble the symlink layer and then creates the required +// tarballs in-process. package builder import ( @@ -32,10 +33,9 @@ import ( "sort" "strings" - "github.com/google/nixery/server/config" - "github.com/google/nixery/server/layers" - "github.com/google/nixery/server/manifest" - "github.com/google/nixery/server/storage" + "github.com/google/nixery/config" + "github.com/google/nixery/manifest" + "github.com/google/nixery/storage" log "github.com/sirupsen/logrus" ) @@ -50,7 +50,7 @@ type State struct { Storage storage.Backend Cache *LocalCache Cfg config.Config - Pop layers.Popularity + Pop Popularity } // Architecture represents the possible CPU architectures for which @@ -128,7 +128,7 @@ type ImageResult struct { Pkgs []string `json:"pkgs"` // These fields are populated in case of success - Graph layers.RuntimeGraph `json:"runtimeGraph"` + Graph runtimeGraph `json:"runtimeGraph"` SymlinkLayer struct { Size int `json:"size"` TarHash string `json:"tarHash"` @@ -265,7 +265,7 @@ func prepareImage(s *State, image *Image) (*ImageResult, error) { "--argstr", "system", image.Arch.nixSystem, } - output, err := callNix("nixery-build-image", image.Name, args) + output, err := callNix("nixery-prepare-image", image.Name, args) if err != nil { // granular error logging is performed in callNix already return nil, err @@ -292,7 +292,7 @@ func prepareImage(s *State, image *Image) (*ImageResult, error) { // added only after successful uploads, which guarantees that entries // retrieved from the cache are present in the bucket. func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageResult) ([]manifest.Entry, error) { - grouped := layers.Group(&result.Graph, &s.Pop, LayerBudget) + grouped := groupLayers(&result.Graph, &s.Pop, LayerBudget) var entries []manifest.Entry @@ -329,7 +329,7 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes var pkgs []string for _, p := range l.Contents { - pkgs = append(pkgs, layers.PackageFromPath(p)) + pkgs = append(pkgs, packageFromPath(p)) } log.WithFields(log.Fields{ diff --git a/tools/nixery/server/builder/builder_test.go b/tools/nixery/builder/builder_test.go index 3fbe2ab40e23..3fbe2ab40e23 100644 --- a/tools/nixery/server/builder/builder_test.go +++ b/tools/nixery/builder/builder_test.go diff --git a/tools/nixery/server/builder/cache.go b/tools/nixery/builder/cache.go index 82bd90927cd0..a4ebe03e1c94 100644 --- a/tools/nixery/server/builder/cache.go +++ b/tools/nixery/builder/cache.go @@ -22,7 +22,7 @@ import ( "os" "sync" - "github.com/google/nixery/server/manifest" + "github.com/google/nixery/manifest" log "github.com/sirupsen/logrus" ) diff --git a/tools/nixery/server/layers/grouping.go b/tools/nixery/builder/layers.go index 3902c8a4ef26..f769e43c5808 100644 --- a/tools/nixery/server/layers/grouping.go +++ b/tools/nixery/builder/layers.go @@ -114,7 +114,7 @@ // // Layer budget: 10 // Layers: { E }, { D, F }, { A }, { B }, { C } -package layers +package builder import ( "crypto/sha1" @@ -128,11 +128,11 @@ import ( "gonum.org/v1/gonum/graph/simple" ) -// RuntimeGraph represents structured information from Nix about the runtime +// runtimeGraph represents structured information from Nix about the runtime // dependencies of a derivation. // // This is generated in Nix by using the exportReferencesGraph feature. -type RuntimeGraph struct { +type runtimeGraph struct { References struct { Graph []string `json:"graph"` } `json:"exportReferencesGraph"` @@ -153,19 +153,19 @@ type Popularity = map[string]int // Layer represents the data returned for each layer that Nix should // build for the container image. -type Layer struct { +type layer struct { Contents []string `json:"contents"` MergeRating uint64 } // Hash the contents of a layer to create a deterministic identifier that can be // used for caching. -func (l *Layer) Hash() string { +func (l *layer) Hash() string { sum := sha1.Sum([]byte(strings.Join(l.Contents, ":"))) return fmt.Sprintf("%x", sum) } -func (a Layer) merge(b Layer) Layer { +func (a layer) merge(b layer) layer { a.Contents = append(a.Contents, b.Contents...) a.MergeRating += b.MergeRating return a @@ -188,12 +188,15 @@ var nixRegexp = regexp.MustCompile(`^/nix/store/[a-z0-9]+-`) // PackageFromPath returns the name of a Nix package based on its // output store path. -func PackageFromPath(path string) string { +func packageFromPath(path string) string { return nixRegexp.ReplaceAllString(path, "") } +// DOTID provides a human-readable package name. The name stems from +// the dot format used by GraphViz, into which the dependency graph +// can be rendered. func (c *closure) DOTID() string { - return PackageFromPath(c.Path) + return packageFromPath(c.Path) } // bigOrPopular checks whether this closure should be considered for @@ -236,7 +239,7 @@ func insertEdges(graph *simple.DirectedGraph, cmap *map[string]*closure, node *c } // Create a graph structure from the references supplied by Nix. -func buildGraph(refs *RuntimeGraph, pop *Popularity) *simple.DirectedGraph { +func buildGraph(refs *runtimeGraph, pop *Popularity) *simple.DirectedGraph { cmap := make(map[string]*closure) graph := simple.NewDirectedGraph() @@ -296,7 +299,7 @@ func buildGraph(refs *RuntimeGraph, pop *Popularity) *simple.DirectedGraph { // Extracts a subgraph starting at the specified root from the // dominator tree. The subgraph is converted into a flat list of // layers, each containing the store paths and merge rating. -func groupLayer(dt *flow.DominatorTree, root *closure) Layer { +func groupLayer(dt *flow.DominatorTree, root *closure) layer { size := root.Size contents := []string{root.Path} children := dt.DominatedBy(root.ID()) @@ -313,7 +316,7 @@ func groupLayer(dt *flow.DominatorTree, root *closure) Layer { // Contents are sorted to ensure that hashing is consistent sort.Strings(contents) - return Layer{ + return layer{ Contents: contents, MergeRating: uint64(root.Popularity) * size, } @@ -324,10 +327,10 @@ func groupLayer(dt *flow.DominatorTree, root *closure) Layer { // // Layers are merged together until they fit into the layer budget, // based on their merge rating. -func dominate(budget int, graph *simple.DirectedGraph) []Layer { +func dominate(budget int, graph *simple.DirectedGraph) []layer { dt := flow.Dominators(graph.Node(0), graph) - var layers []Layer + var layers []layer for _, n := range dt.DominatedBy(dt.Root().ID()) { layers = append(layers, groupLayer(&dt, n.(*closure))) } @@ -352,10 +355,10 @@ func dominate(budget int, graph *simple.DirectedGraph) []Layer { return layers } -// GroupLayers applies the algorithm described above the its input and returns a +// groupLayers applies the algorithm described above the its input and returns a // list of layers, each consisting of a list of Nix store paths that it should // contain. -func Group(refs *RuntimeGraph, pop *Popularity, budget int) []Layer { +func groupLayers(refs *runtimeGraph, pop *Popularity, budget int) []layer { graph := buildGraph(refs, pop) return dominate(budget, graph) } diff --git a/tools/nixery/server/config/config.go b/tools/nixery/config/config.go index 7ec102bd6cee..7ec102bd6cee 100644 --- a/tools/nixery/server/config/config.go +++ b/tools/nixery/config/config.go diff --git a/tools/nixery/server/config/pkgsource.go b/tools/nixery/config/pkgsource.go index 95236c4b0d15..95236c4b0d15 100644 --- a/tools/nixery/server/config/pkgsource.go +++ b/tools/nixery/config/pkgsource.go diff --git a/tools/nixery/default.nix b/tools/nixery/default.nix index 44ac7313adc7..7454c14a8567 100644 --- a/tools/nixery/default.nix +++ b/tools/nixery/default.nix @@ -20,6 +20,8 @@ with pkgs; let + inherit (pkgs) buildGoPackage; + # Hash of all Nixery sources - this is used as the Nixery version in # builds to distinguish errors between deployed versions, see # server/logs.go for details. @@ -30,13 +32,41 @@ let # Go implementation of the Nixery server which implements the # container registry interface. # - # Users should use the nixery-bin derivation below instead. - nixery-server = callPackage ./server { - srcHash = nixery-src-hash; + # Users should use the nixery-bin derivation below instead as it + # provides the paths of files needed at runtime. + nixery-server = buildGoPackage rec { + name = "nixery-server"; + goDeps = ./go-deps.nix; + src = ./.; + + goPackagePath = "github.com/google/nixery"; + doCheck = true; + + # Simplify the Nix build instructions for Go to just the basics + # required to get Nixery up and running with the additional linker + # flags required. + outputs = [ "out" ]; + preConfigure = "bin=$out"; + buildPhase = '' + runHook preBuild + runHook renameImport + + export GOBIN="$out/bin" + go install -ldflags "-X main.version=$(cat ${nixery-src-hash})" ${goPackagePath} + ''; + + fixupPhase = '' + remove-references-to -t ${go} $out/bin/nixery + ''; + + checkPhase = '' + go vet ${goPackagePath} + go test ${goPackagePath} + ''; }; in rec { # Implementation of the Nix image building logic - nixery-build-image = import ./build-image { inherit pkgs; }; + nixery-prepare-image = import ./prepare-image { inherit pkgs; }; # Use mdBook to build a static asset page which Nixery can then # serve. This is primarily used for the public instance at @@ -50,8 +80,8 @@ in rec { # are installing Nixery directly. nixery-bin = writeShellScriptBin "nixery" '' export WEB_DIR="${nixery-book}" - export PATH="${nixery-build-image}/bin:$PATH" - exec ${nixery-server}/bin/server + export PATH="${nixery-prepare-image}/bin:$PATH" + exec ${nixery-server}/bin/nixery ''; nixery-popcount = callPackage ./popcount { }; @@ -104,7 +134,7 @@ in rec { gzip iana-etc nix - nixery-build-image + nixery-prepare-image nixery-launch-script openssh zlib diff --git a/tools/nixery/server/go-deps.nix b/tools/nixery/go-deps.nix index 847b44dce63c..847b44dce63c 100644 --- a/tools/nixery/server/go-deps.nix +++ b/tools/nixery/go-deps.nix diff --git a/tools/nixery/server/logs.go b/tools/nixery/logs/logs.go index 3179402e2e1f..4c755bc8ab0c 100644 --- a/tools/nixery/server/logs.go +++ b/tools/nixery/logs/logs.go @@ -11,7 +11,7 @@ // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the // License for the specific language governing permissions and limitations under // the License. -package main +package logs // This file configures different log formatters via logrus. The // standard formatter uses a structured JSON format that is compatible @@ -112,7 +112,7 @@ func (f stackdriverFormatter) Format(e *log.Entry) ([]byte, error) { return b.Bytes(), err } -func init() { +func Init(version string) { nixeryContext.Version = version log.SetReportCaller(true) log.SetFormatter(stackdriverFormatter{}) diff --git a/tools/nixery/server/main.go b/tools/nixery/main.go index 6ae0730906dc..6cad93740978 100644 --- a/tools/nixery/server/main.go +++ b/tools/nixery/main.go @@ -32,10 +32,10 @@ import ( "net/http" "regexp" - "github.com/google/nixery/server/builder" - "github.com/google/nixery/server/config" - "github.com/google/nixery/server/layers" - "github.com/google/nixery/server/storage" + "github.com/google/nixery/builder" + "github.com/google/nixery/config" + "github.com/google/nixery/logs" + "github.com/google/nixery/storage" log "github.com/sirupsen/logrus" ) @@ -59,7 +59,7 @@ var ( // Downloads the popularity information for the package set from the // URL specified in Nixery's configuration. -func downloadPopularity(url string) (layers.Popularity, error) { +func downloadPopularity(url string) (builder.Popularity, error) { resp, err := http.Get(url) if err != nil { return nil, err @@ -74,7 +74,7 @@ func downloadPopularity(url string) (layers.Popularity, error) { return nil, err } - var pop layers.Popularity + var pop builder.Popularity err = json.Unmarshal(j, &pop) if err != nil { return nil, err @@ -190,6 +190,7 @@ func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } func main() { + logs.Init(version) cfg, err := config.FromEnv() if err != nil { log.WithError(err).Fatal("failed to load configuration") @@ -214,7 +215,7 @@ func main() { log.WithError(err).Fatal("failed to instantiate build cache") } - var pop layers.Popularity + var pop builder.Popularity if cfg.PopUrl != "" { pop, err = downloadPopularity(cfg.PopUrl) if err != nil { diff --git a/tools/nixery/server/manifest/manifest.go b/tools/nixery/manifest/manifest.go index 0d36826fb7e5..0d36826fb7e5 100644 --- a/tools/nixery/server/manifest/manifest.go +++ b/tools/nixery/manifest/manifest.go diff --git a/tools/nixery/popcount/popcount.go b/tools/nixery/popcount/popcount.go index b21cee2e0e7d..992a88e874d1 100644 --- a/tools/nixery/popcount/popcount.go +++ b/tools/nixery/popcount/popcount.go @@ -175,7 +175,7 @@ func fetchNarInfo(i *item) (string, error) { narinfo, err := ioutil.ReadAll(resp.Body) // best-effort write the file to the cache - ioutil.WriteFile("popcache/" + i.hash, narinfo, 0644) + ioutil.WriteFile("popcache/"+i.hash, narinfo, 0644) return string(narinfo), err } diff --git a/tools/nixery/build-image/default.nix b/tools/nixery/prepare-image/default.nix index a61ac06bdd92..60b208f522d5 100644 --- a/tools/nixery/build-image/default.nix +++ b/tools/nixery/prepare-image/default.nix @@ -20,10 +20,10 @@ { pkgs ? import <nixpkgs> {} }: -pkgs.writeShellScriptBin "nixery-build-image" '' +pkgs.writeShellScriptBin "nixery-prepare-image" '' exec ${pkgs.nix}/bin/nix-build \ --show-trace \ --no-out-link "$@" \ --argstr loadPkgs ${./load-pkgs.nix} \ - ${./build-image.nix} + ${./prepare-image.nix} '' diff --git a/tools/nixery/build-image/load-pkgs.nix b/tools/nixery/prepare-image/load-pkgs.nix index cceebfc14dae..cceebfc14dae 100644 --- a/tools/nixery/build-image/load-pkgs.nix +++ b/tools/nixery/prepare-image/load-pkgs.nix diff --git a/tools/nixery/build-image/build-image.nix b/tools/nixery/prepare-image/prepare-image.nix index 4393f2b859a6..4393f2b859a6 100644 --- a/tools/nixery/build-image/build-image.nix +++ b/tools/nixery/prepare-image/prepare-image.nix diff --git a/tools/nixery/server/default.nix b/tools/nixery/server/default.nix deleted file mode 100644 index d497f106b02e..000000000000 --- a/tools/nixery/server/default.nix +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{ buildGoPackage, go, lib, srcHash }: - -buildGoPackage rec { - name = "nixery-server"; - goDeps = ./go-deps.nix; - src = ./.; - - goPackagePath = "github.com/google/nixery/server"; - doCheck = true; - - # The following phase configurations work around the overengineered - # Nix build configuration for Go. - # - # All I want this to do is produce a binary in the standard Nix - # output path, so pretty much all the phases except for the initial - # configuration of the "dependency forest" in $GOPATH have been - # overridden. - # - # This is necessary because the upstream builder does wonky things - # with the build arguments to the compiler, but I need to set some - # complex flags myself - - outputs = [ "out" ]; - preConfigure = "bin=$out"; - buildPhase = '' - runHook preBuild - runHook renameImport - - export GOBIN="$out/bin" - go install -ldflags "-X main.version=$(cat ${srcHash})" ${goPackagePath} - ''; - - fixupPhase = '' - remove-references-to -t ${go} $out/bin/server - ''; - - checkPhase = '' - go vet ${goPackagePath} - go test ${goPackagePath} - ''; - - meta = { - description = "Container image builder serving Nix-backed images"; - homepage = "https://github.com/google/nixery"; - license = lib.licenses.asl20; - maintainers = [ lib.maintainers.tazjin ]; - }; -} diff --git a/tools/nixery/shell.nix b/tools/nixery/shell.nix index 93cd1f4cec62..b37caa83ade3 100644 --- a/tools/nixery/shell.nix +++ b/tools/nixery/shell.nix @@ -20,5 +20,5 @@ let nixery = import ./default.nix { inherit pkgs; }; in pkgs.stdenv.mkDerivation { name = "nixery-dev-shell"; - buildInputs = with pkgs; [ jq nixery.nixery-build-image ]; + buildInputs = with pkgs; [ jq nixery.nixery-prepare-image ]; } diff --git a/tools/nixery/server/storage/filesystem.go b/tools/nixery/storage/filesystem.go index cdbc31c5e046..cdbc31c5e046 100644 --- a/tools/nixery/server/storage/filesystem.go +++ b/tools/nixery/storage/filesystem.go diff --git a/tools/nixery/server/storage/gcs.go b/tools/nixery/storage/gcs.go index c247cca62140..c247cca62140 100644 --- a/tools/nixery/server/storage/gcs.go +++ b/tools/nixery/storage/gcs.go diff --git a/tools/nixery/server/storage/storage.go b/tools/nixery/storage/storage.go index c97b5e4facc6..c97b5e4facc6 100644 --- a/tools/nixery/server/storage/storage.go +++ b/tools/nixery/storage/storage.go |