feat(builder): Implement build cache for manifests & layers

Implements a cache that keeps track of:

a) Manifests that have already been built (for up to 6 hours)
b) Layers that have already been seen (and uploaded to GCS)

This significantly speeds up response times for images that are full
or partial matches with previous images served by an instance.

1 files changed, 53 insertions, 41 deletions

diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/server/builder/builder.go
index c53b702e0537..e241d3d0b004 100644
--- a/tools/nixery/server/builder/builder.go
+++ b/tools/nixery/server/builder/builder.go
@@ -69,10 +69,10 @@ func ImageFromName(name string, tag string) Image {
 //
 // The later field is simply treated as opaque JSON and passed through.
 type BuildResult struct {
-	Error string   `json:"error"`
-	Pkgs  []string `json:"pkgs"`
+	Error    string          `json:"error"`
+	Pkgs     []string        `json:"pkgs"`
+	Manifest json.RawMessage `json:"manifest"`
 
-	Manifest       json.RawMessage `json:"manifest"`
 	LayerLocations map[string]struct {
 		Path string `json:"path"`
 		Md5  []byte `json:"md5"`
@@ -99,50 +99,57 @@ func convenienceNames(packages []string) []string {
 
 // Call out to Nix and request that an image be built. Nix will, upon success,
 // return a manifest for the container image.
-func BuildImage(ctx *context.Context, cfg *config.Config, image *Image, bucket *storage.BucketHandle) (*BuildResult, error) {
-	packages, err := json.Marshal(image.Packages)
-	if err != nil {
-		return nil, err
-	}
+func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, image *Image, bucket *storage.BucketHandle) (*BuildResult, error) {
+	resultFile, cached := cache.manifestFromCache(image)
 
-	args := []string{
-		"--argstr", "name", image.Name,
-		"--argstr", "packages", string(packages),
-	}
+	if !cached {
+		packages, err := json.Marshal(image.Packages)
+		if err != nil {
+			return nil, err
+		}
 
-	if cfg.Pkgs != nil {
-		args = append(args, "--argstr", "pkgSource", cfg.Pkgs.Render(image.Tag))
-	}
-	cmd := exec.Command("nixery-build-image", args...)
+		args := []string{
+			"--argstr", "name", image.Name,
+			"--argstr", "packages", string(packages),
+		}
 
-	outpipe, err := cmd.StdoutPipe()
-	if err != nil {
-		return nil, err
-	}
+		if cfg.Pkgs != nil {
+			args = append(args, "--argstr", "pkgSource", cfg.Pkgs.Render(image.Tag))
+		}
+		cmd := exec.Command("nixery-build-image", args...)
 
-	errpipe, err := cmd.StderrPipe()
-	if err != nil {
-		return nil, err
-	}
+		outpipe, err := cmd.StdoutPipe()
+		if err != nil {
+			return nil, err
+		}
 
-	if err = cmd.Start(); err != nil {
-		log.Println("Error starting nix-build:", err)
-		return nil, err
-	}
-	log.Printf("Started Nix image build for '%s'", image.Name)
+		errpipe, err := cmd.StderrPipe()
+		if err != nil {
+			return nil, err
+		}
 
-	stdout, _ := ioutil.ReadAll(outpipe)
-	stderr, _ := ioutil.ReadAll(errpipe)
+		if err = cmd.Start(); err != nil {
+			log.Println("Error starting nix-build:", err)
+			return nil, err
+		}
+		log.Printf("Started Nix image build for '%s'", image.Name)
 
-	if err = cmd.Wait(); err != nil {
-		// TODO(tazjin): Propagate errors upwards in a usable format.
-		log.Printf("nix-build execution error: %s\nstdout: %s\nstderr: %s\n", err, stdout, stderr)
-		return nil, err
-	}
+		stdout, _ := ioutil.ReadAll(outpipe)
+		stderr, _ := ioutil.ReadAll(errpipe)
 
-	log.Println("Finished Nix image build")
+		if err = cmd.Wait(); err != nil {
+			// TODO(tazjin): Propagate errors upwards in a usable format.
+			log.Printf("nix-build execution error: %s\nstdout: %s\nstderr: %s\n", err, stdout, stderr)
+			return nil, err
+		}
+
+		log.Println("Finished Nix image build")
+
+		resultFile = strings.TrimSpace(string(stdout))
+		cache.cacheManifest(image, resultFile)
+	}
 
-	buildOutput, err := ioutil.ReadFile(strings.TrimSpace(string(stdout)))
+	buildOutput, err := ioutil.ReadFile(resultFile)
 	if err != nil {
 		return nil, err
 	}
@@ -151,15 +158,20 @@ func BuildImage(ctx *context.Context, cfg *config.Config, image *Image, bucket *
 	// contained layers to the bucket. Only the manifest itself is
 	// re-serialised to JSON and returned.
 	var result BuildResult
+
 	err = json.Unmarshal(buildOutput, &result)
 	if err != nil {
 		return nil, err
 	}
 
 	for layer, meta := range result.LayerLocations {
-		err = uploadLayer(ctx, bucket, layer, meta.Path, meta.Md5)
-		if err != nil {
-			return nil, err
+		if !cache.hasSeenLayer(layer) {
+			err = uploadLayer(ctx, bucket, layer, meta.Path, meta.Md5)
+			if err != nil {
+				return nil, err
+			}
+
+			cache.sawLayer(layer)
 		}
 	}