From 1853c74998953048478b8526f408f8c57b129958 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Fri, 11 Oct 2019 12:26:18 +0100
Subject: refactor(server): Only compress symlink forest layer once

Instead of compressing & decompressing again to get the underlying tar
hash, use a similar mechanism as for store path layers for the symlink
layer and only compress it once while uploading.
---
 tools/nixery/build-image/build-image.nix | 13 +++++--------
 tools/nixery/server/builder/builder.go   | 27 +++++++++++++++++++--------
 2 files changed, 24 insertions(+), 16 deletions(-)

(limited to 'tools')

diff --git a/tools/nixery/build-image/build-image.nix b/tools/nixery/build-image/build-image.nix
index ab785006a9..d4df707a8a 100644
--- a/tools/nixery/build-image/build-image.nix
+++ b/tools/nixery/build-image/build-image.nix
@@ -126,9 +126,9 @@ let
   # Image layer that contains the symlink forest created above. This
   # must be included in the image to ensure that the filesystem has a
   # useful layout at runtime.
-  symlinkLayer = runCommand "symlink-layer.tar.gz" {} ''
+  symlinkLayer = runCommand "symlink-layer.tar" {} ''
     cp -r ${contentsEnv}/ ./layer
-    tar --transform='s|^\./||' -C layer --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 -czf $out .
+    tar --transform='s|^\./||' -C layer --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 -cf $out .
   '';
 
   # Metadata about the symlink layer which is required for serving it.
@@ -137,14 +137,11 @@ let
   symlinkLayerMeta = fromJSON (readFile (runCommand "symlink-layer-meta.json" {
     buildInputs = with pkgs; [ coreutils jq openssl ];
   }''
-    gzipHash=$(sha256sum ${symlinkLayer} | cut -d ' ' -f1)
-    tarHash=$(cat ${symlinkLayer} | gzip -d | sha256sum | cut -d ' ' -f1)
+    tarHash=$(sha256sum ${symlinkLayer} | cut -d ' ' -f1)
     layerSize=$(stat --printf '%s' ${symlinkLayer})
 
-    jq -n -c --arg gzipHash $gzipHash --arg tarHash $tarHash --arg size $layerSize \
-      --arg path ${symlinkLayer} \
-      '{ size: ($size | tonumber), tarHash: $tarHash, gzipHash: $gzipHash, path: $path }' \
-      >> $out
+    jq -n -c --arg tarHash $tarHash --arg size $layerSize --arg path ${symlinkLayer} \
+      '{ size: ($size | tonumber), tarHash: $tarHash, path: $path }' >> $out
   ''));
 
   # Final output structure returned to Nixery if the build succeeded
diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/server/builder/builder.go
index 39befd0fb8..2a3aa182ac 100644
--- a/tools/nixery/server/builder/builder.go
+++ b/tools/nixery/server/builder/builder.go
@@ -20,6 +20,7 @@ package builder
 import (
 	"bufio"
 	"bytes"
+	"compress/gzip"
 	"context"
 	"crypto/sha256"
 	"encoding/json"
@@ -117,10 +118,9 @@ type ImageResult struct {
 	// These fields are populated in case of success
 	Graph        layers.RuntimeGraph `json:"runtimeGraph"`
 	SymlinkLayer struct {
-		Size     int    `json:"size"`
-		TarHash  string `json:"tarHash"`
-		GzipHash string `json:"gzipHash"`
-		Path     string `json:"path"`
+		Size    int    `json:"size"`
+		TarHash string `json:"tarHash"`
+		Path    string `json:"path"`
 	} `json:"symlinkLayer"`
 }
 
@@ -309,7 +309,7 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes
 
 	// Symlink layer (built in the first Nix build) needs to be
 	// included here manually:
-	slkey := result.SymlinkLayer.GzipHash
+	slkey := result.SymlinkLayer.TarHash
 	entry, err := uploadHashLayer(ctx, s, slkey, func(w io.Writer) error {
 		f, err := os.Open(result.SymlinkLayer.Path)
 		if err != nil {
@@ -317,14 +317,25 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes
 				"image": image.Name,
 				"tag":   image.Tag,
 				"layer": slkey,
-			}).Error("failed to upload symlink layer")
+			}).Error("failed to open symlink layer")
 
 			return err
 		}
 		defer f.Close()
 
-		_, err = io.Copy(w, f)
-		return err
+		gz := gzip.NewWriter(w)
+		_, err = io.Copy(gz, f)
+		if err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"image": image.Name,
+				"tag":   image.Tag,
+				"layer": slkey,
+			}).Error("failed to upload symlink layer")
+
+			return err
+		}
+
+		return gz.Close()
 	})
 
 	if err != nil {
-- 
cgit 1.4.1