about summary refs log tree commit diff
diff options
context:
space:
mode:
-rw-r--r--tools/nixery/build-image/build-image.nix9
-rw-r--r--tools/nixery/server/builder/archive.go19
-rw-r--r--tools/nixery/server/builder/builder.go24
-rw-r--r--tools/nixery/server/manifest/manifest.go6
4 files changed, 42 insertions, 16 deletions
diff --git a/tools/nixery/build-image/build-image.nix b/tools/nixery/build-image/build-image.nix
index b78ee6626464..ab785006a9af 100644
--- a/tools/nixery/build-image/build-image.nix
+++ b/tools/nixery/build-image/build-image.nix
@@ -137,11 +137,14 @@ let
   symlinkLayerMeta = fromJSON (readFile (runCommand "symlink-layer-meta.json" {
     buildInputs = with pkgs; [ coreutils jq openssl ];
   }''
-    layerSha256=$(sha256sum ${symlinkLayer} | cut -d ' ' -f1)
+    gzipHash=$(sha256sum ${symlinkLayer} | cut -d ' ' -f1)
+    tarHash=$(cat ${symlinkLayer} | gzip -d | sha256sum | cut -d ' ' -f1)
     layerSize=$(stat --printf '%s' ${symlinkLayer})
 
-    jq -n -c --arg sha256 $layerSha256 --arg size $layerSize --arg path ${symlinkLayer} \
-      '{ size: ($size | tonumber), sha256: $sha256, path: $path }' >> $out
+    jq -n -c --arg gzipHash $gzipHash --arg tarHash $tarHash --arg size $layerSize \
+      --arg path ${symlinkLayer} \
+      '{ size: ($size | tonumber), tarHash: $tarHash, gzipHash: $gzipHash, path: $path }' \
+      >> $out
   ''));
 
   # Final output structure returned to Nixery if the build succeeded
diff --git a/tools/nixery/server/builder/archive.go b/tools/nixery/server/builder/archive.go
index 55b3c2a8b79c..a3fb99882fd6 100644
--- a/tools/nixery/server/builder/archive.go
+++ b/tools/nixery/server/builder/archive.go
@@ -10,6 +10,8 @@ package builder
 import (
 	"archive/tar"
 	"compress/gzip"
+	"crypto/sha256"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -19,26 +21,31 @@ import (
 
 // Create a new compressed tarball from each of the paths in the list
 // and write it to the supplied writer.
-func packStorePaths(l *layers.Layer, w io.Writer) error {
+//
+// The uncompressed tarball is hashed because image manifests must
+// contain both the hashes of compressed and uncompressed layers.
+func packStorePaths(l *layers.Layer, w io.Writer) (string, error) {
+	shasum := sha256.New()
 	gz := gzip.NewWriter(w)
-	t := tar.NewWriter(gz)
+	multi := io.MultiWriter(shasum, gz)
+	t := tar.NewWriter(multi)
 
 	for _, path := range l.Contents {
 		err := filepath.Walk(path, tarStorePath(t))
 		if err != nil {
-			return err
+			return "", err
 		}
 	}
 
 	if err := t.Close(); err != nil {
-		return err
+		return "", err
 	}
 
 	if err := gz.Close(); err != nil {
-		return err
+		return "", err
 	}
 
-	return nil
+	return fmt.Sprintf("sha256:%x", shasum.Sum([]byte{})), nil
 }
 
 func tarStorePath(w *tar.Writer) filepath.WalkFunc {
diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/server/builder/builder.go
index 748ff5f67d7b..39befd0fb8f3 100644
--- a/tools/nixery/server/builder/builder.go
+++ b/tools/nixery/server/builder/builder.go
@@ -117,9 +117,10 @@ type ImageResult struct {
 	// These fields are populated in case of success
 	Graph        layers.RuntimeGraph `json:"runtimeGraph"`
 	SymlinkLayer struct {
-		Size   int    `json:"size"`
-		SHA256 string `json:"sha256"`
-		Path   string `json:"path"`
+		Size     int    `json:"size"`
+		TarHash  string `json:"tarHash"`
+		GzipHash string `json:"gzipHash"`
+		Path     string `json:"path"`
 	} `json:"symlinkLayer"`
 }
 
@@ -269,8 +270,18 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes
 			entries = append(entries, *entry)
 		} else {
 			lh := l.Hash()
+
+			// While packing store paths, the SHA sum of
+			// the uncompressed layer is computed and
+			// written to `tarhash`.
+			//
+			// TODO(tazjin): Refactor this to make the
+			// flow of data cleaner.
+			var tarhash string
 			lw := func(w io.Writer) error {
-				return packStorePaths(&l, w)
+				var err error
+				tarhash, err = packStorePaths(&l, w)
+				return err
 			}
 
 			entry, err := uploadHashLayer(ctx, s, lh, lw)
@@ -278,6 +289,7 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes
 				return nil, err
 			}
 			entry.MergeRating = l.MergeRating
+			entry.TarHash = tarhash
 
 			var pkgs []string
 			for _, p := range l.Contents {
@@ -287,6 +299,7 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes
 			log.WithFields(log.Fields{
 				"layer":    lh,
 				"packages": pkgs,
+				"tarhash":  tarhash,
 			}).Info("created image layer")
 
 			go cacheLayer(ctx, s, l.Hash(), *entry)
@@ -296,7 +309,7 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes
 
 	// Symlink layer (built in the first Nix build) needs to be
 	// included here manually:
-	slkey := result.SymlinkLayer.SHA256
+	slkey := result.SymlinkLayer.GzipHash
 	entry, err := uploadHashLayer(ctx, s, slkey, func(w io.Writer) error {
 		f, err := os.Open(result.SymlinkLayer.Path)
 		if err != nil {
@@ -318,6 +331,7 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes
 		return nil, err
 	}
 
+	entry.TarHash = "sha256:" + result.SymlinkLayer.TarHash
 	go cacheLayer(ctx, s, slkey, *entry)
 	entries = append(entries, *entry)
 
diff --git a/tools/nixery/server/manifest/manifest.go b/tools/nixery/server/manifest/manifest.go
index 8e65fa223b18..8ad828239591 100644
--- a/tools/nixery/server/manifest/manifest.go
+++ b/tools/nixery/server/manifest/manifest.go
@@ -29,9 +29,10 @@ type Entry struct {
 	Size      int64  `json:"size"`
 	Digest    string `json:"digest"`
 
-	// This field is internal to Nixery and not part of the
+	// These fields are internal to Nixery and not part of the
 	// serialised entry.
 	MergeRating uint64 `json:"-"`
+	TarHash     string `json:",omitempty"`
 }
 
 type manifest struct {
@@ -102,9 +103,10 @@ func Manifest(layers []Entry) (json.RawMessage, ConfigLayer) {
 
 	hashes := make([]string, len(layers))
 	for i, l := range layers {
+		hashes[i] = l.TarHash
 		l.MediaType = layerType
+		l.TarHash = ""
 		layers[i] = l
-		hashes[i] = l.Digest
 	}
 
 	c := configLayer(hashes)