From e22ff5d176ba53deecf59737d402cac5d0cb9433 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Fri, 11 Oct 2019 11:57:14 +0100 Subject: fix(server): Use uncompressed tarball hashes in image config Docker expects hashes of compressed tarballs in the manifest (as these are used to fetch from the content-addressable layer store), but for some reason it expects hashes in the configuration layer to be of uncompressed tarballs. To achieve this an additional SHA256 hash is calculcated while creating the layer tarballs, but before passing them to the gzip writer. In the current constellation the symlink layer is first compressed and then decompressed again to calculate its hash. This can be refactored in a future change. --- tools/nixery/build-image/build-image.nix | 9 ++++++--- tools/nixery/server/builder/archive.go | 19 +++++++++++++------ tools/nixery/server/builder/builder.go | 24 +++++++++++++++++++----- tools/nixery/server/manifest/manifest.go | 6 ++++-- 4 files changed, 42 insertions(+), 16 deletions(-) (limited to 'tools/nixery') diff --git a/tools/nixery/build-image/build-image.nix b/tools/nixery/build-image/build-image.nix index b78ee6626464..ab785006a9af 100644 --- a/tools/nixery/build-image/build-image.nix +++ b/tools/nixery/build-image/build-image.nix @@ -137,11 +137,14 @@ let symlinkLayerMeta = fromJSON (readFile (runCommand "symlink-layer-meta.json" { buildInputs = with pkgs; [ coreutils jq openssl ]; }'' - layerSha256=$(sha256sum ${symlinkLayer} | cut -d ' ' -f1) + gzipHash=$(sha256sum ${symlinkLayer} | cut -d ' ' -f1) + tarHash=$(cat ${symlinkLayer} | gzip -d | sha256sum | cut -d ' ' -f1) layerSize=$(stat --printf '%s' ${symlinkLayer}) - jq -n -c --arg sha256 $layerSha256 --arg size $layerSize --arg path ${symlinkLayer} \ - '{ size: ($size | tonumber), sha256: $sha256, path: $path }' >> $out + jq -n -c --arg gzipHash $gzipHash --arg tarHash $tarHash --arg size $layerSize \ + --arg path ${symlinkLayer} \ + '{ size: ($size | tonumber), tarHash: $tarHash, gzipHash: $gzipHash, path: $path }' \ + >> $out '')); # Final output structure returned to Nixery if the build succeeded diff --git a/tools/nixery/server/builder/archive.go b/tools/nixery/server/builder/archive.go index 55b3c2a8b79c..a3fb99882fd6 100644 --- a/tools/nixery/server/builder/archive.go +++ b/tools/nixery/server/builder/archive.go @@ -10,6 +10,8 @@ package builder import ( "archive/tar" "compress/gzip" + "crypto/sha256" + "fmt" "io" "os" "path/filepath" @@ -19,26 +21,31 @@ import ( // Create a new compressed tarball from each of the paths in the list // and write it to the supplied writer. -func packStorePaths(l *layers.Layer, w io.Writer) error { +// +// The uncompressed tarball is hashed because image manifests must +// contain both the hashes of compressed and uncompressed layers. +func packStorePaths(l *layers.Layer, w io.Writer) (string, error) { + shasum := sha256.New() gz := gzip.NewWriter(w) - t := tar.NewWriter(gz) + multi := io.MultiWriter(shasum, gz) + t := tar.NewWriter(multi) for _, path := range l.Contents { err := filepath.Walk(path, tarStorePath(t)) if err != nil { - return err + return "", err } } if err := t.Close(); err != nil { - return err + return "", err } if err := gz.Close(); err != nil { - return err + return "", err } - return nil + return fmt.Sprintf("sha256:%x", shasum.Sum([]byte{})), nil } func tarStorePath(w *tar.Writer) filepath.WalkFunc { diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/server/builder/builder.go index 748ff5f67d7b..39befd0fb8f3 100644 --- a/tools/nixery/server/builder/builder.go +++ b/tools/nixery/server/builder/builder.go @@ -117,9 +117,10 @@ type ImageResult struct { // These fields are populated in case of success Graph layers.RuntimeGraph `json:"runtimeGraph"` SymlinkLayer struct { - Size int `json:"size"` - SHA256 string `json:"sha256"` - Path string `json:"path"` + Size int `json:"size"` + TarHash string `json:"tarHash"` + GzipHash string `json:"gzipHash"` + Path string `json:"path"` } `json:"symlinkLayer"` } @@ -269,8 +270,18 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes entries = append(entries, *entry) } else { lh := l.Hash() + + // While packing store paths, the SHA sum of + // the uncompressed layer is computed and + // written to `tarhash`. + // + // TODO(tazjin): Refactor this to make the + // flow of data cleaner. + var tarhash string lw := func(w io.Writer) error { - return packStorePaths(&l, w) + var err error + tarhash, err = packStorePaths(&l, w) + return err } entry, err := uploadHashLayer(ctx, s, lh, lw) @@ -278,6 +289,7 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes return nil, err } entry.MergeRating = l.MergeRating + entry.TarHash = tarhash var pkgs []string for _, p := range l.Contents { @@ -287,6 +299,7 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes log.WithFields(log.Fields{ "layer": lh, "packages": pkgs, + "tarhash": tarhash, }).Info("created image layer") go cacheLayer(ctx, s, l.Hash(), *entry) @@ -296,7 +309,7 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes // Symlink layer (built in the first Nix build) needs to be // included here manually: - slkey := result.SymlinkLayer.SHA256 + slkey := result.SymlinkLayer.GzipHash entry, err := uploadHashLayer(ctx, s, slkey, func(w io.Writer) error { f, err := os.Open(result.SymlinkLayer.Path) if err != nil { @@ -318,6 +331,7 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes return nil, err } + entry.TarHash = "sha256:" + result.SymlinkLayer.TarHash go cacheLayer(ctx, s, slkey, *entry) entries = append(entries, *entry) diff --git a/tools/nixery/server/manifest/manifest.go b/tools/nixery/server/manifest/manifest.go index 8e65fa223b18..8ad828239591 100644 --- a/tools/nixery/server/manifest/manifest.go +++ b/tools/nixery/server/manifest/manifest.go @@ -29,9 +29,10 @@ type Entry struct { Size int64 `json:"size"` Digest string `json:"digest"` - // This field is internal to Nixery and not part of the + // These fields are internal to Nixery and not part of the // serialised entry. MergeRating uint64 `json:"-"` + TarHash string `json:",omitempty"` } type manifest struct { @@ -102,9 +103,10 @@ func Manifest(layers []Entry) (json.RawMessage, ConfigLayer) { hashes := make([]string, len(layers)) for i, l := range layers { + hashes[i] = l.TarHash l.MediaType = layerType + l.TarHash = "" layers[i] = l - hashes[i] = l.Digest } c := configLayer(hashes) -- cgit 1.4.1