about summary refs log tree commit diff
path: root/tools/nixery
diff options
context:
space:
mode:
authorVincent Ambo <tazjin@google.com>2019-10-03T19·18+0100
committerVincent Ambo <github@tazj.in>2019-10-03T21·50+0100
commit48a5ecda97e4b2ea9faa2d3031376078ccc301be (patch)
tree1ce8ab5c6fbe5c653115d30846276013b32f9501 /tools/nixery
parent0d820423e973727ddfc4b461a1063f719873743c (diff)
feat(server): Order layers in image manifest based on merge rating
Image layers in manifests are now sorted in a stable (descending)
order based on their merge rating, meaning that layers more likely to
be shared between images come first.

The reason for this change is Docker's handling of image layers on
overlayfs2: Images are condensed into a single representation on disk
after downloading.

Due to this Docker will constantly redownload all layers that are
applied in a different order in different images (layer order matters
in imperatively created images), based on something it calls the
'ChainID'.

Sorting the layers this way raises the likelihood of a long chain of
matching layers at the beginning of an image.

This relates to #39.
Diffstat (limited to 'tools/nixery')
-rw-r--r--tools/nixery/server/builder/builder.go1
-rw-r--r--tools/nixery/server/layers/grouping.go8
-rw-r--r--tools/nixery/server/manifest/manifest.go15
3 files changed, 20 insertions, 4 deletions
diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/server/builder/builder.go
index d8cbbb8f21c9..614291e660c5 100644
--- a/tools/nixery/server/builder/builder.go
+++ b/tools/nixery/server/builder/builder.go
@@ -250,6 +250,7 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes
 			if err != nil {
 				return nil, err
 			}
+			entry.MergeRating = l.MergeRating
 
 			go cacheLayer(ctx, s, l.Hash(), *entry)
 			entries = append(entries, *entry)
diff --git a/tools/nixery/server/layers/grouping.go b/tools/nixery/server/layers/grouping.go
index 07a9e0e230a5..9992cd3c13d6 100644
--- a/tools/nixery/server/layers/grouping.go
+++ b/tools/nixery/server/layers/grouping.go
@@ -141,7 +141,7 @@ type Popularity = map[string]int
 // build for the container image.
 type Layer struct {
 	Contents    []string `json:"contents"`
-	mergeRating uint64
+	MergeRating uint64
 }
 
 // Hash the contents of a layer to create a deterministic identifier that can be
@@ -153,7 +153,7 @@ func (l *Layer) Hash() string {
 
 func (a Layer) merge(b Layer) Layer {
 	a.Contents = append(a.Contents, b.Contents...)
-	a.mergeRating += b.mergeRating
+	a.MergeRating += b.MergeRating
 	return a
 }
 
@@ -291,7 +291,7 @@ func groupLayer(dt *flow.DominatorTree, root *closure) Layer {
 		// both the size and the popularity when making merge
 		// decisions, but there might be a smarter way to do
 		// it than a plain multiplication.
-		mergeRating: uint64(root.Popularity) * size,
+		MergeRating: uint64(root.Popularity) * size,
 	}
 }
 
@@ -309,7 +309,7 @@ func dominate(budget int, graph *simple.DirectedGraph) []Layer {
 	}
 
 	sort.Slice(layers, func(i, j int) bool {
-		return layers[i].mergeRating < layers[j].mergeRating
+		return layers[i].MergeRating < layers[j].MergeRating
 	})
 
 	if len(layers) > budget {
diff --git a/tools/nixery/server/manifest/manifest.go b/tools/nixery/server/manifest/manifest.go
index f777e3f585df..2f236178b65f 100644
--- a/tools/nixery/server/manifest/manifest.go
+++ b/tools/nixery/server/manifest/manifest.go
@@ -6,6 +6,7 @@ import (
 	"crypto/sha256"
 	"encoding/json"
 	"fmt"
+	"sort"
 )
 
 const (
@@ -27,6 +28,10 @@ type Entry struct {
 	MediaType string `json:"mediaType,omitempty"`
 	Size      int64  `json:"size"`
 	Digest    string `json:"digest"`
+
+	// This field is internal to Nixery and not part of the
+	// serialised entry.
+	MergeRating uint64 `json:"-"`
 }
 
 type manifest struct {
@@ -85,6 +90,16 @@ func configLayer(hashes []string) ConfigLayer {
 //
 // Callers do not need to set the media type for the layer entries.
 func Manifest(layers []Entry) (json.RawMessage, ConfigLayer) {
+	// Sort layers by their merge rating, from highest to lowest.
+	// This makes it likely for a contiguous chain of shared image
+	// layers to appear at the beginning of a layer.
+	//
+	// Due to moby/moby#38446 Docker considers the order of layers
+	// when deciding which layers to download again.
+	sort.Slice(layers, func(i, j int) bool {
+		return layers[i].MergeRating > layers[j].MergeRating
+	})
+
 	hashes := make([]string, len(layers))
 	for i, l := range layers {
 		l.MediaType = "application/vnd.docker.image.rootfs.diff.tar"