about summary refs log tree commit diff
diff options
context:
space:
mode:
authorVincent Ambo <tazjin@google.com>2019-10-03T11·09+0100
committerVincent Ambo <github@tazj.in>2019-10-03T12·21+0100
commitf6b40ed6c78a69dd417bd9e0f64a207904755af4 (patch)
tree164ec793ee5527624b65540075f0a50ec3672066
parent355fe3f5ec05c3c698ea3ba21a5d57454daeceef (diff)
refactor(server): Cache manifest entries for layer builds
MD5 hash checking is no longer performed by Nixery (it does not seem
to be necessary), hence the layer cache now only keeps the SHA256 hash
and size in the form of the manifest entry.

This makes it possible to restructure the builder code to perform
cache-fetching and cache-populating for layers in the same place.
-rw-r--r--tools/nixery/server/builder/cache.go56
-rw-r--r--tools/nixery/server/manifest/manifest.go2
2 files changed, 27 insertions, 31 deletions
diff --git a/tools/nixery/server/builder/cache.go b/tools/nixery/server/builder/cache.go
index a5cbbf6ce469..ab0021f6d2ff 100644
--- a/tools/nixery/server/builder/cache.go
+++ b/tools/nixery/server/builder/cache.go
@@ -21,12 +21,9 @@ import (
 	"io/ioutil"
 	"log"
 	"sync"
-)
 
-type Build struct {
-	SHA256 string `json:"sha256"`
-	MD5    string `json:"md5"`
-}
+	"github.com/google/nixery/manifest"
+)
 
 // LocalCache implements the structure used for local caching of
 // manifests and layer uploads.
@@ -37,13 +34,13 @@ type LocalCache struct {
 
 	// Layer cache
 	lmtx   sync.RWMutex
-	lcache map[string]Build
+	lcache map[string]manifest.Entry
 }
 
 func NewCache() LocalCache {
 	return LocalCache{
 		mcache: make(map[string]string),
-		lcache: make(map[string]Build),
+		lcache: make(map[string]manifest.Entry),
 	}
 }
 
@@ -68,19 +65,19 @@ func (c *LocalCache) localCacheManifest(key, path string) {
 	c.mmtx.Unlock()
 }
 
-// Retrieve a cached build from the local cache.
-func (c *LocalCache) buildFromLocalCache(key string) (*Build, bool) {
+// Retrieve a layer build from the local cache.
+func (c *LocalCache) layerFromLocalCache(key string) (*manifest.Entry, bool) {
 	c.lmtx.RLock()
-	b, ok := c.lcache[key]
+	e, ok := c.lcache[key]
 	c.lmtx.RUnlock()
 
-	return &b, ok
+	return &e, ok
 }
 
-// Add a build result to the local cache.
-func (c *LocalCache) localCacheBuild(key string, b Build) {
+// Add a layer build result to the local cache.
+func (c *LocalCache) localCacheLayer(key string, e manifest.Entry) {
 	c.lmtx.Lock()
-	c.lcache[key] = b
+	c.lcache[key] = e
 	c.lmtx.Unlock()
 }
 
@@ -141,12 +138,11 @@ func cacheManifest(ctx context.Context, s *State, key string, m json.RawMessage)
 	log.Printf("Cached manifest sha1:%s (%v bytes written)\n", key, size)
 }
 
-// Retrieve a build from the cache, first checking the local cache
-// followed by the bucket cache.
-func buildFromCache(ctx context.Context, s *State, key string) (*Build, bool) {
-	build, cached := s.Cache.buildFromLocalCache(key)
-	if cached {
-		return build, true
+// Retrieve a layer build from the cache, first checking the local
+// cache followed by the bucket cache.
+func layerFromCache(ctx context.Context, s *State, key string) (*manifest.Entry, bool) {
+	if entry, cached := s.Cache.layerFromLocalCache(key); cached {
+		return entry, true
 	}
 
 	obj := s.Bucket.Object("builds/" + key)
@@ -157,7 +153,7 @@ func buildFromCache(ctx context.Context, s *State, key string) (*Build, bool) {
 
 	r, err := obj.NewReader(ctx)
 	if err != nil {
-		log.Printf("Failed to retrieve build '%s' from cache: %s\n", key, err)
+		log.Printf("Failed to retrieve layer build '%s' from cache: %s\n", key, err)
 		return nil, false
 	}
 	defer r.Close()
@@ -165,27 +161,27 @@ func buildFromCache(ctx context.Context, s *State, key string) (*Build, bool) {
 	jb := bytes.NewBuffer([]byte{})
 	_, err = io.Copy(jb, r)
 	if err != nil {
-		log.Printf("Failed to read build '%s' from cache: %s\n", key, err)
+		log.Printf("Failed to read layer build '%s' from cache: %s\n", key, err)
 		return nil, false
 	}
 
-	var b Build
-	err = json.Unmarshal(jb.Bytes(), &build)
+	var entry manifest.Entry
+	err = json.Unmarshal(jb.Bytes(), &entry)
 	if err != nil {
-		log.Printf("Failed to unmarshal build '%s' from cache: %s\n", key, err)
+		log.Printf("Failed to unmarshal layer build '%s' from cache: %s\n", key, err)
 		return nil, false
 	}
 
-	go s.Cache.localCacheBuild(key, b)
-	return &b, true
+	go s.Cache.localCacheLayer(key, entry)
+	return &entry, true
 }
 
-func cacheBuild(ctx context.Context, s *State, key string, build Build) {
-	go s.Cache.localCacheBuild(key, build)
+func cacheLayer(ctx context.Context, s *State, key string, entry manifest.Entry) {
+	s.Cache.localCacheLayer(key, entry)
 
 	obj := s.Bucket.Object("builds/" + key)
 
-	j, _ := json.Marshal(&build)
+	j, _ := json.Marshal(&entry)
 
 	w := obj.NewWriter(ctx)
 
diff --git a/tools/nixery/server/manifest/manifest.go b/tools/nixery/server/manifest/manifest.go
index dd447796cc78..61d280a7fbab 100644
--- a/tools/nixery/server/manifest/manifest.go
+++ b/tools/nixery/server/manifest/manifest.go
@@ -25,7 +25,7 @@ const (
 )
 
 type Entry struct {
-	MediaType string `json:"mediaType"`
+	MediaType string `json:"mediaType,omitempty"`
 	Size      int64  `json:"size"`
 	Digest    string `json:"digest"`
 }