From f6b40ed6c78a69dd417bd9e0f64a207904755af4 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 3 Oct 2019 12:09:24 +0100 Subject: refactor(server): Cache manifest entries for layer builds MD5 hash checking is no longer performed by Nixery (it does not seem to be necessary), hence the layer cache now only keeps the SHA256 hash and size in the form of the manifest entry. This makes it possible to restructure the builder code to perform cache-fetching and cache-populating for layers in the same place. --- tools/nixery/server/builder/cache.go | 56 +++++++++++++++----------------- tools/nixery/server/manifest/manifest.go | 2 +- 2 files changed, 27 insertions(+), 31 deletions(-) (limited to 'tools') diff --git a/tools/nixery/server/builder/cache.go b/tools/nixery/server/builder/cache.go index a5cbbf6ce4..ab0021f6d2 100644 --- a/tools/nixery/server/builder/cache.go +++ b/tools/nixery/server/builder/cache.go @@ -21,12 +21,9 @@ import ( "io/ioutil" "log" "sync" -) -type Build struct { - SHA256 string `json:"sha256"` - MD5 string `json:"md5"` -} + "github.com/google/nixery/manifest" +) // LocalCache implements the structure used for local caching of // manifests and layer uploads. @@ -37,13 +34,13 @@ type LocalCache struct { // Layer cache lmtx sync.RWMutex - lcache map[string]Build + lcache map[string]manifest.Entry } func NewCache() LocalCache { return LocalCache{ mcache: make(map[string]string), - lcache: make(map[string]Build), + lcache: make(map[string]manifest.Entry), } } @@ -68,19 +65,19 @@ func (c *LocalCache) localCacheManifest(key, path string) { c.mmtx.Unlock() } -// Retrieve a cached build from the local cache. -func (c *LocalCache) buildFromLocalCache(key string) (*Build, bool) { +// Retrieve a layer build from the local cache. +func (c *LocalCache) layerFromLocalCache(key string) (*manifest.Entry, bool) { c.lmtx.RLock() - b, ok := c.lcache[key] + e, ok := c.lcache[key] c.lmtx.RUnlock() - return &b, ok + return &e, ok } -// Add a build result to the local cache. -func (c *LocalCache) localCacheBuild(key string, b Build) { +// Add a layer build result to the local cache. +func (c *LocalCache) localCacheLayer(key string, e manifest.Entry) { c.lmtx.Lock() - c.lcache[key] = b + c.lcache[key] = e c.lmtx.Unlock() } @@ -141,12 +138,11 @@ func cacheManifest(ctx context.Context, s *State, key string, m json.RawMessage) log.Printf("Cached manifest sha1:%s (%v bytes written)\n", key, size) } -// Retrieve a build from the cache, first checking the local cache -// followed by the bucket cache. -func buildFromCache(ctx context.Context, s *State, key string) (*Build, bool) { - build, cached := s.Cache.buildFromLocalCache(key) - if cached { - return build, true +// Retrieve a layer build from the cache, first checking the local +// cache followed by the bucket cache. +func layerFromCache(ctx context.Context, s *State, key string) (*manifest.Entry, bool) { + if entry, cached := s.Cache.layerFromLocalCache(key); cached { + return entry, true } obj := s.Bucket.Object("builds/" + key) @@ -157,7 +153,7 @@ func buildFromCache(ctx context.Context, s *State, key string) (*Build, bool) { r, err := obj.NewReader(ctx) if err != nil { - log.Printf("Failed to retrieve build '%s' from cache: %s\n", key, err) + log.Printf("Failed to retrieve layer build '%s' from cache: %s\n", key, err) return nil, false } defer r.Close() @@ -165,27 +161,27 @@ func buildFromCache(ctx context.Context, s *State, key string) (*Build, bool) { jb := bytes.NewBuffer([]byte{}) _, err = io.Copy(jb, r) if err != nil { - log.Printf("Failed to read build '%s' from cache: %s\n", key, err) + log.Printf("Failed to read layer build '%s' from cache: %s\n", key, err) return nil, false } - var b Build - err = json.Unmarshal(jb.Bytes(), &build) + var entry manifest.Entry + err = json.Unmarshal(jb.Bytes(), &entry) if err != nil { - log.Printf("Failed to unmarshal build '%s' from cache: %s\n", key, err) + log.Printf("Failed to unmarshal layer build '%s' from cache: %s\n", key, err) return nil, false } - go s.Cache.localCacheBuild(key, b) - return &b, true + go s.Cache.localCacheLayer(key, entry) + return &entry, true } -func cacheBuild(ctx context.Context, s *State, key string, build Build) { - go s.Cache.localCacheBuild(key, build) +func cacheLayer(ctx context.Context, s *State, key string, entry manifest.Entry) { + s.Cache.localCacheLayer(key, entry) obj := s.Bucket.Object("builds/" + key) - j, _ := json.Marshal(&build) + j, _ := json.Marshal(&entry) w := obj.NewWriter(ctx) diff --git a/tools/nixery/server/manifest/manifest.go b/tools/nixery/server/manifest/manifest.go index dd447796cc..61d280a7fb 100644 --- a/tools/nixery/server/manifest/manifest.go +++ b/tools/nixery/server/manifest/manifest.go @@ -25,7 +25,7 @@ const ( ) type Entry struct { - MediaType string `json:"mediaType"` + MediaType string `json:"mediaType,omitempty"` Size int64 `json:"size"` Digest string `json:"digest"` } -- cgit 1.4.1