feat(server): Cache built manifests to the GCS bucket

Caches manifests under `manifests/$cacheKey` in the GCS bucket and
introduces two-tiered retrieval of manifests from the caches (local
first, bucket second).

There is some cleanup to be done in this code, but the initial version
works.

3 files changed, 96 insertions, 23 deletions

diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/server/builder/builder.go
index cfe03511f68e..dd26ccc310aa 100644
--- a/tools/nixery/server/builder/builder.go
+++ b/tools/nixery/server/builder/builder.go
@@ -109,8 +109,8 @@ func convenienceNames(packages []string) []string {
 
 // Call out to Nix and request that an image be built. Nix will, upon success,
 // return a manifest for the container image.
-func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, image *Image, bucket *storage.BucketHandle) (*BuildResult, error) {
-	resultFile, cached := cache.manifestFromCache(cfg.Pkgs, image)
+func BuildImage(ctx *context.Context, cfg *config.Config, cache *LocalCache, image *Image, bucket *storage.BucketHandle) (*BuildResult, error) {
+	resultFile, cached := manifestFromCache(ctx, bucket, cfg.Pkgs, cache, image)
 
 	if !cached {
 		packages, err := json.Marshal(image.Packages)
@@ -158,7 +158,7 @@ func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, ima
 		log.Println("Finished Nix image build")
 
 		resultFile = strings.TrimSpace(string(stdout))
-		cache.cacheManifest(cfg.Pkgs, image, resultFile)
+		cacheManifest(ctx, bucket, cfg.Pkgs, cache, image, resultFile)
 	}
 
 	buildOutput, err := ioutil.ReadFile(resultFile)
diff --git a/tools/nixery/server/builder/cache.go b/tools/nixery/server/builder/cache.go
index 8ade88037265..32f55e3a681c 100644
--- a/tools/nixery/server/builder/cache.go
+++ b/tools/nixery/server/builder/cache.go
@@ -14,13 +14,21 @@
 package builder
 
 import (
-	"github.com/google/nixery/config"
+	"context"
+	"io"
+	"log"
+	"os"
 	"sync"
+
+	"cloud.google.com/go/storage"
+	"github.com/google/nixery/config"
 )
 
 type void struct{}
 
-type BuildCache struct {
+// LocalCache implements the structure used for local caching of
+// manifests and layer uploads.
+type LocalCache struct {
 	mmtx   sync.RWMutex
 	mcache map[string]string
 
@@ -28,8 +36,8 @@ type BuildCache struct {
 	lcache map[string]void
 }
 
-func NewCache() BuildCache {
-	return BuildCache{
+func NewCache() LocalCache {
+	return LocalCache{
 		mcache: make(map[string]string),
 		lcache: make(map[string]void),
 	}
@@ -38,7 +46,7 @@ func NewCache() BuildCache {
 // Has this layer hash already been seen by this Nixery instance? If
 // yes, we can skip upload checking and such because it has already
 // been done.
-func (c *BuildCache) hasSeenLayer(hash string) bool {
+func (c *LocalCache) hasSeenLayer(hash string) bool {
 	c.lmtx.RLock()
 	defer c.lmtx.RUnlock()
 	_, seen := c.lcache[hash]
@@ -46,19 +54,14 @@ func (c *BuildCache) hasSeenLayer(hash string) bool {
 }
 
 // Layer has now been seen and should be stored.
-func (c *BuildCache) sawLayer(hash string) {
+func (c *LocalCache) sawLayer(hash string) {
 	c.lmtx.Lock()
 	defer c.lmtx.Unlock()
 	c.lcache[hash] = void{}
 }
 
 // Retrieve a cached manifest if the build is cacheable and it exists.
-func (c *BuildCache) manifestFromCache(src config.PkgSource, image *Image) (string, bool) {
-	key := src.CacheKey(image.Packages, image.Tag)
-	if key == "" {
-		return "", false
-	}
-
+func (c *LocalCache) manifestFromLocalCache(key string) (string, bool) {
 	c.mmtx.RLock()
 	path, ok := c.mcache[key]
 	c.mmtx.RUnlock()
@@ -70,15 +73,85 @@ func (c *BuildCache) manifestFromCache(src config.PkgSource, image *Image) (stri
 	return path, true
 }
 
-// Adds the result of a manifest build to the cache, if the manifest
-// is considered cacheable.
-func (c *BuildCache) cacheManifest(src config.PkgSource, image *Image, path string) {
-	key := src.CacheKey(image.Packages, image.Tag)
+// Adds the result of a manifest build to the local cache, if the
+// manifest is considered cacheable.
+func (c *LocalCache) localCacheManifest(key, path string) {
+	c.mmtx.Lock()
+	c.mcache[key] = path
+	c.mmtx.Unlock()
+}
+
+// Retrieve a manifest from the cache(s). First the local cache is
+// checked, then the GCS-bucket cache.
+func manifestFromCache(ctx *context.Context, bucket *storage.BucketHandle, pkgs config.PkgSource, cache *LocalCache, image *Image) (string, bool) {
+	key := pkgs.CacheKey(image.Packages, image.Tag)
+	if key == "" {
+		return "", false
+	}
+
+	path, cached := cache.manifestFromLocalCache(key)
+	if cached {
+		return path, true
+	}
+
+	obj := bucket.Object("manifests/" + key)
+
+	// Probe whether the file exists before trying to fetch it.
+	_, err := obj.Attrs(*ctx)
+	if err != nil {
+		return "", false
+	}
+
+	r, err := obj.NewReader(*ctx)
+	if err != nil {
+		log.Printf("Failed to retrieve manifest '%s' from cache: %s\n", key, err)
+		return "", false
+	}
+	defer r.Close()
+
+	path = os.TempDir() + "/" + key
+	f, _ := os.Create(path)
+	defer f.Close()
+
+	_, err = io.Copy(f, r)
+	if err != nil {
+		log.Printf("Failed to read cached manifest for '%s': %s\n", key, err)
+	}
+
+	log.Printf("Retrieved manifest for '%s' (%s) from GCS\n", image.Name, key)
+	cache.localCacheManifest(key, path)
+
+	return path, true
+}
+
+func cacheManifest(ctx *context.Context, bucket *storage.BucketHandle, pkgs config.PkgSource, cache *LocalCache, image *Image, path string) {
+	key := pkgs.CacheKey(image.Packages, image.Tag)
 	if key == "" {
 		return
 	}
 
-	c.mmtx.Lock()
-	c.mcache[key] = path
-	c.mmtx.Unlock()
+	cache.localCacheManifest(key, path)
+
+	obj := bucket.Object("manifests/" + key)
+	w := obj.NewWriter(*ctx)
+
+	f, err := os.Open(path)
+	if err != nil {
+		log.Printf("failed to open '%s' manifest for cache upload: %s\n", image.Name, err)
+		return
+	}
+	defer f.Close()
+
+	size, err := io.Copy(w, f)
+	if err != nil {
+		log.Printf("failed to cache manifest sha1:%s: %s\n", key, err)
+		return
+	}
+
+	if err = w.Close(); err != nil {
+		log.Printf("failed to cache manifest sha1:%s: %s\n", key, err)
+		return
+	}
+
+	log.Printf("Cached manifest sha1:%s (%v bytes written)\n", key, size)
 }
diff --git a/tools/nixery/server/main.go b/tools/nixery/server/main.go
index 49db1cdf8a5f..9242a3731af0 100644
--- a/tools/nixery/server/main.go
+++ b/tools/nixery/server/main.go
@@ -125,7 +125,7 @@ type registryHandler struct {
 	cfg    *config.Config
 	ctx    *context.Context
 	bucket *storage.BucketHandle
-	cache  *builder.BuildCache
+	cache  *builder.LocalCache
 }
 
 func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {