From 4a58b0ab4d21473723834dec651c876da2dec220 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Mon, 9 Sep 2019 16:41:52 +0100 Subject: feat(server): Cache built manifests to the GCS bucket Caches manifests under `manifests/$cacheKey` in the GCS bucket and introduces two-tiered retrieval of manifests from the caches (local first, bucket second). There is some cleanup to be done in this code, but the initial version works. --- tools/nixery/server/builder/builder.go | 6 +- tools/nixery/server/builder/cache.go | 111 +++++++++++++++++++++++++++------ tools/nixery/server/main.go | 2 +- 3 files changed, 96 insertions(+), 23 deletions(-) (limited to 'tools') diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/server/builder/builder.go index cfe03511f68e..dd26ccc310aa 100644 --- a/tools/nixery/server/builder/builder.go +++ b/tools/nixery/server/builder/builder.go @@ -109,8 +109,8 @@ func convenienceNames(packages []string) []string { // Call out to Nix and request that an image be built. Nix will, upon success, // return a manifest for the container image. -func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, image *Image, bucket *storage.BucketHandle) (*BuildResult, error) { - resultFile, cached := cache.manifestFromCache(cfg.Pkgs, image) +func BuildImage(ctx *context.Context, cfg *config.Config, cache *LocalCache, image *Image, bucket *storage.BucketHandle) (*BuildResult, error) { + resultFile, cached := manifestFromCache(ctx, bucket, cfg.Pkgs, cache, image) if !cached { packages, err := json.Marshal(image.Packages) @@ -158,7 +158,7 @@ func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, ima log.Println("Finished Nix image build") resultFile = strings.TrimSpace(string(stdout)) - cache.cacheManifest(cfg.Pkgs, image, resultFile) + cacheManifest(ctx, bucket, cfg.Pkgs, cache, image, resultFile) } buildOutput, err := ioutil.ReadFile(resultFile) diff --git a/tools/nixery/server/builder/cache.go b/tools/nixery/server/builder/cache.go index 8ade88037265..32f55e3a681c 100644 --- a/tools/nixery/server/builder/cache.go +++ b/tools/nixery/server/builder/cache.go @@ -14,13 +14,21 @@ package builder import ( - "github.com/google/nixery/config" + "context" + "io" + "log" + "os" "sync" + + "cloud.google.com/go/storage" + "github.com/google/nixery/config" ) type void struct{} -type BuildCache struct { +// LocalCache implements the structure used for local caching of +// manifests and layer uploads. +type LocalCache struct { mmtx sync.RWMutex mcache map[string]string @@ -28,8 +36,8 @@ type BuildCache struct { lcache map[string]void } -func NewCache() BuildCache { - return BuildCache{ +func NewCache() LocalCache { + return LocalCache{ mcache: make(map[string]string), lcache: make(map[string]void), } @@ -38,7 +46,7 @@ func NewCache() BuildCache { // Has this layer hash already been seen by this Nixery instance? If // yes, we can skip upload checking and such because it has already // been done. -func (c *BuildCache) hasSeenLayer(hash string) bool { +func (c *LocalCache) hasSeenLayer(hash string) bool { c.lmtx.RLock() defer c.lmtx.RUnlock() _, seen := c.lcache[hash] @@ -46,19 +54,14 @@ func (c *BuildCache) hasSeenLayer(hash string) bool { } // Layer has now been seen and should be stored. -func (c *BuildCache) sawLayer(hash string) { +func (c *LocalCache) sawLayer(hash string) { c.lmtx.Lock() defer c.lmtx.Unlock() c.lcache[hash] = void{} } // Retrieve a cached manifest if the build is cacheable and it exists. -func (c *BuildCache) manifestFromCache(src config.PkgSource, image *Image) (string, bool) { - key := src.CacheKey(image.Packages, image.Tag) - if key == "" { - return "", false - } - +func (c *LocalCache) manifestFromLocalCache(key string) (string, bool) { c.mmtx.RLock() path, ok := c.mcache[key] c.mmtx.RUnlock() @@ -70,15 +73,85 @@ func (c *BuildCache) manifestFromCache(src config.PkgSource, image *Image) (stri return path, true } -// Adds the result of a manifest build to the cache, if the manifest -// is considered cacheable. -func (c *BuildCache) cacheManifest(src config.PkgSource, image *Image, path string) { - key := src.CacheKey(image.Packages, image.Tag) +// Adds the result of a manifest build to the local cache, if the +// manifest is considered cacheable. +func (c *LocalCache) localCacheManifest(key, path string) { + c.mmtx.Lock() + c.mcache[key] = path + c.mmtx.Unlock() +} + +// Retrieve a manifest from the cache(s). First the local cache is +// checked, then the GCS-bucket cache. +func manifestFromCache(ctx *context.Context, bucket *storage.BucketHandle, pkgs config.PkgSource, cache *LocalCache, image *Image) (string, bool) { + key := pkgs.CacheKey(image.Packages, image.Tag) + if key == "" { + return "", false + } + + path, cached := cache.manifestFromLocalCache(key) + if cached { + return path, true + } + + obj := bucket.Object("manifests/" + key) + + // Probe whether the file exists before trying to fetch it. + _, err := obj.Attrs(*ctx) + if err != nil { + return "", false + } + + r, err := obj.NewReader(*ctx) + if err != nil { + log.Printf("Failed to retrieve manifest '%s' from cache: %s\n", key, err) + return "", false + } + defer r.Close() + + path = os.TempDir() + "/" + key + f, _ := os.Create(path) + defer f.Close() + + _, err = io.Copy(f, r) + if err != nil { + log.Printf("Failed to read cached manifest for '%s': %s\n", key, err) + } + + log.Printf("Retrieved manifest for '%s' (%s) from GCS\n", image.Name, key) + cache.localCacheManifest(key, path) + + return path, true +} + +func cacheManifest(ctx *context.Context, bucket *storage.BucketHandle, pkgs config.PkgSource, cache *LocalCache, image *Image, path string) { + key := pkgs.CacheKey(image.Packages, image.Tag) if key == "" { return } - c.mmtx.Lock() - c.mcache[key] = path - c.mmtx.Unlock() + cache.localCacheManifest(key, path) + + obj := bucket.Object("manifests/" + key) + w := obj.NewWriter(*ctx) + + f, err := os.Open(path) + if err != nil { + log.Printf("failed to open '%s' manifest for cache upload: %s\n", image.Name, err) + return + } + defer f.Close() + + size, err := io.Copy(w, f) + if err != nil { + log.Printf("failed to cache manifest sha1:%s: %s\n", key, err) + return + } + + if err = w.Close(); err != nil { + log.Printf("failed to cache manifest sha1:%s: %s\n", key, err) + return + } + + log.Printf("Cached manifest sha1:%s (%v bytes written)\n", key, size) } diff --git a/tools/nixery/server/main.go b/tools/nixery/server/main.go index 49db1cdf8a5f..9242a3731af0 100644 --- a/tools/nixery/server/main.go +++ b/tools/nixery/server/main.go @@ -125,7 +125,7 @@ type registryHandler struct { cfg *config.Config ctx *context.Context bucket *storage.BucketHandle - cache *builder.BuildCache + cache *builder.LocalCache } func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { -- cgit 1.4.1