// Copyright 2019 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); you may not // use this file except in compliance with the License. You may obtain a copy of // the License at // // https://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the // License for the specific language governing permissions and limitations under // the License. package builder import ( "bytes" "context" "encoding/json" "io" "io/ioutil" "os" "sync" "github.com/google/nixery/server/manifest" log "github.com/sirupsen/logrus" ) // LocalCache implements the structure used for local caching of // manifests and layer uploads. type LocalCache struct { // Manifest cache mmtx sync.RWMutex mdir string // Layer cache lmtx sync.RWMutex lcache map[string]manifest.Entry } // Creates an in-memory cache and ensures that the local file path for // manifest caching exists. func NewCache() (LocalCache, error) { path := os.TempDir() + "/nixery" err := os.MkdirAll(path, 0755) if err != nil { return LocalCache{}, err } return LocalCache{ mdir: path + "/", lcache: make(map[string]manifest.Entry), }, nil } // Retrieve a cached manifest if the build is cacheable and it exists. func (c *LocalCache) manifestFromLocalCache(key string) (json.RawMessage, bool) { c.mmtx.RLock() defer c.mmtx.RUnlock() f, err := os.Open(c.mdir + key) if err != nil { // TODO(tazjin): Once log levels are available, this // might warrant a debug log. return nil, false } defer f.Close() m, err := ioutil.ReadAll(f) if err != nil { log.Printf("Failed to read manifest '%s' from local cache: %s\n", key, err) return nil, false } return json.RawMessage(m), true } // Adds the result of a manifest build to the local cache, if the // manifest is considered cacheable. // // Manifests can be quite large and are cached on disk instead of in // memory. func (c *LocalCache) localCacheManifest(key string, m json.RawMessage) { c.mmtx.Lock() defer c.mmtx.Unlock() err := ioutil.WriteFile(c.mdir+key, []byte(m), 0644) if err != nil { log.Printf("Failed to locally cache manifest for '%s': %s\n", key, err) } } // Retrieve a layer build from the local cache. func (c *LocalCache) layerFromLocalCache(key string) (*manifest.Entry, bool) { c.lmtx.RLock() e, ok := c.lcache[key] c.lmtx.RUnlock() return &e, ok } // Add a layer build result to the local cache. func (c *LocalCache) localCacheLayer(key string, e manifest.Entry) { c.lmtx.Lock() c.lcache[key] = e c.lmtx.Unlock() } // Retrieve a manifest from the cache(s). First the local cache is // checked, then the GCS-bucket cache. func manifestFromCache(ctx context.Context, s *State, key string) (json.RawMessage, bool) { if m, cached := s.Cache.manifestFromLocalCache(key); cached { return m, true } obj := s.Bucket.Object("manifests/" + key) // Probe whether the file exists before trying to fetch it. _, err := obj.Attrs(ctx) if err != nil { return nil, false } r, err := obj.NewReader(ctx) if err != nil { log.Printf("Failed to retrieve manifest '%s' from cache: %s\n", key, err) return nil, false } defer r.Close() m, err := ioutil.ReadAll(r) if err != nil { log.Printf("Failed to read cached manifest for '%s': %s\n", key, err) } go s.Cache.localCacheManifest(key, m) log.Printf("Retrieved manifest for sha1:%s from GCS\n", key) return json.RawMessage(m), true } // Add a manifest to the bucket & local caches func cacheManifest(ctx context.Context, s *State, key string, m json.RawMessage) { go s.Cache.localCacheManifest(key, m) obj := s.Bucket.Object("manifests/" + key) w := obj.NewWriter(ctx) r := bytes.NewReader([]byte(m)) size, err := io.Copy(w, r) if err != nil { log.Printf("failed to cache manifest sha1:%s: %s\n", key, err) return } if err = w.Close(); err != nil { log.Printf("failed to cache manifest sha1:%s: %s\n", key, err) return } log.Printf("Cached manifest sha1:%s (%v bytes written)\n", key, size) } // Retrieve a layer build from the cache, first checking the local // cache followed by the bucket cache. func layerFromCache(ctx context.Context, s *State, key string) (*manifest.Entry, bool) { if entry, cached := s.Cache.layerFromLocalCache(key); cached { return entry, true } obj := s.Bucket.Object("builds/" + key) _, err := obj.Attrs(ctx) if err != nil { return nil, false } r, err := obj.NewReader(ctx) if err != nil { log.Printf("Failed to retrieve layer build '%s' from cache: %s\n", key, err) return nil, false } defer r.Close() jb := bytes.NewBuffer([]byte{}) _, err = io.Copy(jb, r) if err != nil { log.Printf("Failed to read layer build '%s' from cache: %s\n", key, err) return nil, false } var entry manifest.Entry err = json.Unmarshal(jb.Bytes(), &entry) if err != nil { log.Printf("Failed to unmarshal layer build '%s' from cache: %s\n", key, err) return nil, false } go s.Cache.localCacheLayer(key, entry) return &entry, true } func cacheLayer(ctx context.Context, s *State, key string, entry manifest.Entry) { s.Cache.localCacheLayer(key, entry) obj := s.Bucket.Object("builds/" + key) j, _ := json.Marshal(&entry) w := obj.NewWriter(ctx) _, err := io.Copy(w, bytes.NewReader(j)) if err != nil { log.Printf("failed to cache build '%s': %s\n", key, err) return } if err = w.Close(); err != nil { log.Printf("failed to cache build '%s': %s\n", key, err) return } }