diff options
| -rw-r--r-- | tools/nixery/server/builder/builder.go | 11 | ||||
| -rw-r--r-- | tools/nixery/server/builder/cache.go | 67 | ||||
| -rw-r--r-- | tools/nixery/server/builder/state.go | 24 | ||||
| -rw-r--r-- | tools/nixery/server/config/config.go | 6 | ||||
| -rw-r--r-- | tools/nixery/server/main.go | 13 |
5 files changed, 70 insertions, 51 deletions
diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/server/builder/builder.go index 9f529b226918..e622f815a423 100644 --- a/tools/nixery/server/builder/builder.go +++ b/tools/nixery/server/builder/builder.go @@ -34,6 +34,8 @@ import ( "sort" "strings" + "cloud.google.com/go/storage" + "github.com/google/nixery/config" "github.com/google/nixery/layers" "github.com/google/nixery/manifest" "golang.org/x/oauth2/google" @@ -50,6 +52,15 @@ const gcsScope = "https://www.googleapis.com/auth/devstorage.read_write" // HTTP client to use for direct calls to APIs that are not part of the SDK var client = &http.Client{} +// State holds the runtime state that is carried around in Nixery and +// passed to builder functions. +type State struct { + Bucket *storage.BucketHandle + Cache *LocalCache + Cfg config.Config + Pop layers.Popularity +} + // Image represents the information necessary for building a container image. // This can be either a list of package names (corresponding to keys in the // nixpkgs set) or a Nix expression that results in a *list* of derivations. diff --git a/tools/nixery/server/builder/cache.go b/tools/nixery/server/builder/cache.go index ab0021f6d2ff..060ed9a84b13 100644 --- a/tools/nixery/server/builder/cache.go +++ b/tools/nixery/server/builder/cache.go @@ -20,6 +20,7 @@ import ( "io" "io/ioutil" "log" + "os" "sync" "github.com/google/nixery/manifest" @@ -29,40 +30,64 @@ import ( // manifests and layer uploads. type LocalCache struct { // Manifest cache - mmtx sync.RWMutex - mcache map[string]string + mmtx sync.RWMutex + mdir string // Layer cache lmtx sync.RWMutex lcache map[string]manifest.Entry } -func NewCache() LocalCache { +// Creates an in-memory cache and ensures that the local file path for +// manifest caching exists. +func NewCache() (LocalCache, error) { + path := os.TempDir() + "/nixery" + err := os.MkdirAll(path, 0755) + if err != nil { + return LocalCache{}, err + } + return LocalCache{ - mcache: make(map[string]string), + mdir: path + "/", lcache: make(map[string]manifest.Entry), - } + }, nil } // Retrieve a cached manifest if the build is cacheable and it exists. -func (c *LocalCache) manifestFromLocalCache(key string) (string, bool) { +func (c *LocalCache) manifestFromLocalCache(key string) (json.RawMessage, bool) { c.mmtx.RLock() - path, ok := c.mcache[key] - c.mmtx.RUnlock() + defer c.mmtx.RUnlock() - if !ok { - return "", false + f, err := os.Open(c.mdir + key) + if err != nil { + // TODO(tazjin): Once log levels are available, this + // might warrant a debug log. + return nil, false } + defer f.Close() - return path, true + m, err := ioutil.ReadAll(f) + if err != nil { + log.Printf("Failed to read manifest '%s' from local cache: %s\n", key, err) + return nil, false + } + + return json.RawMessage(m), true } // Adds the result of a manifest build to the local cache, if the // manifest is considered cacheable. -func (c *LocalCache) localCacheManifest(key, path string) { +// +// Manifests can be quite large and are cached on disk instead of in +// memory. +func (c *LocalCache) localCacheManifest(key string, m json.RawMessage) { c.mmtx.Lock() - c.mcache[key] = path - c.mmtx.Unlock() + defer c.mmtx.Unlock() + + err := ioutil.WriteFile(c.mdir+key, []byte(m), 0644) + if err != nil { + log.Printf("Failed to locally cache manifest for '%s': %s\n", key, err) + } } // Retrieve a layer build from the local cache. @@ -84,11 +109,9 @@ func (c *LocalCache) localCacheLayer(key string, e manifest.Entry) { // Retrieve a manifest from the cache(s). First the local cache is // checked, then the GCS-bucket cache. func manifestFromCache(ctx context.Context, s *State, key string) (json.RawMessage, bool) { - // path, cached := s.Cache.manifestFromLocalCache(key) - // if cached { - // return path, true - // } - // TODO: local cache? + if m, cached := s.Cache.manifestFromLocalCache(key); cached { + return m, true + } obj := s.Bucket.Object("manifests/" + key) @@ -110,15 +133,15 @@ func manifestFromCache(ctx context.Context, s *State, key string) (json.RawMessa log.Printf("Failed to read cached manifest for '%s': %s\n", key, err) } - // TODO: locally cache manifest, but the cache needs to be changed + go s.Cache.localCacheManifest(key, m) log.Printf("Retrieved manifest for sha1:%s from GCS\n", key) + return json.RawMessage(m), true } // Add a manifest to the bucket & local caches func cacheManifest(ctx context.Context, s *State, key string, m json.RawMessage) { - // go s.Cache.localCacheManifest(key, path) - // TODO local cache + go s.Cache.localCacheManifest(key, m) obj := s.Bucket.Object("manifests/" + key) w := obj.NewWriter(ctx) diff --git a/tools/nixery/server/builder/state.go b/tools/nixery/server/builder/state.go deleted file mode 100644 index 1c7f58821b6b..000000000000 --- a/tools/nixery/server/builder/state.go +++ /dev/null @@ -1,24 +0,0 @@ -package builder - -import ( - "cloud.google.com/go/storage" - "github.com/google/nixery/config" - "github.com/google/nixery/layers" -) - -// State holds the runtime state that is carried around in Nixery and -// passed to builder functions. -type State struct { - Bucket *storage.BucketHandle - Cache LocalCache - Cfg config.Config - Pop layers.Popularity -} - -func NewState(bucket *storage.BucketHandle, cfg config.Config) State { - return State{ - Bucket: bucket, - Cfg: cfg, - Cache: NewCache(), - } -} diff --git a/tools/nixery/server/config/config.go b/tools/nixery/server/config/config.go index 30f727db1112..84c2b89d13d0 100644 --- a/tools/nixery/server/config/config.go +++ b/tools/nixery/server/config/config.go @@ -71,13 +71,13 @@ type Config struct { PopUrl string // URL to the Nix package popularity count } -func FromEnv() (*Config, error) { +func FromEnv() (Config, error) { pkgs, err := pkgSourceFromEnv() if err != nil { - return nil, err + return Config{}, err } - return &Config{ + return Config{ Bucket: getConfig("BUCKET", "GCS bucket for layer storage", ""), Port: getConfig("PORT", "HTTP port", ""), Pkgs: pkgs, diff --git a/tools/nixery/server/main.go b/tools/nixery/server/main.go index 1ff3a471f94f..aeb70163da6a 100644 --- a/tools/nixery/server/main.go +++ b/tools/nixery/server/main.go @@ -194,8 +194,17 @@ func main() { } ctx := context.Background() - bucket := prepareBucket(ctx, cfg) - state := builder.NewState(bucket, *cfg) + bucket := prepareBucket(ctx, &cfg) + cache, err := builder.NewCache() + if err != nil { + log.Fatalln("Failed to instantiate build cache", err) + } + + state := builder.State{ + Bucket: bucket, + Cache: &cache, + Cfg: cfg, + } log.Printf("Starting Nixery on port %s\n", cfg.Port) |