From e8fd6b67348610ff7bbd4585036567b9e56945b7 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sun, 27 Oct 2019 18:33:57 +0100 Subject: refactor(server): Change setup to create new storage backends --- tools/nixery/server/builder/builder.go | 4 ---- tools/nixery/server/builder/cache.go | 2 +- tools/nixery/server/config/config.go | 19 +++++++++++++++++++ tools/nixery/server/main.go | 20 +++++++++++++++++--- tools/nixery/server/storage/gcs.go | 14 +++++++------- 5 files changed, 44 insertions(+), 15 deletions(-) (limited to 'tools') diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/server/builder/builder.go index e2982b993dac..17ea1b8e6209 100644 --- a/tools/nixery/server/builder/builder.go +++ b/tools/nixery/server/builder/builder.go @@ -27,7 +27,6 @@ import ( "fmt" "io" "io/ioutil" - "net/http" "os" "os/exec" "sort" @@ -45,9 +44,6 @@ import ( // use up is set at a lower point. const LayerBudget int = 94 -// HTTP client to use for direct calls to APIs that are not part of the SDK -var client = &http.Client{} - // State holds the runtime state that is carried around in Nixery and // passed to builder functions. type State struct { diff --git a/tools/nixery/server/builder/cache.go b/tools/nixery/server/builder/cache.go index 2af214cd9188..4aed4b53463f 100644 --- a/tools/nixery/server/builder/cache.go +++ b/tools/nixery/server/builder/cache.go @@ -125,7 +125,7 @@ func manifestFromCache(ctx context.Context, s *State, key string) (json.RawMessa log.WithError(err).WithFields(log.Fields{ "manifest": key, "backend": s.Storage.Name(), - }) + }).Error("failed to fetch manifest from cache") return nil, false } diff --git a/tools/nixery/server/config/config.go b/tools/nixery/server/config/config.go index ad6dff40431f..6cc69fa1f599 100644 --- a/tools/nixery/server/config/config.go +++ b/tools/nixery/server/config/config.go @@ -37,6 +37,13 @@ func getConfig(key, desc, def string) string { return value } +// Backend represents the possible storage backend types +type Backend int + +const ( + GCS = iota +) + // Config holds the Nixery configuration options. type Config struct { Port string // Port on which to launch HTTP server @@ -44,6 +51,7 @@ type Config struct { Timeout string // Timeout for a single Nix builder (seconds) WebDir string // Directory with static web assets PopUrl string // URL to the Nix package popularity count + Backend Backend // Storage backend to use for Nixery } func FromEnv() (Config, error) { @@ -52,11 +60,22 @@ func FromEnv() (Config, error) { return Config{}, err } + var b Backend + switch os.Getenv("NIXERY_STORAGE_BACKEND") { + case "gcs": + b = GCS + default: + log.WithField("values", []string{ + "gcs", + }).Fatal("NIXERY_STORAGE_BUCKET must be set to a supported value") + } + return Config{ Port: getConfig("PORT", "HTTP port", ""), Pkgs: pkgs, Timeout: getConfig("NIX_TIMEOUT", "Nix builder timeout", "60"), WebDir: getConfig("WEB_DIR", "Static web file dir", ""), PopUrl: os.Getenv("NIX_POPULARITY_URL"), + Backend: b, }, nil } diff --git a/tools/nixery/server/main.go b/tools/nixery/server/main.go index 22ed6f1a5e2c..b5d7091ed002 100644 --- a/tools/nixery/server/main.go +++ b/tools/nixery/server/main.go @@ -36,6 +36,7 @@ import ( "github.com/google/nixery/server/builder" "github.com/google/nixery/server/config" "github.com/google/nixery/server/layers" + "github.com/google/nixery/server/storage" log "github.com/sirupsen/logrus" ) @@ -196,6 +197,18 @@ func main() { log.WithError(err).Fatal("failed to load configuration") } + var s storage.Backend + + switch cfg.Backend { + case config.GCS: + s, err = storage.NewGCSBackend() + } + if err != nil { + log.WithError(err).Fatal("failed to initialise storage backend") + } + + log.WithField("backend", s.Name()).Info("initialised storage backend") + ctx := context.Background() cache, err := builder.NewCache() if err != nil { @@ -212,9 +225,10 @@ func main() { } state := builder.State{ - Cache: &cache, - Cfg: cfg, - Pop: pop, + Cache: &cache, + Cfg: cfg, + Pop: pop, + Storage: s, } log.WithFields(log.Fields{ diff --git a/tools/nixery/server/storage/gcs.go b/tools/nixery/server/storage/gcs.go index 1b75722bbb77..feb6d30d681e 100644 --- a/tools/nixery/server/storage/gcs.go +++ b/tools/nixery/server/storage/gcs.go @@ -30,10 +30,10 @@ type GCSBackend struct { // Constructs a new GCS bucket backend based on the configured // environment variables. -func New() (GCSBackend, error) { +func NewGCSBackend() (*GCSBackend, error) { bucket := os.Getenv("GCS_BUCKET") if bucket == "" { - return GCSBackend{}, fmt.Errorf("GCS_BUCKET must be configured for GCS usage") + return nil, fmt.Errorf("GCS_BUCKET must be configured for GCS usage") } ctx := context.Background() @@ -46,16 +46,16 @@ func New() (GCSBackend, error) { if _, err := handle.Attrs(ctx); err != nil { log.WithError(err).WithField("bucket", bucket).Error("could not access configured bucket") - return GCSBackend{}, err + return nil, err } signing, err := signingOptsFromEnv() if err != nil { log.WithError(err).Error("failed to configure GCS bucket signing") - return GCSBackend{}, err + return nil, err } - return GCSBackend{ + return &GCSBackend{ bucket: bucket, handle: handle, signing: signing, @@ -66,7 +66,7 @@ func (b *GCSBackend) Name() string { return "Google Cloud Storage (" + b.bucket + ")" } -func (b *GCSBackend) Persist(path string, f func(io.Writer) (string, int, error)) (string, int, error) { +func (b *GCSBackend) Persist(path string, f func(io.Writer) (string, int64, error)) (string, int64, error) { ctx := context.Background() obj := b.handle.Object(path) w := obj.NewWriter(ctx) @@ -139,7 +139,7 @@ func (b *GCSBackend) Move(old, new string) error { return nil } -func (b *GCSBackend) Serve(digest string, w http.ResponseWriter) error { +func (b *GCSBackend) ServeLayer(digest string, w http.ResponseWriter) error { url, err := b.constructLayerUrl(digest) if err != nil { log.WithError(err).WithFields(log.Fields{ -- cgit 1.4.1