diff options
| -rw-r--r-- | tools/nixery/server/builder/builder.go | 101 | ||||
| -rw-r--r-- | tools/nixery/server/builder/cache.go | 94 | ||||
| -rw-r--r-- | tools/nixery/server/config/config.go | 45 | ||||
| -rw-r--r-- | tools/nixery/server/main.go | 66 | ||||
| -rw-r--r-- | tools/nixery/server/storage/storage.go | 34 |
5 files changed, 111 insertions, 229 deletions
diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/server/builder/builder.go index 2a3aa182ac14..e2982b993dac 100644 --- a/tools/nixery/server/builder/builder.go +++ b/tools/nixery/server/builder/builder.go @@ -28,18 +28,16 @@ import ( "io" "io/ioutil" "net/http" - "net/url" "os" "os/exec" "sort" "strings" - "cloud.google.com/go/storage" "github.com/google/nixery/server/config" "github.com/google/nixery/server/layers" "github.com/google/nixery/server/manifest" + "github.com/google/nixery/server/storage" log "github.com/sirupsen/logrus" - "golang.org/x/oauth2/google" ) // The maximum number of layers in an image is 125. To allow for @@ -47,19 +45,16 @@ import ( // use up is set at a lower point. const LayerBudget int = 94 -// API scope needed for renaming objects in GCS -const gcsScope = "https://www.googleapis.com/auth/devstorage.read_write" - // HTTP client to use for direct calls to APIs that are not part of the SDK var client = &http.Client{} // State holds the runtime state that is carried around in Nixery and // passed to builder functions. type State struct { - Bucket *storage.BucketHandle - Cache *LocalCache - Cfg config.Config - Pop layers.Popularity + Storage storage.Backend + Cache *LocalCache + Cfg config.Config + Pop layers.Popularity } // Image represents the information necessary for building a container image. @@ -349,53 +344,6 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes return entries, nil } -// renameObject renames an object in the specified Cloud Storage -// bucket. -// -// The Go API for Cloud Storage does not support renaming objects, but -// the HTTP API does. The code below makes the relevant call manually. -func renameObject(ctx context.Context, s *State, old, new string) error { - bucket := s.Cfg.Bucket - - creds, err := google.FindDefaultCredentials(ctx, gcsScope) - if err != nil { - return err - } - - token, err := creds.TokenSource.Token() - if err != nil { - return err - } - - // as per https://cloud.google.com/storage/docs/renaming-copying-moving-objects#rename - url := fmt.Sprintf( - "https://www.googleapis.com/storage/v1/b/%s/o/%s/rewriteTo/b/%s/o/%s", - url.PathEscape(bucket), url.PathEscape(old), - url.PathEscape(bucket), url.PathEscape(new), - ) - - req, err := http.NewRequest("POST", url, nil) - req.Header.Add("Authorization", "Bearer "+token.AccessToken) - _, err = client.Do(req) - if err != nil { - return err - } - - // It seems that 'rewriteTo' copies objects instead of - // renaming/moving them, hence a deletion call afterwards is - // required. - if err = s.Bucket.Object(old).Delete(ctx); err != nil { - log.WithError(err).WithFields(log.Fields{ - "new": new, - "old": old, - }).Warn("failed to delete renamed object") - - // this error should not break renaming and is not returned - } - - return nil -} - // layerWriter is the type for functions that can write a layer to the // multiwriter used for uploading & hashing. // @@ -430,33 +378,32 @@ func (b *byteCounter) Write(p []byte) (n int, err error) { // The return value is the layer's SHA256 hash, which is used in the // image manifest. func uploadHashLayer(ctx context.Context, s *State, key string, lw layerWriter) (*manifest.Entry, error) { - staging := s.Bucket.Object("staging/" + key) - - // Sets up a "multiwriter" that simultaneously runs both hash - // algorithms and uploads to the bucket - sw := staging.NewWriter(ctx) - shasum := sha256.New() - counter := &byteCounter{} - multi := io.MultiWriter(sw, shasum, counter) + path := "staging/" + key + sha256sum, size, err := s.Storage.Persist(path, func(sw io.Writer) (string, int64, error) { + // Sets up a "multiwriter" that simultaneously runs both hash + // algorithms and uploads to the storage backend. + shasum := sha256.New() + counter := &byteCounter{} + multi := io.MultiWriter(sw, shasum, counter) + + err := lw(multi) + sha256sum := fmt.Sprintf("%x", shasum.Sum([]byte{})) + + return sha256sum, counter.count, err + }) - err := lw(multi) if err != nil { - log.WithError(err).WithField("layer", key). - Error("failed to create and upload layer") + log.WithError(err).WithFields(log.Fields{ + "layer": key, + "backend": s.Storage.Name(), + }).Error("failed to create and store layer") return nil, err } - if err = sw.Close(); err != nil { - log.WithError(err).WithField("layer", key). - Error("failed to upload layer to staging") - } - - sha256sum := fmt.Sprintf("%x", shasum.Sum([]byte{})) - // Hashes are now known and the object is in the bucket, what // remains is to move it to the correct location and cache it. - err = renameObject(ctx, s, "staging/"+key, "layers/"+sha256sum) + err = s.Storage.Move("staging/"+key, "layers/"+sha256sum) if err != nil { log.WithError(err).WithField("layer", key). Error("failed to move layer from staging") @@ -464,8 +411,6 @@ func uploadHashLayer(ctx context.Context, s *State, key string, lw layerWriter) return nil, err } - size := counter.count - log.WithFields(log.Fields{ "layer": key, "sha256": sha256sum, diff --git a/tools/nixery/server/builder/cache.go b/tools/nixery/server/builder/cache.go index 88bf30de4812..2af214cd9188 100644 --- a/tools/nixery/server/builder/cache.go +++ b/tools/nixery/server/builder/cache.go @@ -114,24 +114,18 @@ func (c *LocalCache) localCacheLayer(key string, e manifest.Entry) { } // Retrieve a manifest from the cache(s). First the local cache is -// checked, then the GCS-bucket cache. +// checked, then the storage backend. func manifestFromCache(ctx context.Context, s *State, key string) (json.RawMessage, bool) { if m, cached := s.Cache.manifestFromLocalCache(key); cached { return m, true } - obj := s.Bucket.Object("manifests/" + key) - - // Probe whether the file exists before trying to fetch it. - _, err := obj.Attrs(ctx) + r, err := s.Storage.Fetch("manifests/" + key) if err != nil { - return nil, false - } - - r, err := obj.NewReader(ctx) - if err != nil { - log.WithError(err).WithField("manifest", key). - Error("failed to retrieve manifest from bucket cache") + log.WithError(err).WithFields(log.Fields{ + "manifest": key, + "backend": s.Storage.Name(), + }) return nil, false } @@ -139,8 +133,10 @@ func manifestFromCache(ctx context.Context, s *State, key string) (json.RawMessa m, err := ioutil.ReadAll(r) if err != nil { - log.WithError(err).WithField("manifest", key). - Error("failed to read cached manifest from bucket") + log.WithError(err).WithFields(log.Fields{ + "manifest": key, + "backend": s.Storage.Name(), + }).Error("failed to read cached manifest from storage backend") return nil, false } @@ -155,21 +151,17 @@ func manifestFromCache(ctx context.Context, s *State, key string) (json.RawMessa func cacheManifest(ctx context.Context, s *State, key string, m json.RawMessage) { go s.Cache.localCacheManifest(key, m) - obj := s.Bucket.Object("manifests/" + key) - w := obj.NewWriter(ctx) - r := bytes.NewReader([]byte(m)) + path := "manifests/" + key + _, size, err := s.Storage.Persist(path, func(w io.Writer) (string, int64, error) { + size, err := io.Copy(w, bytes.NewReader([]byte(m))) + return "", size, err + }) - size, err := io.Copy(w, r) if err != nil { - log.WithError(err).WithField("manifest", key). - Error("failed to cache manifest to GCS") - - return - } - - if err = w.Close(); err != nil { - log.WithError(err).WithField("manifest", key). - Error("failed to cache manifest to GCS") + log.WithError(err).WithFields(log.Fields{ + "manifest": key, + "backend": s.Storage.Name(), + }).Error("failed to cache manifest to storage backend") return } @@ -177,7 +169,8 @@ func cacheManifest(ctx context.Context, s *State, key string, m json.RawMessage) log.WithFields(log.Fields{ "manifest": key, "size": size, - }).Info("cached manifest to GCS") + "backend": s.Storage.Name(), + }).Info("cached manifest to storage backend") } // Retrieve a layer build from the cache, first checking the local @@ -187,16 +180,12 @@ func layerFromCache(ctx context.Context, s *State, key string) (*manifest.Entry, return entry, true } - obj := s.Bucket.Object("builds/" + key) - _, err := obj.Attrs(ctx) - if err != nil { - return nil, false - } - - r, err := obj.NewReader(ctx) + r, err := s.Storage.Fetch("builds/" + key) if err != nil { - log.WithError(err).WithField("layer", key). - Error("failed to retrieve cached layer from GCS") + log.WithError(err).WithFields(log.Fields{ + "layer": key, + "backend": s.Storage.Name(), + }).Warn("failed to retrieve cached layer from storage backend") return nil, false } @@ -205,8 +194,10 @@ func layerFromCache(ctx context.Context, s *State, key string) (*manifest.Entry, jb := bytes.NewBuffer([]byte{}) _, err = io.Copy(jb, r) if err != nil { - log.WithError(err).WithField("layer", key). - Error("failed to read cached layer from GCS") + log.WithError(err).WithFields(log.Fields{ + "layer": key, + "backend": s.Storage.Name(), + }).Error("failed to read cached layer from storage backend") return nil, false } @@ -227,24 +218,19 @@ func layerFromCache(ctx context.Context, s *State, key string) (*manifest.Entry, func cacheLayer(ctx context.Context, s *State, key string, entry manifest.Entry) { s.Cache.localCacheLayer(key, entry) - obj := s.Bucket.Object("builds/" + key) - j, _ := json.Marshal(&entry) + path := "builds/" + key + _, _, err := s.Storage.Persist(path, func(w io.Writer) (string, int64, error) { + size, err := io.Copy(w, bytes.NewReader(j)) + return "", size, err + }) - w := obj.NewWriter(ctx) - - _, err := io.Copy(w, bytes.NewReader(j)) if err != nil { - log.WithError(err).WithField("layer", key). - Error("failed to cache layer") - - return + log.WithError(err).WithFields(log.Fields{ + "layer": key, + "backend": s.Storage.Name(), + }).Error("failed to cache layer") } - if err = w.Close(); err != nil { - log.WithError(err).WithField("layer", key). - Error("failed to cache layer") - - return - } + return } diff --git a/tools/nixery/server/config/config.go b/tools/nixery/server/config/config.go index 6c1baafce8c1..ad6dff40431f 100644 --- a/tools/nixery/server/config/config.go +++ b/tools/nixery/server/config/config.go @@ -18,42 +18,11 @@ package config import ( - "io/ioutil" "os" - "cloud.google.com/go/storage" log "github.com/sirupsen/logrus" - "golang.org/x/oauth2/google" ) -// Configure GCS URL signing in the presence of a service account key -// (toggled if the user has set GOOGLE_APPLICATION_CREDENTIALS). -func signingOptsFromEnv() *storage.SignedURLOptions { - path := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") - if path == "" { - return nil - } - - key, err := ioutil.ReadFile(path) - if err != nil { - log.WithError(err).WithField("file", path).Fatal("failed to read service account key") - } - - conf, err := google.JWTConfigFromJSON(key) - if err != nil { - log.WithError(err).WithField("file", path).Fatal("failed to parse service account key") - } - - log.WithField("account", conf.Email).Info("GCS URL signing enabled") - - return &storage.SignedURLOptions{ - Scheme: storage.SigningSchemeV4, - GoogleAccessID: conf.Email, - PrivateKey: conf.PrivateKey, - Method: "GET", - } -} - func getConfig(key, desc, def string) string { value := os.Getenv(key) if value == "" && def == "" { @@ -70,13 +39,11 @@ func getConfig(key, desc, def string) string { // Config holds the Nixery configuration options. type Config struct { - Bucket string // GCS bucket to cache & serve layers - Signing *storage.SignedURLOptions // Signing options to use for GCS URLs - Port string // Port on which to launch HTTP server - Pkgs PkgSource // Source for Nix package set - Timeout string // Timeout for a single Nix builder (seconds) - WebDir string // Directory with static web assets - PopUrl string // URL to the Nix package popularity count + Port string // Port on which to launch HTTP server + Pkgs PkgSource // Source for Nix package set + Timeout string // Timeout for a single Nix builder (seconds) + WebDir string // Directory with static web assets + PopUrl string // URL to the Nix package popularity count } func FromEnv() (Config, error) { @@ -86,10 +53,8 @@ func FromEnv() (Config, error) { } return Config{ - Bucket: getConfig("BUCKET", "GCS bucket for layer storage", ""), Port: getConfig("PORT", "HTTP port", ""), Pkgs: pkgs, - Signing: signingOptsFromEnv(), Timeout: getConfig("NIX_TIMEOUT", "Nix builder timeout", "60"), WebDir: getConfig("WEB_DIR", "Static web file dir", ""), PopUrl: os.Getenv("NIX_POPULARITY_URL"), diff --git a/tools/nixery/server/main.go b/tools/nixery/server/main.go index f38fab2f2abd..22ed6f1a5e2c 100644 --- a/tools/nixery/server/main.go +++ b/tools/nixery/server/main.go @@ -32,9 +32,7 @@ import ( "io/ioutil" "net/http" "regexp" - "time" - "cloud.google.com/go/storage" "github.com/google/nixery/server/builder" "github.com/google/nixery/server/config" "github.com/google/nixery/server/layers" @@ -59,49 +57,6 @@ var ( layerRegex = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/blobs/sha256:(\w+)$`) ) -// layerRedirect constructs the public URL of the layer object in the Cloud -// Storage bucket, signs it and redirects the user there. -// -// Signing the URL allows unauthenticated clients to retrieve objects from the -// bucket. -// -// The Docker client is known to follow redirects, but this might not be true -// for all other registry clients. -func constructLayerUrl(cfg *config.Config, digest string) (string, error) { - log.WithField("layer", digest).Info("redirecting layer request to bucket") - object := "layers/" + digest - - if cfg.Signing != nil { - opts := *cfg.Signing - opts.Expires = time.Now().Add(5 * time.Minute) - return storage.SignedURL(cfg.Bucket, object, &opts) - } else { - return ("https://storage.googleapis.com/" + cfg.Bucket + "/" + object), nil - } -} - -// prepareBucket configures the handle to a Cloud Storage bucket in which -// individual layers will be stored after Nix builds. Nixery does not directly -// serve layers to registry clients, instead it redirects them to the public -// URLs of the Cloud Storage bucket. -// -// The bucket is required for Nixery to function correctly, hence fatal errors -// are generated in case it fails to be set up correctly. -func prepareBucket(ctx context.Context, cfg *config.Config) *storage.BucketHandle { - client, err := storage.NewClient(ctx) - if err != nil { - log.WithError(err).Fatal("failed to set up Cloud Storage client") - } - - bkt := client.Bucket(cfg.Bucket) - - if _, err := bkt.Attrs(ctx); err != nil { - log.WithError(err).WithField("bucket", cfg.Bucket).Fatal("could not access configured bucket") - } - - return bkt -} - // Downloads the popularity information for the package set from the // URL specified in Nixery's configuration. func downloadPopularity(url string) (layers.Popularity, error) { @@ -218,16 +173,15 @@ func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { layerMatches := layerRegex.FindStringSubmatch(r.RequestURI) if len(layerMatches) == 3 { digest := layerMatches[2] - url, err := constructLayerUrl(&h.state.Cfg, digest) - + storage := h.state.Storage + err := storage.ServeLayer(digest, w) if err != nil { - log.WithError(err).WithField("layer", digest).Error("failed to sign GCS URL") - writeError(w, 500, "UNKNOWN", "could not serve layer") - return + log.WithError(err).WithFields(log.Fields{ + "layer": digest, + "backend": storage.Name(), + }).Error("failed to serve layer from storage backend") } - w.Header().Set("Location", url) - w.WriteHeader(303) return } @@ -243,7 +197,6 @@ func main() { } ctx := context.Background() - bucket := prepareBucket(ctx, &cfg) cache, err := builder.NewCache() if err != nil { log.WithError(err).Fatal("failed to instantiate build cache") @@ -259,10 +212,9 @@ func main() { } state := builder.State{ - Bucket: bucket, - Cache: &cache, - Cfg: cfg, - Pop: pop, + Cache: &cache, + Cfg: cfg, + Pop: pop, } log.WithFields(log.Fields{ diff --git a/tools/nixery/server/storage/storage.go b/tools/nixery/server/storage/storage.go new file mode 100644 index 000000000000..15b8355e6ef5 --- /dev/null +++ b/tools/nixery/server/storage/storage.go @@ -0,0 +1,34 @@ +// Package storage implements an interface that can be implemented by +// storage backends, such as Google Cloud Storage or the local +// filesystem. +package storage + +import ( + "io" + "net/http" +) + +type Backend interface { + // Name returns the name of the storage backend, for use in + // log messages and such. + Name() string + + // Persist provides a user-supplied function with a writer + // that stores data in the storage backend. + // + // It needs to return the SHA256 hash of the data written as + // well as the total number of bytes, as those are required + // for the image manifest. + Persist(string, func(io.Writer) (string, int64, error)) (string, int64, error) + + // Fetch retrieves data from the storage backend. + Fetch(path string) (io.ReadCloser, error) + + // Move renames a path inside the storage backend. This is + // used for staging uploads while calculating their hashes. + Move(old, new string) error + + // Serve provides a handler function to serve HTTP requests + // for layers in the storage backend. + ServeLayer(digest string, w http.ResponseWriter) error +} |