From 07ef06dcfadef108cfb43df7610db710568a1c45 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sat, 3 Aug 2019 01:21:21 +0100 Subject: feat(go): Support signed GCS URLs with static keys Google Cloud Storage supports granting access to protected objects via time-restricted URLs that are cryptographically signed. This makes it possible to store private data in buckets and to distribute it to eligible clients without having to make those clients aware of GCS authentication methods. Nixery now uses this feature to sign URLs for GCS buckets when returning layer URLs to clients on image pulls. This means that a private Nixery instance can run a bucket with restricted access just fine. Under the hood Nixery uses a key provided via environment variables to sign the URL with a 5 minute expiration time. This can be set up by adding the following two environment variables: * GCS_SIGNING_KEY: Path to the PEM file containing the signing key. * GCS_SIGNING_ACCOUNT: Account ("e-mail" address) to use for signing. If the variables are not set, the previous behaviour is not modified. --- tools/nixery/main.go | 77 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 20 deletions(-) diff --git a/tools/nixery/main.go b/tools/nixery/main.go index a78250d4c4f8..291cdf52d7e6 100644 --- a/tools/nixery/main.go +++ b/tools/nixery/main.go @@ -23,10 +23,6 @@ // request and a Nix-build is initiated that eventually responds with the // manifest as well as information linking each layer digest to a local // filesystem path. -// -// Nixery caches the filesystem paths and returns the manifest to the client. -// Subsequent requests for layer content per digest are then fulfilled by -// serving the files from disk. package main import ( @@ -42,6 +38,7 @@ import ( "os/exec" "regexp" "strings" + "time" "cloud.google.com/go/storage" ) @@ -98,13 +95,37 @@ func pkgSourceFromEnv() *pkgSource { return nil } +// Load (optional) GCS bucket signing data from the GCS_SIGNING_KEY and +// GCS_SIGNING_ACCOUNT envvars. +func signingOptsFromEnv() *storage.SignedURLOptions { + path := os.Getenv("GCS_SIGNING_KEY") + id := os.Getenv("GCS_SIGNING_ACCOUNT") + + if path == "" || id == "" { + log.Println("GCS URL signing disabled") + return nil + } + + log.Printf("GCS URL signing enabled with account %q\n", id) + k, err := ioutil.ReadFile(path) + if err != nil { + log.Fatalf("Failed to read GCS signing key: %s\n", err) + } + + return &storage.SignedURLOptions{ + GoogleAccessID: id, + PrivateKey: k, + Method: "GET", + } +} + // config holds the Nixery configuration options. type config struct { - bucket string // GCS bucket to cache & serve layers - builder string // Nix derivation for building images - web string // Static files to serve over HTTP - port string // Port on which to launch HTTP server - pkgs *pkgSource // Source for Nix package set + bucket string // GCS bucket to cache & serve layers + signing *storage.SignedURLOptions // Signing options to use for GCS URLs + builder string // Nix derivation for building images + port string // Port on which to launch HTTP server + pkgs *pkgSource // Source for Nix package set } // ManifestMediaType is the Content-Type used for the manifest itself. This @@ -117,10 +138,7 @@ const manifestMediaType string = "application/vnd.docker.distribution.manifest.v // This can be either a list of package names (corresponding to keys in the // nixpkgs set) or a Nix expression that results in a *list* of derivations. type image struct { - // Name of the container image. name string - - // Tag requested (only relevant for package sets from git repositories) tag string // Names of packages to include in the image. These must correspond @@ -294,15 +312,24 @@ func uploadLayer(ctx *context.Context, bucket *storage.BucketHandle, layer strin } // layerRedirect constructs the public URL of the layer object in the Cloud -// Storage bucket and redirects the client there. +// Storage bucket, signs it and redirects the user there. +// +// Signing the URL allows unauthenticated clients to retrieve objects from the +// bucket. // // The Docker client is known to follow redirects, but this might not be true // for all other registry clients. -func layerRedirect(w http.ResponseWriter, cfg *config, digest string) { +func constructLayerUrl(cfg *config, digest string) (string, error) { log.Printf("Redirecting layer '%s' request to bucket '%s'\n", digest, cfg.bucket) - url := fmt.Sprintf("https://storage.googleapis.com/%s/layers/%s", cfg.bucket, digest) - w.Header().Set("Location", url) - w.WriteHeader(303) + object := "layers/" + digest + + if cfg.signing != nil { + opts := *cfg.signing + opts.Expires = time.Now().Add(5 * time.Minute) + return storage.SignedURL(cfg.bucket, object, &opts) + } else { + return ("https://storage.googleapis.com" + cfg.bucket + "/" + object), nil + } } // prepareBucket configures the handle to a Cloud Storage bucket in which @@ -410,7 +437,16 @@ func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { layerMatches := layerRegex.FindStringSubmatch(r.RequestURI) if len(layerMatches) == 3 { digest := layerMatches[2] - layerRedirect(w, h.cfg, digest) + url, err := constructLayerUrl(h.cfg, digest) + + if err != nil { + log.Printf("Failed to sign GCS URL: %s\n", err) + writeError(w, 500, "UNKNOWN", "could not serve layer") + return + } + + w.Header().Set("Location", url) + w.WriteHeader(303) return } @@ -431,9 +467,9 @@ func main() { cfg := &config{ bucket: getConfig("BUCKET", "GCS bucket for layer storage"), builder: getConfig("NIX_BUILDER", "Nix image builder code"), - web: getConfig("WEB_DIR", "Static web file dir"), port: getConfig("PORT", "HTTP port"), pkgs: pkgSourceFromEnv(), + signing: signingOptsFromEnv(), } ctx := context.Background() @@ -449,7 +485,8 @@ func main() { }) // All other roots are served by the static file server. - http.Handle("/", http.FileServer(http.Dir(cfg.web))) + webDir := http.Dir(getConfig("WEB_DIR", "Static web file dir")) + http.Handle("/", http.FileServer(webDir)) log.Fatal(http.ListenAndServe(":"+cfg.port, nil)) } -- cgit 1.4.1