From cc35bf0fc3a900dccf4f9edcc581cadb5956c439 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 29 Oct 2020 16:13:53 +0100 Subject: feat(storage): Add support for content-types (GCS only) Extends storage.Persist to accept a Content-Type argument, which in the GCS backend is persisted with the object to ensure that the object is served back with this content-type. This is not yet implemented for the filesystem backend, where the parameter is simply ignored. This should help in the case of clients which expect the returned objects to have content-types set when, for example, fetching layers by digest. --- tools/nixery/builder/builder.go | 2 +- tools/nixery/builder/cache.go | 4 ++-- tools/nixery/main.go | 3 ++- tools/nixery/manifest/manifest.go | 8 ++++---- tools/nixery/storage/filesystem.go | 3 ++- tools/nixery/storage/gcs.go | 25 ++++++++++++++++++++++--- tools/nixery/storage/storage.go | 2 +- 7 files changed, 34 insertions(+), 13 deletions(-) (limited to 'tools/nixery') diff --git a/tools/nixery/builder/builder.go b/tools/nixery/builder/builder.go index 028bcc57690a..115f1e37ef32 100644 --- a/tools/nixery/builder/builder.go +++ b/tools/nixery/builder/builder.go @@ -420,7 +420,7 @@ func (b *byteCounter) Write(p []byte) (n int, err error) { // image manifest. func uploadHashLayer(ctx context.Context, s *State, key string, lw layerWriter) (*manifest.Entry, error) { path := "staging/" + key - sha256sum, size, err := s.Storage.Persist(ctx, path, func(sw io.Writer) (string, int64, error) { + sha256sum, size, err := s.Storage.Persist(ctx, path, manifest.LayerType, func(sw io.Writer) (string, int64, error) { // Sets up a "multiwriter" that simultaneously runs both hash // algorithms and uploads to the storage backend. shasum := sha256.New() diff --git a/tools/nixery/builder/cache.go b/tools/nixery/builder/cache.go index a4ebe03e1c94..35b563e52496 100644 --- a/tools/nixery/builder/cache.go +++ b/tools/nixery/builder/cache.go @@ -152,7 +152,7 @@ func cacheManifest(ctx context.Context, s *State, key string, m json.RawMessage) go s.Cache.localCacheManifest(key, m) path := "manifests/" + key - _, size, err := s.Storage.Persist(ctx, path, func(w io.Writer) (string, int64, error) { + _, size, err := s.Storage.Persist(ctx, path, manifest.ManifestType, func(w io.Writer) (string, int64, error) { size, err := io.Copy(w, bytes.NewReader([]byte(m))) return "", size, err }) @@ -220,7 +220,7 @@ func cacheLayer(ctx context.Context, s *State, key string, entry manifest.Entry) j, _ := json.Marshal(&entry) path := "builds/" + key - _, _, err := s.Storage.Persist(ctx, path, func(w io.Writer) (string, int64, error) { + _, _, err := s.Storage.Persist(ctx, path, "", func(w io.Writer) (string, int64, error) { size, err := io.Copy(w, bytes.NewReader(j)) return "", size, err }) diff --git a/tools/nixery/main.go b/tools/nixery/main.go index 39cfbc525528..d94d51b4681e 100644 --- a/tools/nixery/main.go +++ b/tools/nixery/main.go @@ -38,6 +38,7 @@ import ( "github.com/google/nixery/builder" "github.com/google/nixery/config" "github.com/google/nixery/logs" + mf "github.com/google/nixery/manifest" "github.com/google/nixery/storage" log "github.com/sirupsen/logrus" ) @@ -170,7 +171,7 @@ func (h *registryHandler) serveManifestTag(w http.ResponseWriter, r *http.Reques path := "layers/" + sha256sum ctx := context.TODO() - _, _, err = h.state.Storage.Persist(ctx, path, func(sw io.Writer) (string, int64, error) { + _, _, err = h.state.Storage.Persist(ctx, path, mf.ManifestType, func(sw io.Writer) (string, int64, error) { // We already know the hash, so no additional hash needs to be // constructed here. written, err := sw.Write(manifest) diff --git a/tools/nixery/manifest/manifest.go b/tools/nixery/manifest/manifest.go index 0d36826fb7e5..e499920075f0 100644 --- a/tools/nixery/manifest/manifest.go +++ b/tools/nixery/manifest/manifest.go @@ -28,8 +28,8 @@ const ( schemaVersion = 2 // media types - manifestType = "application/vnd.docker.distribution.manifest.v2+json" - layerType = "application/vnd.docker.image.rootfs.diff.tar.gzip" + ManifestType = "application/vnd.docker.distribution.manifest.v2+json" + LayerType = "application/vnd.docker.image.rootfs.diff.tar.gzip" configType = "application/vnd.docker.container.image.v1+json" // image config constants @@ -117,7 +117,7 @@ func Manifest(arch string, layers []Entry) (json.RawMessage, ConfigLayer) { hashes := make([]string, len(layers)) for i, l := range layers { hashes[i] = l.TarHash - l.MediaType = layerType + l.MediaType = LayerType l.TarHash = "" layers[i] = l } @@ -126,7 +126,7 @@ func Manifest(arch string, layers []Entry) (json.RawMessage, ConfigLayer) { m := manifest{ SchemaVersion: schemaVersion, - MediaType: manifestType, + MediaType: ManifestType, Config: Entry{ MediaType: configType, Size: int64(len(c.Config)), diff --git a/tools/nixery/storage/filesystem.go b/tools/nixery/storage/filesystem.go index 926e9257a1e0..bd757587b4db 100644 --- a/tools/nixery/storage/filesystem.go +++ b/tools/nixery/storage/filesystem.go @@ -49,7 +49,8 @@ func (b *FSBackend) Name() string { return fmt.Sprintf("Filesystem (%s)", b.path) } -func (b *FSBackend) Persist(ctx context.Context, key string, f Persister) (string, int64, error) { +// TODO(tazjin): Implement support for persisting content-types for the filesystem backend. +func (b *FSBackend) Persist(ctx context.Context, key, _type string, f Persister) (string, int64, error) { full := path.Join(b.path, key) dir := path.Dir(full) err := os.MkdirAll(dir, 0755) diff --git a/tools/nixery/storage/gcs.go b/tools/nixery/storage/gcs.go index 61b5dea52351..eac34461af76 100644 --- a/tools/nixery/storage/gcs.go +++ b/tools/nixery/storage/gcs.go @@ -80,17 +80,36 @@ func (b *GCSBackend) Name() string { return "Google Cloud Storage (" + b.bucket + ")" } -func (b *GCSBackend) Persist(ctx context.Context, path string, f Persister) (string, int64, error) { +func (b *GCSBackend) Persist(ctx context.Context, path, contentType string, f Persister) (string, int64, error) { obj := b.handle.Object(path) w := obj.NewWriter(ctx) hash, size, err := f(w) if err != nil { - log.WithError(err).WithField("path", path).Error("failed to upload to GCS") + log.WithError(err).WithField("path", path).Error("failed to write to GCS") return hash, size, err } - return hash, size, w.Close() + err = w.Close() + if err != nil { + log.WithError(err).WithField("path", path).Error("failed to complete GCS upload") + return hash, size, err + } + + // GCS natively supports content types for objects, which will be + // used when serving them back. + if contentType != "" { + _, err = obj.Update(ctx, storage.ObjectAttrsToUpdate{ + ContentType: contentType, + }) + + if err != nil { + log.WithError(err).WithField("path", path).Error("failed to update object attrs") + return hash, size, err + } + } + + return hash, size, nil } func (b *GCSBackend) Fetch(ctx context.Context, path string) (io.ReadCloser, error) { diff --git a/tools/nixery/storage/storage.go b/tools/nixery/storage/storage.go index 4040ef08dc9c..fd496f440ae3 100644 --- a/tools/nixery/storage/storage.go +++ b/tools/nixery/storage/storage.go @@ -36,7 +36,7 @@ type Backend interface { // It needs to return the SHA256 hash of the data written as // well as the total number of bytes, as those are required // for the image manifest. - Persist(context.Context, string, Persister) (string, int64, error) + Persist(ctx context.Context, path, contentType string, f Persister) (string, int64, error) // Fetch retrieves data from the storage backend. Fetch(ctx context.Context, path string) (io.ReadCloser, error) -- cgit 1.4.1