From 94e04a76b6d2baf0cc5060c3168a428fae6c28ab Mon Sep 17 00:00:00 2001 From: Jérôme Petazzoni Date: Thu, 22 Apr 2021 16:52:12 +0200 Subject: feat(storage): Store blob content-type in extended attributes After the discussion in #116, this stores the blob content types in extended attributes when using the filesystem backend. If the underlying filesystem doesn't support extended attributes, storing blobs won't work; also, if extended attributes get removed, blobs won't be served anymore. We can relax this behavior if needed (i.e. log errors but still accept to store or serve blobs). However, since the Docker Engine (and possibly other container engines) won't accept to pull images from a registry that doesn't use correct content types for manifest files, it could be argued that it's better to give a hard fail. (Otherwise, the container engine gives cryptic error messages like "missing signature key".) I can change that behavior (and log errors but still store/serve blobs to the filesystem) if you think it's better. --- tools/nixery/default.nix | 2 +- tools/nixery/go.mod | 1 + tools/nixery/go.sum | 3 +++ tools/nixery/storage/filesystem.go | 17 +++++++++++++++-- 4 files changed, 20 insertions(+), 3 deletions(-) (limited to 'tools/nixery') diff --git a/tools/nixery/default.nix b/tools/nixery/default.nix index a5cc61e7e2..3900353751 100644 --- a/tools/nixery/default.nix +++ b/tools/nixery/default.nix @@ -38,7 +38,7 @@ let doCheck = true; # Needs to be updated after every modification of go.mod/go.sum - vendorSha256 = "1ff0kfww6fy6pnvyva7x8cc6l1d12aafps48wrkwawk2qjy9a8b9"; + vendorSha256 = "1adjav0dxb97ws0w2k50rhk6r46wvfry6aj4sik3ninl525kd15s"; buildFlagsArray = [ "-ldflags=-s -w -X main.version=${nixery-commit-hash}" diff --git a/tools/nixery/go.mod b/tools/nixery/go.mod index e24b2b6ff5..3b819a7965 100644 --- a/tools/nixery/go.mod +++ b/tools/nixery/go.mod @@ -5,6 +5,7 @@ go 1.15 require ( cloud.google.com/go/storage v1.15.0 github.com/google/go-cmp v0.5.5 + github.com/pkg/xattr v0.4.3 github.com/sirupsen/logrus v1.8.1 golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c gonum.org/v1/gonum v0.9.1 diff --git a/tools/nixery/go.sum b/tools/nixery/go.sum index 4930796837..812babcf3e 100644 --- a/tools/nixery/go.sum +++ b/tools/nixery/go.sum @@ -158,6 +158,8 @@ github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2 github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/xattr v0.4.3 h1:5Jx4GCg5ABtqWZH8WLzeI4fOtM1HyX4RBawuCoua1es= +github.com/pkg/xattr v0.4.3/go.mod h1:sBD3RAqlr8Q+RC3FutZcikpT8nyDrIEEBw2J744gVWs= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -322,6 +324,7 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/tools/nixery/storage/filesystem.go b/tools/nixery/storage/filesystem.go index bd757587b4..2be3457f32 100644 --- a/tools/nixery/storage/filesystem.go +++ b/tools/nixery/storage/filesystem.go @@ -23,6 +23,7 @@ import ( "os" "path" + "github.com/pkg/xattr" log "github.com/sirupsen/logrus" ) @@ -49,8 +50,7 @@ func (b *FSBackend) Name() string { return fmt.Sprintf("Filesystem (%s)", b.path) } -// TODO(tazjin): Implement support for persisting content-types for the filesystem backend. -func (b *FSBackend) Persist(ctx context.Context, key, _type string, f Persister) (string, int64, error) { +func (b *FSBackend) Persist(ctx context.Context, key, contentType string, f Persister) (string, int64, error) { full := path.Join(b.path, key) dir := path.Dir(full) err := os.MkdirAll(dir, 0755) @@ -66,6 +66,12 @@ func (b *FSBackend) Persist(ctx context.Context, key, _type string, f Persister) } defer file.Close() + err = xattr.Set(full, "user.mime_type", []byte(contentType)) + if err != nil { + log.WithError(err).WithField("file", full).Error("failed to store file type in xattrs") + return "", 0, err + } + return f(file) } @@ -92,6 +98,13 @@ func (b *FSBackend) Serve(digest string, r *http.Request, w http.ResponseWriter) "path": p, }).Info("serving blob from filesystem") + contentType, err := xattr.Get(p, "user.mime_type") + if err != nil { + log.WithError(err).WithField("file", p).Error("failed to read file type from xattrs") + return err + } + w.Header().Add("Content-Type", string(contentType)) + http.ServeFile(w, r, p) return nil } -- cgit 1.4.1