diff options
Diffstat (limited to 'tools/nixery/server')
| -rw-r--r-- | tools/nixery/server/builder/archive.go | 116 | ||||
| -rw-r--r-- | tools/nixery/server/builder/builder.go | 521 | ||||
| -rw-r--r-- | tools/nixery/server/builder/builder_test.go | 123 | ||||
| -rw-r--r-- | tools/nixery/server/builder/cache.go | 236 | ||||
| -rw-r--r-- | tools/nixery/server/config/config.go | 84 | ||||
| -rw-r--r-- | tools/nixery/server/config/pkgsource.go | 159 | ||||
| -rw-r--r-- | tools/nixery/server/default.nix | 62 | ||||
| -rw-r--r-- | tools/nixery/server/go-deps.nix | 129 | ||||
| -rw-r--r-- | tools/nixery/server/layers/grouping.go | 361 | ||||
| -rw-r--r-- | tools/nixery/server/logs.go | 119 | ||||
| -rw-r--r-- | tools/nixery/server/main.go | 248 | ||||
| -rw-r--r-- | tools/nixery/server/manifest/manifest.go | 141 | ||||
| -rw-r--r-- | tools/nixery/server/storage/filesystem.go | 96 | ||||
| -rw-r--r-- | tools/nixery/server/storage/gcs.go | 219 | ||||
| -rw-r--r-- | tools/nixery/server/storage/storage.go | 51 |
15 files changed, 0 insertions, 2665 deletions
diff --git a/tools/nixery/server/builder/archive.go b/tools/nixery/server/builder/archive.go deleted file mode 100644 index e0fb76d44b..0000000000 --- a/tools/nixery/server/builder/archive.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. -package builder - -// This file implements logic for walking through a directory and creating a -// tarball of it. -// -// The tarball is written straight to the supplied reader, which makes it -// possible to create an image layer from the specified store paths, hash it and -// upload it in one reading pass. - -import ( - "archive/tar" - "compress/gzip" - "crypto/sha256" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/google/nixery/server/layers" -) - -// Create a new compressed tarball from each of the paths in the list -// and write it to the supplied writer. -// -// The uncompressed tarball is hashed because image manifests must -// contain both the hashes of compressed and uncompressed layers. -func packStorePaths(l *layers.Layer, w io.Writer) (string, error) { - shasum := sha256.New() - gz := gzip.NewWriter(w) - multi := io.MultiWriter(shasum, gz) - t := tar.NewWriter(multi) - - for _, path := range l.Contents { - err := filepath.Walk(path, tarStorePath(t)) - if err != nil { - return "", err - } - } - - if err := t.Close(); err != nil { - return "", err - } - - if err := gz.Close(); err != nil { - return "", err - } - - return fmt.Sprintf("sha256:%x", shasum.Sum([]byte{})), nil -} - -func tarStorePath(w *tar.Writer) filepath.WalkFunc { - return func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - // If the entry is not a symlink or regular file, skip it. - if info.Mode()&os.ModeSymlink == 0 && !info.Mode().IsRegular() { - return nil - } - - // the symlink target is read if this entry is a symlink, as it - // is required when creating the file header - var link string - if info.Mode()&os.ModeSymlink != 0 { - link, err = os.Readlink(path) - if err != nil { - return err - } - } - - header, err := tar.FileInfoHeader(info, link) - if err != nil { - return err - } - - // The name retrieved from os.FileInfo only contains the file's - // basename, but the full path is required within the layer - // tarball. - header.Name = path - if err = w.WriteHeader(header); err != nil { - return err - } - - // At this point, return if no file content needs to be written - if !info.Mode().IsRegular() { - return nil - } - - f, err := os.Open(path) - if err != nil { - return err - } - - if _, err := io.Copy(w, f); err != nil { - return err - } - - f.Close() - - return nil - } -} diff --git a/tools/nixery/server/builder/builder.go b/tools/nixery/server/builder/builder.go deleted file mode 100644 index da9dede1ac..0000000000 --- a/tools/nixery/server/builder/builder.go +++ /dev/null @@ -1,521 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -// Package builder implements the code required to build images via Nix. Image -// build data is cached for up to 24 hours to avoid duplicated calls to Nix -// (which are costly even if no building is performed). -package builder - -import ( - "bufio" - "bytes" - "compress/gzip" - "context" - "crypto/sha256" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "os" - "os/exec" - "sort" - "strings" - - "github.com/google/nixery/server/config" - "github.com/google/nixery/server/layers" - "github.com/google/nixery/server/manifest" - "github.com/google/nixery/server/storage" - log "github.com/sirupsen/logrus" -) - -// The maximum number of layers in an image is 125. To allow for -// extensibility, the actual number of layers Nixery is "allowed" to -// use up is set at a lower point. -const LayerBudget int = 94 - -// State holds the runtime state that is carried around in Nixery and -// passed to builder functions. -type State struct { - Storage storage.Backend - Cache *LocalCache - Cfg config.Config - Pop layers.Popularity -} - -// Architecture represents the possible CPU architectures for which -// container images can be built. -// -// The default architecture is amd64, but support for ARM platforms is -// available within nixpkgs and can be toggled via meta-packages. -type Architecture struct { - // Name of the system tuple to pass to Nix - nixSystem string - - // Name of the architecture as used in the OCI manifests - imageArch string -} - -var amd64 = Architecture{"x86_64-linux", "amd64"} -var arm64 = Architecture{"aarch64-linux", "arm64"} - -// Image represents the information necessary for building a container image. -// This can be either a list of package names (corresponding to keys in the -// nixpkgs set) or a Nix expression that results in a *list* of derivations. -type Image struct { - Name string - Tag string - - // Names of packages to include in the image. These must correspond - // directly to top-level names of Nix packages in the nixpkgs tree. - Packages []string - - // Architecture for which to build the image. Nixery defaults - // this to amd64 if not specified via meta-packages. - Arch *Architecture -} - -// BuildResult represents the data returned from the server to the -// HTTP handlers. Error information is propagated straight from Nix -// for errors inside of the build that should be fed back to the -// client (such as missing packages). -type BuildResult struct { - Error string `json:"error"` - Pkgs []string `json:"pkgs"` - Manifest json.RawMessage `json:"manifest"` -} - -// ImageFromName parses an image name into the corresponding structure which can -// be used to invoke Nix. -// -// It will expand convenience names under the hood (see the `convenienceNames` -// function below) and append packages that are always included (cacert, iana-etc). -// -// Once assembled the image structure uses a sorted representation of -// the name. This is to avoid unnecessarily cache-busting images if -// only the order of requested packages has changed. -func ImageFromName(name string, tag string) Image { - pkgs := strings.Split(name, "/") - arch, expanded := metaPackages(pkgs) - expanded = append(expanded, "cacert", "iana-etc") - - sort.Strings(pkgs) - sort.Strings(expanded) - - return Image{ - Name: strings.Join(pkgs, "/"), - Tag: tag, - Packages: expanded, - Arch: arch, - } -} - -// ImageResult represents the output of calling the Nix derivation -// responsible for preparing an image. -type ImageResult struct { - // These fields are populated in case of an error - Error string `json:"error"` - Pkgs []string `json:"pkgs"` - - // These fields are populated in case of success - Graph layers.RuntimeGraph `json:"runtimeGraph"` - SymlinkLayer struct { - Size int `json:"size"` - TarHash string `json:"tarHash"` - Path string `json:"path"` - } `json:"symlinkLayer"` -} - -// metaPackages expands package names defined by Nixery which either -// include sets of packages or trigger certain image-building -// behaviour. -// -// Meta-packages must be specified as the first packages in an image -// name. -// -// Currently defined meta-packages are: -// -// * `shell`: Includes bash, coreutils and other common command-line tools -// * `arm64`: Causes Nixery to build images for the ARM64 architecture -func metaPackages(packages []string) (*Architecture, []string) { - arch := &amd64 - - var metapkgs []string - lastMeta := 0 - for idx, p := range packages { - if p == "shell" || p == "arm64" { - metapkgs = append(metapkgs, p) - lastMeta = idx + 1 - } else { - break - } - } - - // Chop off the meta-packages from the front of the package - // list - packages = packages[lastMeta:] - - for _, p := range metapkgs { - switch p { - case "shell": - packages = append(packages, "bashInteractive", "coreutils", "moreutils", "nano") - case "arm64": - arch = &arm64 - } - } - - return arch, packages -} - -// logNix logs each output line from Nix. It runs in a goroutine per -// output channel that should be live-logged. -func logNix(image, cmd string, r io.ReadCloser) { - scanner := bufio.NewScanner(r) - for scanner.Scan() { - log.WithFields(log.Fields{ - "image": image, - "cmd": cmd, - }).Info("[nix] " + scanner.Text()) - } -} - -func callNix(program, image string, args []string) ([]byte, error) { - cmd := exec.Command(program, args...) - - outpipe, err := cmd.StdoutPipe() - if err != nil { - return nil, err - } - - errpipe, err := cmd.StderrPipe() - if err != nil { - return nil, err - } - go logNix(program, image, errpipe) - - if err = cmd.Start(); err != nil { - log.WithError(err).WithFields(log.Fields{ - "image": image, - "cmd": program, - }).Error("error invoking Nix") - - return nil, err - } - - log.WithFields(log.Fields{ - "cmd": program, - "image": image, - }).Info("invoked Nix build") - - stdout, _ := ioutil.ReadAll(outpipe) - - if err = cmd.Wait(); err != nil { - log.WithError(err).WithFields(log.Fields{ - "image": image, - "cmd": program, - "stdout": stdout, - }).Info("failed to invoke Nix") - - return nil, err - } - - resultFile := strings.TrimSpace(string(stdout)) - buildOutput, err := ioutil.ReadFile(resultFile) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "image": image, - "file": resultFile, - }).Info("failed to read Nix result file") - - return nil, err - } - - return buildOutput, nil -} - -// Call out to Nix and request metadata for the image to be built. All -// required store paths for the image will be realised, but layers -// will not yet be created from them. -// -// This function is only invoked if the manifest is not found in any -// cache. -func prepareImage(s *State, image *Image) (*ImageResult, error) { - packages, err := json.Marshal(image.Packages) - if err != nil { - return nil, err - } - - srcType, srcArgs := s.Cfg.Pkgs.Render(image.Tag) - - args := []string{ - "--timeout", s.Cfg.Timeout, - "--argstr", "packages", string(packages), - "--argstr", "srcType", srcType, - "--argstr", "srcArgs", srcArgs, - "--argstr", "system", image.Arch.nixSystem, - } - - output, err := callNix("nixery-build-image", image.Name, args) - if err != nil { - // granular error logging is performed in callNix already - return nil, err - } - - log.WithFields(log.Fields{ - "image": image.Name, - "tag": image.Tag, - }).Info("finished image preparation via Nix") - - var result ImageResult - err = json.Unmarshal(output, &result) - if err != nil { - return nil, err - } - - return &result, nil -} - -// Groups layers and checks whether they are present in the cache -// already, otherwise calls out to Nix to assemble layers. -// -// Newly built layers are uploaded to the bucket. Cache entries are -// added only after successful uploads, which guarantees that entries -// retrieved from the cache are present in the bucket. -func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageResult) ([]manifest.Entry, error) { - grouped := layers.Group(&result.Graph, &s.Pop, LayerBudget) - - var entries []manifest.Entry - - // Splits the layers into those which are already present in - // the cache, and those that are missing. - // - // Missing layers are built and uploaded to the storage - // bucket. - for _, l := range grouped { - if entry, cached := layerFromCache(ctx, s, l.Hash()); cached { - entries = append(entries, *entry) - } else { - lh := l.Hash() - - // While packing store paths, the SHA sum of - // the uncompressed layer is computed and - // written to `tarhash`. - // - // TODO(tazjin): Refactor this to make the - // flow of data cleaner. - var tarhash string - lw := func(w io.Writer) error { - var err error - tarhash, err = packStorePaths(&l, w) - return err - } - - entry, err := uploadHashLayer(ctx, s, lh, lw) - if err != nil { - return nil, err - } - entry.MergeRating = l.MergeRating - entry.TarHash = tarhash - - var pkgs []string - for _, p := range l.Contents { - pkgs = append(pkgs, layers.PackageFromPath(p)) - } - - log.WithFields(log.Fields{ - "layer": lh, - "packages": pkgs, - "tarhash": tarhash, - }).Info("created image layer") - - go cacheLayer(ctx, s, l.Hash(), *entry) - entries = append(entries, *entry) - } - } - - // Symlink layer (built in the first Nix build) needs to be - // included here manually: - slkey := result.SymlinkLayer.TarHash - entry, err := uploadHashLayer(ctx, s, slkey, func(w io.Writer) error { - f, err := os.Open(result.SymlinkLayer.Path) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "image": image.Name, - "tag": image.Tag, - "layer": slkey, - }).Error("failed to open symlink layer") - - return err - } - defer f.Close() - - gz := gzip.NewWriter(w) - _, err = io.Copy(gz, f) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "image": image.Name, - "tag": image.Tag, - "layer": slkey, - }).Error("failed to upload symlink layer") - - return err - } - - return gz.Close() - }) - - if err != nil { - return nil, err - } - - entry.TarHash = "sha256:" + result.SymlinkLayer.TarHash - go cacheLayer(ctx, s, slkey, *entry) - entries = append(entries, *entry) - - return entries, nil -} - -// layerWriter is the type for functions that can write a layer to the -// multiwriter used for uploading & hashing. -// -// This type exists to avoid duplication between the handling of -// symlink layers and store path layers. -type layerWriter func(w io.Writer) error - -// byteCounter is a special io.Writer that counts all bytes written to -// it and does nothing else. -// -// This is required because the ad-hoc writing of tarballs leaves no -// single place to count the final tarball size otherwise. -type byteCounter struct { - count int64 -} - -func (b *byteCounter) Write(p []byte) (n int, err error) { - b.count += int64(len(p)) - return len(p), nil -} - -// Upload a layer tarball to the storage bucket, while hashing it at -// the same time. The supplied function is expected to provide the -// layer data to the writer. -// -// The initial upload is performed in a 'staging' folder, as the -// SHA256-hash is not yet available when the upload is initiated. -// -// After a successful upload, the file is moved to its final location -// in the bucket and the build cache is populated. -// -// The return value is the layer's SHA256 hash, which is used in the -// image manifest. -func uploadHashLayer(ctx context.Context, s *State, key string, lw layerWriter) (*manifest.Entry, error) { - path := "staging/" + key - sha256sum, size, err := s.Storage.Persist(ctx, path, func(sw io.Writer) (string, int64, error) { - // Sets up a "multiwriter" that simultaneously runs both hash - // algorithms and uploads to the storage backend. - shasum := sha256.New() - counter := &byteCounter{} - multi := io.MultiWriter(sw, shasum, counter) - - err := lw(multi) - sha256sum := fmt.Sprintf("%x", shasum.Sum([]byte{})) - - return sha256sum, counter.count, err - }) - - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "layer": key, - "backend": s.Storage.Name(), - }).Error("failed to create and store layer") - - return nil, err - } - - // Hashes are now known and the object is in the bucket, what - // remains is to move it to the correct location and cache it. - err = s.Storage.Move(ctx, "staging/"+key, "layers/"+sha256sum) - if err != nil { - log.WithError(err).WithField("layer", key). - Error("failed to move layer from staging") - - return nil, err - } - - log.WithFields(log.Fields{ - "layer": key, - "sha256": sha256sum, - "size": size, - }).Info("created and persisted layer") - - entry := manifest.Entry{ - Digest: "sha256:" + sha256sum, - Size: size, - } - - return &entry, nil -} - -func BuildImage(ctx context.Context, s *State, image *Image) (*BuildResult, error) { - key := s.Cfg.Pkgs.CacheKey(image.Packages, image.Tag) - if key != "" { - if m, c := manifestFromCache(ctx, s, key); c { - return &BuildResult{ - Manifest: m, - }, nil - } - } - - imageResult, err := prepareImage(s, image) - if err != nil { - return nil, err - } - - if imageResult.Error != "" { - return &BuildResult{ - Error: imageResult.Error, - Pkgs: imageResult.Pkgs, - }, nil - } - - layers, err := prepareLayers(ctx, s, image, imageResult) - if err != nil { - return nil, err - } - - m, c := manifest.Manifest(image.Arch.imageArch, layers) - - lw := func(w io.Writer) error { - r := bytes.NewReader(c.Config) - _, err := io.Copy(w, r) - return err - } - - if _, err = uploadHashLayer(ctx, s, c.SHA256, lw); err != nil { - log.WithError(err).WithFields(log.Fields{ - "image": image.Name, - "tag": image.Tag, - }).Error("failed to upload config") - - return nil, err - } - - if key != "" { - go cacheManifest(ctx, s, key, m) - } - - result := BuildResult{ - Manifest: m, - } - return &result, nil -} diff --git a/tools/nixery/server/builder/builder_test.go b/tools/nixery/server/builder/builder_test.go deleted file mode 100644 index 3fbe2ab40e..0000000000 --- a/tools/nixery/server/builder/builder_test.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. -package builder - -import ( - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "testing" -) - -var ignoreArch = cmpopts.IgnoreFields(Image{}, "Arch") - -func TestImageFromNameSimple(t *testing.T) { - image := ImageFromName("hello", "latest") - expected := Image{ - Name: "hello", - Tag: "latest", - Packages: []string{ - "cacert", - "hello", - "iana-etc", - }, - } - - if diff := cmp.Diff(expected, image, ignoreArch); diff != "" { - t.Fatalf("Image(\"hello\", \"latest\") mismatch:\n%s", diff) - } -} - -func TestImageFromNameMultiple(t *testing.T) { - image := ImageFromName("hello/git/htop", "latest") - expected := Image{ - Name: "git/hello/htop", - Tag: "latest", - Packages: []string{ - "cacert", - "git", - "hello", - "htop", - "iana-etc", - }, - } - - if diff := cmp.Diff(expected, image, ignoreArch); diff != "" { - t.Fatalf("Image(\"hello/git/htop\", \"latest\") mismatch:\n%s", diff) - } -} - -func TestImageFromNameShell(t *testing.T) { - image := ImageFromName("shell", "latest") - expected := Image{ - Name: "shell", - Tag: "latest", - Packages: []string{ - "bashInteractive", - "cacert", - "coreutils", - "iana-etc", - "moreutils", - "nano", - }, - } - - if diff := cmp.Diff(expected, image, ignoreArch); diff != "" { - t.Fatalf("Image(\"shell\", \"latest\") mismatch:\n%s", diff) - } -} - -func TestImageFromNameShellMultiple(t *testing.T) { - image := ImageFromName("shell/htop", "latest") - expected := Image{ - Name: "htop/shell", - Tag: "latest", - Packages: []string{ - "bashInteractive", - "cacert", - "coreutils", - "htop", - "iana-etc", - "moreutils", - "nano", - }, - } - - if diff := cmp.Diff(expected, image, ignoreArch); diff != "" { - t.Fatalf("Image(\"shell/htop\", \"latest\") mismatch:\n%s", diff) - } -} - -func TestImageFromNameShellArm64(t *testing.T) { - image := ImageFromName("shell/arm64", "latest") - expected := Image{ - Name: "arm64/shell", - Tag: "latest", - Packages: []string{ - "bashInteractive", - "cacert", - "coreutils", - "iana-etc", - "moreutils", - "nano", - }, - } - - if diff := cmp.Diff(expected, image, ignoreArch); diff != "" { - t.Fatalf("Image(\"shell/arm64\", \"latest\") mismatch:\n%s", diff) - } - - if image.Arch.imageArch != "arm64" { - t.Fatal("Image(\"shell/arm64\"): Expected arch arm64") - } -} diff --git a/tools/nixery/server/builder/cache.go b/tools/nixery/server/builder/cache.go deleted file mode 100644 index 82bd90927c..0000000000 --- a/tools/nixery/server/builder/cache.go +++ /dev/null @@ -1,236 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. -package builder - -import ( - "bytes" - "context" - "encoding/json" - "io" - "io/ioutil" - "os" - "sync" - - "github.com/google/nixery/server/manifest" - log "github.com/sirupsen/logrus" -) - -// LocalCache implements the structure used for local caching of -// manifests and layer uploads. -type LocalCache struct { - // Manifest cache - mmtx sync.RWMutex - mdir string - - // Layer cache - lmtx sync.RWMutex - lcache map[string]manifest.Entry -} - -// Creates an in-memory cache and ensures that the local file path for -// manifest caching exists. -func NewCache() (LocalCache, error) { - path := os.TempDir() + "/nixery" - err := os.MkdirAll(path, 0755) - if err != nil { - return LocalCache{}, err - } - - return LocalCache{ - mdir: path + "/", - lcache: make(map[string]manifest.Entry), - }, nil -} - -// Retrieve a cached manifest if the build is cacheable and it exists. -func (c *LocalCache) manifestFromLocalCache(key string) (json.RawMessage, bool) { - c.mmtx.RLock() - defer c.mmtx.RUnlock() - - f, err := os.Open(c.mdir + key) - if err != nil { - // This is a debug log statement because failure to - // read the manifest key is currently expected if it - // is not cached. - log.WithError(err).WithField("manifest", key). - Debug("failed to read manifest from local cache") - - return nil, false - } - defer f.Close() - - m, err := ioutil.ReadAll(f) - if err != nil { - log.WithError(err).WithField("manifest", key). - Error("failed to read manifest from local cache") - - return nil, false - } - - return json.RawMessage(m), true -} - -// Adds the result of a manifest build to the local cache, if the -// manifest is considered cacheable. -// -// Manifests can be quite large and are cached on disk instead of in -// memory. -func (c *LocalCache) localCacheManifest(key string, m json.RawMessage) { - c.mmtx.Lock() - defer c.mmtx.Unlock() - - err := ioutil.WriteFile(c.mdir+key, []byte(m), 0644) - if err != nil { - log.WithError(err).WithField("manifest", key). - Error("failed to locally cache manifest") - } -} - -// Retrieve a layer build from the local cache. -func (c *LocalCache) layerFromLocalCache(key string) (*manifest.Entry, bool) { - c.lmtx.RLock() - e, ok := c.lcache[key] - c.lmtx.RUnlock() - - return &e, ok -} - -// Add a layer build result to the local cache. -func (c *LocalCache) localCacheLayer(key string, e manifest.Entry) { - c.lmtx.Lock() - c.lcache[key] = e - c.lmtx.Unlock() -} - -// Retrieve a manifest from the cache(s). First the local cache is -// checked, then the storage backend. -func manifestFromCache(ctx context.Context, s *State, key string) (json.RawMessage, bool) { - if m, cached := s.Cache.manifestFromLocalCache(key); cached { - return m, true - } - - r, err := s.Storage.Fetch(ctx, "manifests/"+key) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "manifest": key, - "backend": s.Storage.Name(), - }).Error("failed to fetch manifest from cache") - - return nil, false - } - defer r.Close() - - m, err := ioutil.ReadAll(r) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "manifest": key, - "backend": s.Storage.Name(), - }).Error("failed to read cached manifest from storage backend") - - return nil, false - } - - go s.Cache.localCacheManifest(key, m) - log.WithField("manifest", key).Info("retrieved manifest from GCS") - - return json.RawMessage(m), true -} - -// Add a manifest to the bucket & local caches -func cacheManifest(ctx context.Context, s *State, key string, m json.RawMessage) { - go s.Cache.localCacheManifest(key, m) - - path := "manifests/" + key - _, size, err := s.Storage.Persist(ctx, path, func(w io.Writer) (string, int64, error) { - size, err := io.Copy(w, bytes.NewReader([]byte(m))) - return "", size, err - }) - - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "manifest": key, - "backend": s.Storage.Name(), - }).Error("failed to cache manifest to storage backend") - - return - } - - log.WithFields(log.Fields{ - "manifest": key, - "size": size, - "backend": s.Storage.Name(), - }).Info("cached manifest to storage backend") -} - -// Retrieve a layer build from the cache, first checking the local -// cache followed by the bucket cache. -func layerFromCache(ctx context.Context, s *State, key string) (*manifest.Entry, bool) { - if entry, cached := s.Cache.layerFromLocalCache(key); cached { - return entry, true - } - - r, err := s.Storage.Fetch(ctx, "builds/"+key) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "layer": key, - "backend": s.Storage.Name(), - }).Debug("failed to retrieve cached layer from storage backend") - - return nil, false - } - defer r.Close() - - jb := bytes.NewBuffer([]byte{}) - _, err = io.Copy(jb, r) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "layer": key, - "backend": s.Storage.Name(), - }).Error("failed to read cached layer from storage backend") - - return nil, false - } - - var entry manifest.Entry - err = json.Unmarshal(jb.Bytes(), &entry) - if err != nil { - log.WithError(err).WithField("layer", key). - Error("failed to unmarshal cached layer") - - return nil, false - } - - go s.Cache.localCacheLayer(key, entry) - return &entry, true -} - -func cacheLayer(ctx context.Context, s *State, key string, entry manifest.Entry) { - s.Cache.localCacheLayer(key, entry) - - j, _ := json.Marshal(&entry) - path := "builds/" + key - _, _, err := s.Storage.Persist(ctx, path, func(w io.Writer) (string, int64, error) { - size, err := io.Copy(w, bytes.NewReader(j)) - return "", size, err - }) - - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "layer": key, - "backend": s.Storage.Name(), - }).Error("failed to cache layer") - } - - return -} diff --git a/tools/nixery/server/config/config.go b/tools/nixery/server/config/config.go deleted file mode 100644 index 7ec102bd6c..0000000000 --- a/tools/nixery/server/config/config.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -// Package config implements structures to store Nixery's configuration at -// runtime as well as the logic for instantiating this configuration from the -// environment. -package config - -import ( - "os" - - log "github.com/sirupsen/logrus" -) - -func getConfig(key, desc, def string) string { - value := os.Getenv(key) - if value == "" && def == "" { - log.WithFields(log.Fields{ - "option": key, - "description": desc, - }).Fatal("missing required configuration envvar") - } else if value == "" { - return def - } - - return value -} - -// Backend represents the possible storage backend types -type Backend int - -const ( - GCS = iota - FileSystem -) - -// Config holds the Nixery configuration options. -type Config struct { - Port string // Port on which to launch HTTP server - Pkgs PkgSource // Source for Nix package set - Timeout string // Timeout for a single Nix builder (seconds) - WebDir string // Directory with static web assets - PopUrl string // URL to the Nix package popularity count - Backend Backend // Storage backend to use for Nixery -} - -func FromEnv() (Config, error) { - pkgs, err := pkgSourceFromEnv() - if err != nil { - return Config{}, err - } - - var b Backend - switch os.Getenv("NIXERY_STORAGE_BACKEND") { - case "gcs": - b = GCS - case "filesystem": - b = FileSystem - default: - log.WithField("values", []string{ - "gcs", - }).Fatal("NIXERY_STORAGE_BUCKET must be set to a supported value") - } - - return Config{ - Port: getConfig("PORT", "HTTP port", ""), - Pkgs: pkgs, - Timeout: getConfig("NIX_TIMEOUT", "Nix builder timeout", "60"), - WebDir: getConfig("WEB_DIR", "Static web file dir", ""), - PopUrl: os.Getenv("NIX_POPULARITY_URL"), - Backend: b, - }, nil -} diff --git a/tools/nixery/server/config/pkgsource.go b/tools/nixery/server/config/pkgsource.go deleted file mode 100644 index 95236c4b0d..0000000000 --- a/tools/nixery/server/config/pkgsource.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. -package config - -import ( - "crypto/sha1" - "encoding/json" - "fmt" - "os" - "regexp" - "strings" - - log "github.com/sirupsen/logrus" -) - -// PkgSource represents the source from which the Nix package set used -// by Nixery is imported. Users configure the source by setting one of -// the supported environment variables. -type PkgSource interface { - // Convert the package source into the representation required - // for calling Nix. - Render(tag string) (string, string) - - // Create a key by which builds for this source and iamge - // combination can be cached. - // - // The empty string means that this value is not cacheable due - // to the package source being a moving target (such as a - // channel). - CacheKey(pkgs []string, tag string) string -} - -type GitSource struct { - repository string -} - -// Regex to determine whether a git reference is a commit hash or -// something else (branch/tag). -// -// Used to check whether a git reference is cacheable, and to pass the -// correct git structure to Nix. -// -// Note: If a user creates a branch or tag with the name of a commit -// and references it intentionally, this heuristic will fail. -var commitRegex = regexp.MustCompile(`^[0-9a-f]{40}$`) - -func (g *GitSource) Render(tag string) (string, string) { - args := map[string]string{ - "url": g.repository, - } - - // The 'git' source requires a tag to be present. If the user - // has not specified one, it is assumed that the default - // 'master' branch should be used. - if tag == "latest" || tag == "" { - tag = "master" - } - - if commitRegex.MatchString(tag) { - args["rev"] = tag - } else { - args["ref"] = tag - } - - j, _ := json.Marshal(args) - - return "git", string(j) -} - -func (g *GitSource) CacheKey(pkgs []string, tag string) string { - // Only full commit hashes can be used for caching, as - // everything else is potentially a moving target. - if !commitRegex.MatchString(tag) { - return "" - } - - unhashed := strings.Join(pkgs, "") + tag - hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed))) - - return hashed -} - -type NixChannel struct { - channel string -} - -func (n *NixChannel) Render(tag string) (string, string) { - return "nixpkgs", n.channel -} - -func (n *NixChannel) CacheKey(pkgs []string, tag string) string { - // Since Nix channels are downloaded from the nixpkgs-channels - // Github, users can specify full commit hashes as the - // "channel", in which case builds are cacheable. - if !commitRegex.MatchString(n.channel) { - return "" - } - - unhashed := strings.Join(pkgs, "") + n.channel - hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed))) - - return hashed -} - -type PkgsPath struct { - path string -} - -func (p *PkgsPath) Render(tag string) (string, string) { - return "path", p.path -} - -func (p *PkgsPath) CacheKey(pkgs []string, tag string) string { - // Path-based builds are not currently cacheable because we - // have no local hash of the package folder's state easily - // available. - return "" -} - -// Retrieve a package source from the environment. If no source is -// specified, the Nix code will default to a recent NixOS channel. -func pkgSourceFromEnv() (PkgSource, error) { - if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" { - log.WithField("channel", channel).Info("using Nix package set from Nix channel or commit") - - return &NixChannel{ - channel: channel, - }, nil - } - - if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" { - log.WithField("repo", git).Info("using NIx package set from git repository") - - return &GitSource{ - repository: git, - }, nil - } - - if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" { - log.WithField("path", path).Info("using Nix package set at local path") - - return &PkgsPath{ - path: path, - }, nil - } - - return nil, fmt.Errorf("no valid package source has been specified") -} diff --git a/tools/nixery/server/default.nix b/tools/nixery/server/default.nix deleted file mode 100644 index d497f106b0..0000000000 --- a/tools/nixery/server/default.nix +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{ buildGoPackage, go, lib, srcHash }: - -buildGoPackage rec { - name = "nixery-server"; - goDeps = ./go-deps.nix; - src = ./.; - - goPackagePath = "github.com/google/nixery/server"; - doCheck = true; - - # The following phase configurations work around the overengineered - # Nix build configuration for Go. - # - # All I want this to do is produce a binary in the standard Nix - # output path, so pretty much all the phases except for the initial - # configuration of the "dependency forest" in $GOPATH have been - # overridden. - # - # This is necessary because the upstream builder does wonky things - # with the build arguments to the compiler, but I need to set some - # complex flags myself - - outputs = [ "out" ]; - preConfigure = "bin=$out"; - buildPhase = '' - runHook preBuild - runHook renameImport - - export GOBIN="$out/bin" - go install -ldflags "-X main.version=$(cat ${srcHash})" ${goPackagePath} - ''; - - fixupPhase = '' - remove-references-to -t ${go} $out/bin/server - ''; - - checkPhase = '' - go vet ${goPackagePath} - go test ${goPackagePath} - ''; - - meta = { - description = "Container image builder serving Nix-backed images"; - homepage = "https://github.com/google/nixery"; - license = lib.licenses.asl20; - maintainers = [ lib.maintainers.tazjin ]; - }; -} diff --git a/tools/nixery/server/go-deps.nix b/tools/nixery/server/go-deps.nix deleted file mode 100644 index 847b44dce6..0000000000 --- a/tools/nixery/server/go-deps.nix +++ /dev/null @@ -1,129 +0,0 @@ -# This file was generated by https://github.com/kamilchm/go2nix v1.3.0 -[ - { - goPackagePath = "cloud.google.com/go"; - fetch = { - type = "git"; - url = "https://code.googlesource.com/gocloud"; - rev = "77f6a3a292a7dbf66a5329de0d06326f1906b450"; - sha256 = "1c9pkx782nbcp8jnl5lprcbzf97van789ky5qsncjgywjyymhigi"; - }; - } - { - goPackagePath = "github.com/golang/protobuf"; - fetch = { - type = "git"; - url = "https://github.com/golang/protobuf"; - rev = "6c65a5562fc06764971b7c5d05c76c75e84bdbf7"; - sha256 = "1k1wb4zr0qbwgpvz9q5ws9zhlal8hq7dmq62pwxxriksayl6hzym"; - }; - } - { - goPackagePath = "github.com/googleapis/gax-go"; - fetch = { - type = "git"; - url = "https://github.com/googleapis/gax-go"; - rev = "bd5b16380fd03dc758d11cef74ba2e3bc8b0e8c2"; - sha256 = "1lxawwngv6miaqd25s3ba0didfzylbwisd2nz7r4gmbmin6jsjrx"; - }; - } - { - goPackagePath = "github.com/hashicorp/golang-lru"; - fetch = { - type = "git"; - url = "https://github.com/hashicorp/golang-lru"; - rev = "59383c442f7d7b190497e9bb8fc17a48d06cd03f"; - sha256 = "0yzwl592aa32vfy73pl7wdc21855w17zssrp85ckw2nisky8rg9c"; - }; - } - { - goPackagePath = "go.opencensus.io"; - fetch = { - type = "git"; - url = "https://github.com/census-instrumentation/opencensus-go"; - rev = "b4a14686f0a98096416fe1b4cb848e384fb2b22b"; - sha256 = "1aidyp301v5ngwsnnc8v1s09vvbsnch1jc4vd615f7qv77r9s7dn"; - }; - } - { - goPackagePath = "golang.org/x/net"; - fetch = { - type = "git"; - url = "https://go.googlesource.com/net"; - rev = "da137c7871d730100384dbcf36e6f8fa493aef5b"; - sha256 = "1qsiyr3irmb6ii06hivm9p2c7wqyxczms1a9v1ss5698yjr3fg47"; - }; - } - { - goPackagePath = "golang.org/x/oauth2"; - fetch = { - type = "git"; - url = "https://go.googlesource.com/oauth2"; - rev = "0f29369cfe4552d0e4bcddc57cc75f4d7e672a33"; - sha256 = "06jwpvx0x2gjn2y959drbcir5kd7vg87k0r1216abk6rrdzzrzi2"; - }; - } - { - goPackagePath = "golang.org/x/sys"; - fetch = { - type = "git"; - url = "https://go.googlesource.com/sys"; - rev = "51ab0e2deafac1f46c46ad59cf0921be2f180c3d"; - sha256 = "0xdhpckbql3bsqkpc2k5b1cpnq3q1qjqjjq2j3p707rfwb8nm91a"; - }; - } - { - goPackagePath = "golang.org/x/text"; - fetch = { - type = "git"; - url = "https://go.googlesource.com/text"; - rev = "342b2e1fbaa52c93f31447ad2c6abc048c63e475"; - sha256 = "0flv9idw0jm5nm8lx25xqanbkqgfiym6619w575p7nrdh0riqwqh"; - }; - } - { - goPackagePath = "google.golang.org/api"; - fetch = { - type = "git"; - url = "https://code.googlesource.com/google-api-go-client"; - rev = "069bea57b1be6ad0671a49ea7a1128025a22b73f"; - sha256 = "19q2b610lkf3z3y9hn6rf11dd78xr9q4340mdyri7kbijlj2r44q"; - }; - } - { - goPackagePath = "google.golang.org/genproto"; - fetch = { - type = "git"; - url = "https://github.com/google/go-genproto"; - rev = "c506a9f9061087022822e8da603a52fc387115a8"; - sha256 = "03hh80aqi58dqi5ykj4shk3chwkzrgq2f3k6qs5qhgvmcy79y2py"; - }; - } - { - goPackagePath = "google.golang.org/grpc"; - fetch = { - type = "git"; - url = "https://github.com/grpc/grpc-go"; - rev = "977142214c45640483838b8672a43c46f89f90cb"; - sha256 = "05wig23l2sil3bfdv19gq62sya7hsabqj9l8pzr1sm57qsvj218d"; - }; - } - { - goPackagePath = "gonum.org/v1/gonum"; - fetch = { - type = "git"; - url = "https://github.com/gonum/gonum"; - rev = "ced62fe5104b907b6c16cb7e575c17b2e62ceddd"; - sha256 = "1b7q6haabnp53igpmvr6a2414yralhbrldixx4kbxxg1apy8jdjg"; - }; - } - { - goPackagePath = "github.com/sirupsen/logrus"; - fetch = { - type = "git"; - url = "https://github.com/sirupsen/logrus"; - rev = "de736cf91b921d56253b4010270681d33fdf7cb5"; - sha256 = "1qixss8m5xy7pzbf0qz2k3shjw0asklm9sj6zyczp7mryrari0aj"; - }; - } -] diff --git a/tools/nixery/server/layers/grouping.go b/tools/nixery/server/layers/grouping.go deleted file mode 100644 index 3902c8a4ef..0000000000 --- a/tools/nixery/server/layers/grouping.go +++ /dev/null @@ -1,361 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -// This package reads an export reference graph (i.e. a graph representing the -// runtime dependencies of a set of derivations) created by Nix and groups it in -// a way that is likely to match the grouping for other derivation sets with -// overlapping dependencies. -// -// This is used to determine which derivations to include in which layers of a -// container image. -// -// # Inputs -// -// * a graph of Nix runtime dependencies, generated via exportReferenceGraph -// * popularity values of each package in the Nix package set (in the form of a -// direct reference count) -// * a maximum number of layers to allocate for the image (the "layer budget") -// -// # Algorithm -// -// It works by first creating a (directed) dependency tree: -// -// img (root node) -// │ -// ├───> A ─────┐ -// │ v -// ├───> B ───> E -// │ ^ -// ├───> C ─────┘ -// │ │ -// │ v -// └───> D ───> F -// │ -// └────> G -// -// Each node (i.e. package) is then visited to determine how important -// it is to separate this node into its own layer, specifically: -// -// 1. Is the node within a certain threshold percentile of absolute -// popularity within all of nixpkgs? (e.g. `glibc`, `openssl`) -// -// 2. Is the node's runtime closure above a threshold size? (e.g. 100MB) -// -// In either case, a bit is flipped for this node representing each -// condition and an edge to it is inserted directly from the image -// root, if it does not already exist. -// -// For the rest of the example we assume 'G' is above the threshold -// size and 'E' is popular. -// -// This tree is then transformed into a dominator tree: -// -// img -// │ -// ├───> A -// ├───> B -// ├───> C -// ├───> E -// ├───> D ───> F -// └───> G -// -// Specifically this means that the paths to A, B, C, E, G, and D -// always pass through the root (i.e. are dominated by it), whilst F -// is dominated by D (all paths go through it). -// -// The top-level subtrees are considered as the initially selected -// layers. -// -// If the list of layers fits within the layer budget, it is returned. -// -// Otherwise, a merge rating is calculated for each layer. This is the -// product of the layer's total size and its root node's popularity. -// -// Layers are then merged in ascending order of merge ratings until -// they fit into the layer budget. -// -// # Threshold values -// -// Threshold values for the partitioning conditions mentioned above -// have not yet been determined, but we will make a good first guess -// based on gut feeling and proceed to measure their impact on cache -// hits/misses. -// -// # Example -// -// Using the logic described above as well as the example presented in -// the introduction, this program would create the following layer -// groupings (assuming no additional partitioning): -// -// Layer budget: 1 -// Layers: { A, B, C, D, E, F, G } -// -// Layer budget: 2 -// Layers: { G }, { A, B, C, D, E, F } -// -// Layer budget: 3 -// Layers: { G }, { E }, { A, B, C, D, F } -// -// Layer budget: 4 -// Layers: { G }, { E }, { D, F }, { A, B, C } -// -// ... -// -// Layer budget: 10 -// Layers: { E }, { D, F }, { A }, { B }, { C } -package layers - -import ( - "crypto/sha1" - "fmt" - "regexp" - "sort" - "strings" - - log "github.com/sirupsen/logrus" - "gonum.org/v1/gonum/graph/flow" - "gonum.org/v1/gonum/graph/simple" -) - -// RuntimeGraph represents structured information from Nix about the runtime -// dependencies of a derivation. -// -// This is generated in Nix by using the exportReferencesGraph feature. -type RuntimeGraph struct { - References struct { - Graph []string `json:"graph"` - } `json:"exportReferencesGraph"` - - Graph []struct { - Size uint64 `json:"closureSize"` - Path string `json:"path"` - Refs []string `json:"references"` - } `json:"graph"` -} - -// Popularity data for each Nix package that was calculated in advance. -// -// Popularity is a number from 1-100 that represents the -// popularity percentile in which this package resides inside -// of the nixpkgs tree. -type Popularity = map[string]int - -// Layer represents the data returned for each layer that Nix should -// build for the container image. -type Layer struct { - Contents []string `json:"contents"` - MergeRating uint64 -} - -// Hash the contents of a layer to create a deterministic identifier that can be -// used for caching. -func (l *Layer) Hash() string { - sum := sha1.Sum([]byte(strings.Join(l.Contents, ":"))) - return fmt.Sprintf("%x", sum) -} - -func (a Layer) merge(b Layer) Layer { - a.Contents = append(a.Contents, b.Contents...) - a.MergeRating += b.MergeRating - return a -} - -// closure as pointed to by the graph nodes. -type closure struct { - GraphID int64 - Path string - Size uint64 - Refs []string - Popularity int -} - -func (c *closure) ID() int64 { - return c.GraphID -} - -var nixRegexp = regexp.MustCompile(`^/nix/store/[a-z0-9]+-`) - -// PackageFromPath returns the name of a Nix package based on its -// output store path. -func PackageFromPath(path string) string { - return nixRegexp.ReplaceAllString(path, "") -} - -func (c *closure) DOTID() string { - return PackageFromPath(c.Path) -} - -// bigOrPopular checks whether this closure should be considered for -// separation into its own layer, even if it would otherwise only -// appear in a subtree of the dominator tree. -func (c *closure) bigOrPopular() bool { - const sizeThreshold = 100 * 1000000 // 100MB - - if c.Size > sizeThreshold { - return true - } - - // Threshold value is picked arbitrarily right now. The reason - // for this is that some packages (such as `cacert`) have very - // few direct dependencies, but are required by pretty much - // everything. - if c.Popularity >= 100 { - return true - } - - return false -} - -func insertEdges(graph *simple.DirectedGraph, cmap *map[string]*closure, node *closure) { - // Big or popular nodes get a separate edge from the top to - // flag them for their own layer. - if node.bigOrPopular() && !graph.HasEdgeFromTo(0, node.ID()) { - edge := graph.NewEdge(graph.Node(0), node) - graph.SetEdge(edge) - } - - for _, c := range node.Refs { - // Nix adds a self reference to each node, which - // should not be inserted. - if c != node.Path { - edge := graph.NewEdge(node, (*cmap)[c]) - graph.SetEdge(edge) - } - } -} - -// Create a graph structure from the references supplied by Nix. -func buildGraph(refs *RuntimeGraph, pop *Popularity) *simple.DirectedGraph { - cmap := make(map[string]*closure) - graph := simple.NewDirectedGraph() - - // Insert all closures into the graph, as well as a fake root - // closure which serves as the top of the tree. - // - // A map from store paths to IDs is kept to actually insert - // edges below. - root := &closure{ - GraphID: 0, - Path: "image_root", - } - graph.AddNode(root) - - for idx, c := range refs.Graph { - node := &closure{ - GraphID: int64(idx + 1), // inc because of root node - Path: c.Path, - Size: c.Size, - Refs: c.Refs, - } - - // The packages `nss-cacert` and `iana-etc` are added - // by Nixery to *every single image* and should have a - // very high popularity. - // - // Other popularity values are populated from the data - // set assembled by Nixery's popcount. - id := node.DOTID() - if strings.HasPrefix(id, "nss-cacert") || strings.HasPrefix(id, "iana-etc") { - // glibc has ~300k references, these packages need *more* - node.Popularity = 500000 - } else if p, ok := (*pop)[id]; ok { - node.Popularity = p - } else { - node.Popularity = 1 - } - - graph.AddNode(node) - cmap[c.Path] = node - } - - // Insert the top-level closures with edges from the root - // node, then insert all edges for each closure. - for _, p := range refs.References.Graph { - edge := graph.NewEdge(root, cmap[p]) - graph.SetEdge(edge) - } - - for _, c := range cmap { - insertEdges(graph, &cmap, c) - } - - return graph -} - -// Extracts a subgraph starting at the specified root from the -// dominator tree. The subgraph is converted into a flat list of -// layers, each containing the store paths and merge rating. -func groupLayer(dt *flow.DominatorTree, root *closure) Layer { - size := root.Size - contents := []string{root.Path} - children := dt.DominatedBy(root.ID()) - - // This iteration does not use 'range' because the list being - // iterated is modified during the iteration (yes, I'm sorry). - for i := 0; i < len(children); i++ { - child := children[i].(*closure) - size += child.Size - contents = append(contents, child.Path) - children = append(children, dt.DominatedBy(child.ID())...) - } - - // Contents are sorted to ensure that hashing is consistent - sort.Strings(contents) - - return Layer{ - Contents: contents, - MergeRating: uint64(root.Popularity) * size, - } -} - -// Calculate the dominator tree of the entire package set and group -// each top-level subtree into a layer. -// -// Layers are merged together until they fit into the layer budget, -// based on their merge rating. -func dominate(budget int, graph *simple.DirectedGraph) []Layer { - dt := flow.Dominators(graph.Node(0), graph) - - var layers []Layer - for _, n := range dt.DominatedBy(dt.Root().ID()) { - layers = append(layers, groupLayer(&dt, n.(*closure))) - } - - sort.Slice(layers, func(i, j int) bool { - return layers[i].MergeRating < layers[j].MergeRating - }) - - if len(layers) > budget { - log.WithFields(log.Fields{ - "layers": len(layers), - "budget": budget, - }).Info("ideal image exceeds layer budget") - } - - for len(layers) > budget { - merged := layers[0].merge(layers[1]) - layers[1] = merged - layers = layers[1:] - } - - return layers -} - -// GroupLayers applies the algorithm described above the its input and returns a -// list of layers, each consisting of a list of Nix store paths that it should -// contain. -func Group(refs *RuntimeGraph, pop *Popularity, budget int) []Layer { - graph := buildGraph(refs, pop) - return dominate(budget, graph) -} diff --git a/tools/nixery/server/logs.go b/tools/nixery/server/logs.go deleted file mode 100644 index 3179402e2e..0000000000 --- a/tools/nixery/server/logs.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. -package main - -// This file configures different log formatters via logrus. The -// standard formatter uses a structured JSON format that is compatible -// with Stackdriver Error Reporting. -// -// https://cloud.google.com/error-reporting/docs/formatting-error-messages - -import ( - "bytes" - "encoding/json" - log "github.com/sirupsen/logrus" -) - -type stackdriverFormatter struct{} - -type serviceContext struct { - Service string `json:"service"` - Version string `json:"version"` -} - -type reportLocation struct { - FilePath string `json:"filePath"` - LineNumber int `json:"lineNumber"` - FunctionName string `json:"functionName"` -} - -var nixeryContext = serviceContext{ - Service: "nixery", -} - -// isError determines whether an entry should be logged as an error -// (i.e. with attached `context`). -// -// This requires the caller information to be present on the log -// entry, as stacktraces are not available currently. -func isError(e *log.Entry) bool { - l := e.Level - return (l == log.ErrorLevel || l == log.FatalLevel || l == log.PanicLevel) && - e.HasCaller() -} - -// logSeverity formats the entry's severity into a format compatible -// with Stackdriver Logging. -// -// The two formats that are being mapped do not have an equivalent set -// of severities/levels, so the mapping is somewhat arbitrary for a -// handful of them. -// -// https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#LogSeverity -func logSeverity(l log.Level) string { - switch l { - case log.TraceLevel: - return "DEBUG" - case log.DebugLevel: - return "DEBUG" - case log.InfoLevel: - return "INFO" - case log.WarnLevel: - return "WARNING" - case log.ErrorLevel: - return "ERROR" - case log.FatalLevel: - return "CRITICAL" - case log.PanicLevel: - return "EMERGENCY" - default: - return "DEFAULT" - } -} - -func (f stackdriverFormatter) Format(e *log.Entry) ([]byte, error) { - msg := e.Data - msg["serviceContext"] = &nixeryContext - msg["message"] = &e.Message - msg["eventTime"] = &e.Time - msg["severity"] = logSeverity(e.Level) - - if e, ok := msg[log.ErrorKey]; ok { - if err, isError := e.(error); isError { - msg[log.ErrorKey] = err.Error() - } else { - delete(msg, log.ErrorKey) - } - } - - if isError(e) { - loc := reportLocation{ - FilePath: e.Caller.File, - LineNumber: e.Caller.Line, - FunctionName: e.Caller.Function, - } - msg["context"] = &loc - } - - b := new(bytes.Buffer) - err := json.NewEncoder(b).Encode(&msg) - - return b.Bytes(), err -} - -func init() { - nixeryContext.Version = version - log.SetReportCaller(true) - log.SetFormatter(stackdriverFormatter{}) -} diff --git a/tools/nixery/server/main.go b/tools/nixery/server/main.go deleted file mode 100644 index 6ae0730906..0000000000 --- a/tools/nixery/server/main.go +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -// The nixery server implements a container registry that transparently builds -// container images based on Nix derivations. -// -// The Nix derivation used for image creation is responsible for creating -// objects that are compatible with the registry API. The targeted registry -// protocol is currently Docker's. -// -// When an image is requested, the required contents are parsed out of the -// request and a Nix-build is initiated that eventually responds with the -// manifest as well as information linking each layer digest to a local -// filesystem path. -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "regexp" - - "github.com/google/nixery/server/builder" - "github.com/google/nixery/server/config" - "github.com/google/nixery/server/layers" - "github.com/google/nixery/server/storage" - log "github.com/sirupsen/logrus" -) - -// ManifestMediaType is the Content-Type used for the manifest itself. This -// corresponds to the "Image Manifest V2, Schema 2" described on this page: -// -// https://docs.docker.com/registry/spec/manifest-v2-2/ -const manifestMediaType string = "application/vnd.docker.distribution.manifest.v2+json" - -// This variable will be initialised during the build process and set -// to the hash of the entire Nixery source tree. -var version string = "devel" - -// Regexes matching the V2 Registry API routes. This only includes the -// routes required for serving images, since pushing and other such -// functionality is not available. -var ( - manifestRegex = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/manifests/([\w|\-|\.|\_]+)$`) - layerRegex = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/blobs/sha256:(\w+)$`) -) - -// Downloads the popularity information for the package set from the -// URL specified in Nixery's configuration. -func downloadPopularity(url string) (layers.Popularity, error) { - resp, err := http.Get(url) - if err != nil { - return nil, err - } - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("popularity download from '%s' returned status: %s\n", url, resp.Status) - } - - j, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - var pop layers.Popularity - err = json.Unmarshal(j, &pop) - if err != nil { - return nil, err - } - - return pop, nil -} - -// Error format corresponding to the registry protocol V2 specification. This -// allows feeding back errors to clients in a way that can be presented to -// users. -type registryError struct { - Code string `json:"code"` - Message string `json:"message"` -} - -type registryErrors struct { - Errors []registryError `json:"errors"` -} - -func writeError(w http.ResponseWriter, status int, code, message string) { - err := registryErrors{ - Errors: []registryError{ - {code, message}, - }, - } - json, _ := json.Marshal(err) - - w.WriteHeader(status) - w.Header().Add("Content-Type", "application/json") - w.Write(json) -} - -type registryHandler struct { - state *builder.State -} - -func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - // Acknowledge that we speak V2 with an empty response - if r.RequestURI == "/v2/" { - return - } - - // Serve the manifest (straight from Nix) - manifestMatches := manifestRegex.FindStringSubmatch(r.RequestURI) - if len(manifestMatches) == 3 { - imageName := manifestMatches[1] - imageTag := manifestMatches[2] - - log.WithFields(log.Fields{ - "image": imageName, - "tag": imageTag, - }).Info("requesting image manifest") - - image := builder.ImageFromName(imageName, imageTag) - buildResult, err := builder.BuildImage(r.Context(), h.state, &image) - - if err != nil { - writeError(w, 500, "UNKNOWN", "image build failure") - - log.WithError(err).WithFields(log.Fields{ - "image": imageName, - "tag": imageTag, - }).Error("failed to build image manifest") - - return - } - - // Some error types have special handling, which is applied - // here. - if buildResult.Error == "not_found" { - s := fmt.Sprintf("Could not find Nix packages: %v", buildResult.Pkgs) - writeError(w, 404, "MANIFEST_UNKNOWN", s) - - log.WithFields(log.Fields{ - "image": imageName, - "tag": imageTag, - "packages": buildResult.Pkgs, - }).Warn("could not find Nix packages") - - return - } - - // This marshaling error is ignored because we know that this - // field represents valid JSON data. - manifest, _ := json.Marshal(buildResult.Manifest) - w.Header().Add("Content-Type", manifestMediaType) - w.Write(manifest) - return - } - - // Serve an image layer. For this we need to first ask Nix for - // the manifest, then proceed to extract the correct layer from - // it. - layerMatches := layerRegex.FindStringSubmatch(r.RequestURI) - if len(layerMatches) == 3 { - digest := layerMatches[2] - storage := h.state.Storage - err := storage.ServeLayer(digest, r, w) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "layer": digest, - "backend": storage.Name(), - }).Error("failed to serve layer from storage backend") - } - - return - } - - log.WithField("uri", r.RequestURI).Info("unsupported registry route") - - w.WriteHeader(404) -} - -func main() { - cfg, err := config.FromEnv() - if err != nil { - log.WithError(err).Fatal("failed to load configuration") - } - - var s storage.Backend - - switch cfg.Backend { - case config.GCS: - s, err = storage.NewGCSBackend() - case config.FileSystem: - s, err = storage.NewFSBackend() - } - if err != nil { - log.WithError(err).Fatal("failed to initialise storage backend") - } - - log.WithField("backend", s.Name()).Info("initialised storage backend") - - cache, err := builder.NewCache() - if err != nil { - log.WithError(err).Fatal("failed to instantiate build cache") - } - - var pop layers.Popularity - if cfg.PopUrl != "" { - pop, err = downloadPopularity(cfg.PopUrl) - if err != nil { - log.WithError(err).WithField("popURL", cfg.PopUrl). - Fatal("failed to fetch popularity information") - } - } - - state := builder.State{ - Cache: &cache, - Cfg: cfg, - Pop: pop, - Storage: s, - } - - log.WithFields(log.Fields{ - "version": version, - "port": cfg.Port, - }).Info("starting Nixery") - - // All /v2/ requests belong to the registry handler. - http.Handle("/v2/", ®istryHandler{ - state: &state, - }) - - // All other roots are served by the static file server. - webDir := http.Dir(cfg.WebDir) - http.Handle("/", http.FileServer(webDir)) - - log.Fatal(http.ListenAndServe(":"+cfg.Port, nil)) -} diff --git a/tools/nixery/server/manifest/manifest.go b/tools/nixery/server/manifest/manifest.go deleted file mode 100644 index 0d36826fb7..0000000000 --- a/tools/nixery/server/manifest/manifest.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -// Package image implements logic for creating the image metadata -// (such as the image manifest and configuration). -package manifest - -import ( - "crypto/sha256" - "encoding/json" - "fmt" - "sort" -) - -const ( - // manifest constants - schemaVersion = 2 - - // media types - manifestType = "application/vnd.docker.distribution.manifest.v2+json" - layerType = "application/vnd.docker.image.rootfs.diff.tar.gzip" - configType = "application/vnd.docker.container.image.v1+json" - - // image config constants - os = "linux" - fsType = "layers" -) - -type Entry struct { - MediaType string `json:"mediaType,omitempty"` - Size int64 `json:"size"` - Digest string `json:"digest"` - - // These fields are internal to Nixery and not part of the - // serialised entry. - MergeRating uint64 `json:"-"` - TarHash string `json:",omitempty"` -} - -type manifest struct { - SchemaVersion int `json:"schemaVersion"` - MediaType string `json:"mediaType"` - Config Entry `json:"config"` - Layers []Entry `json:"layers"` -} - -type imageConfig struct { - Architecture string `json:"architecture"` - OS string `json:"os"` - - RootFS struct { - FSType string `json:"type"` - DiffIDs []string `json:"diff_ids"` - } `json:"rootfs"` - - // sic! empty struct (rather than `null`) is required by the - // image metadata deserialiser in Kubernetes - Config struct{} `json:"config"` -} - -// ConfigLayer represents the configuration layer to be included in -// the manifest, containing its JSON-serialised content and SHA256 -// hash. -type ConfigLayer struct { - Config []byte - SHA256 string -} - -// imageConfig creates an image configuration with the values set to -// the constant defaults. -// -// Outside of this module the image configuration is treated as an -// opaque blob and it is thus returned as an already serialised byte -// array and its SHA256-hash. -func configLayer(arch string, hashes []string) ConfigLayer { - c := imageConfig{} - c.Architecture = arch - c.OS = os - c.RootFS.FSType = fsType - c.RootFS.DiffIDs = hashes - - j, _ := json.Marshal(c) - - return ConfigLayer{ - Config: j, - SHA256: fmt.Sprintf("%x", sha256.Sum256(j)), - } -} - -// Manifest creates an image manifest from the specified layer entries -// and returns its JSON-serialised form as well as the configuration -// layer. -// -// Callers do not need to set the media type for the layer entries. -func Manifest(arch string, layers []Entry) (json.RawMessage, ConfigLayer) { - // Sort layers by their merge rating, from highest to lowest. - // This makes it likely for a contiguous chain of shared image - // layers to appear at the beginning of a layer. - // - // Due to moby/moby#38446 Docker considers the order of layers - // when deciding which layers to download again. - sort.Slice(layers, func(i, j int) bool { - return layers[i].MergeRating > layers[j].MergeRating - }) - - hashes := make([]string, len(layers)) - for i, l := range layers { - hashes[i] = l.TarHash - l.MediaType = layerType - l.TarHash = "" - layers[i] = l - } - - c := configLayer(arch, hashes) - - m := manifest{ - SchemaVersion: schemaVersion, - MediaType: manifestType, - Config: Entry{ - MediaType: configType, - Size: int64(len(c.Config)), - Digest: "sha256:" + c.SHA256, - }, - Layers: layers, - } - - j, _ := json.Marshal(m) - - return json.RawMessage(j), c -} diff --git a/tools/nixery/server/storage/filesystem.go b/tools/nixery/server/storage/filesystem.go deleted file mode 100644 index cdbc31c5e0..0000000000 --- a/tools/nixery/server/storage/filesystem.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -// Filesystem storage backend for Nixery. -package storage - -import ( - "context" - "fmt" - "io" - "net/http" - "os" - "path" - - log "github.com/sirupsen/logrus" -) - -type FSBackend struct { - path string -} - -func NewFSBackend() (*FSBackend, error) { - p := os.Getenv("STORAGE_PATH") - if p == "" { - return nil, fmt.Errorf("STORAGE_PATH must be set for filesystem storage") - } - - p = path.Clean(p) - err := os.MkdirAll(p, 0755) - if err != nil { - return nil, fmt.Errorf("failed to create storage dir: %s", err) - } - - return &FSBackend{p}, nil -} - -func (b *FSBackend) Name() string { - return fmt.Sprintf("Filesystem (%s)", b.path) -} - -func (b *FSBackend) Persist(ctx context.Context, key string, f Persister) (string, int64, error) { - full := path.Join(b.path, key) - dir := path.Dir(full) - err := os.MkdirAll(dir, 0755) - if err != nil { - log.WithError(err).WithField("path", dir).Error("failed to create storage directory") - return "", 0, err - } - - file, err := os.OpenFile(full, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - log.WithError(err).WithField("file", full).Error("failed to write file") - return "", 0, err - } - defer file.Close() - - return f(file) -} - -func (b *FSBackend) Fetch(ctx context.Context, key string) (io.ReadCloser, error) { - full := path.Join(b.path, key) - return os.Open(full) -} - -func (b *FSBackend) Move(ctx context.Context, old, new string) error { - newpath := path.Join(b.path, new) - err := os.MkdirAll(path.Dir(newpath), 0755) - if err != nil { - return err - } - - return os.Rename(path.Join(b.path, old), newpath) -} - -func (b *FSBackend) ServeLayer(digest string, r *http.Request, w http.ResponseWriter) error { - p := path.Join(b.path, "layers", digest) - - log.WithFields(log.Fields{ - "layer": digest, - "path": p, - }).Info("serving layer from filesystem") - - http.ServeFile(w, r, p) - return nil -} diff --git a/tools/nixery/server/storage/gcs.go b/tools/nixery/server/storage/gcs.go deleted file mode 100644 index c247cca621..0000000000 --- a/tools/nixery/server/storage/gcs.go +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -// Google Cloud Storage backend for Nixery. -package storage - -import ( - "context" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "os" - "time" - - "cloud.google.com/go/storage" - log "github.com/sirupsen/logrus" - "golang.org/x/oauth2/google" -) - -// HTTP client to use for direct calls to APIs that are not part of the SDK -var client = &http.Client{} - -// API scope needed for renaming objects in GCS -const gcsScope = "https://www.googleapis.com/auth/devstorage.read_write" - -type GCSBackend struct { - bucket string - handle *storage.BucketHandle - signing *storage.SignedURLOptions -} - -// Constructs a new GCS bucket backend based on the configured -// environment variables. -func NewGCSBackend() (*GCSBackend, error) { - bucket := os.Getenv("GCS_BUCKET") - if bucket == "" { - return nil, fmt.Errorf("GCS_BUCKET must be configured for GCS usage") - } - - ctx := context.Background() - client, err := storage.NewClient(ctx) - if err != nil { - log.WithError(err).Fatal("failed to set up Cloud Storage client") - } - - handle := client.Bucket(bucket) - - if _, err := handle.Attrs(ctx); err != nil { - log.WithError(err).WithField("bucket", bucket).Error("could not access configured bucket") - return nil, err - } - - signing, err := signingOptsFromEnv() - if err != nil { - log.WithError(err).Error("failed to configure GCS bucket signing") - return nil, err - } - - return &GCSBackend{ - bucket: bucket, - handle: handle, - signing: signing, - }, nil -} - -func (b *GCSBackend) Name() string { - return "Google Cloud Storage (" + b.bucket + ")" -} - -func (b *GCSBackend) Persist(ctx context.Context, path string, f Persister) (string, int64, error) { - obj := b.handle.Object(path) - w := obj.NewWriter(ctx) - - hash, size, err := f(w) - if err != nil { - log.WithError(err).WithField("path", path).Error("failed to upload to GCS") - return hash, size, err - } - - return hash, size, w.Close() -} - -func (b *GCSBackend) Fetch(ctx context.Context, path string) (io.ReadCloser, error) { - obj := b.handle.Object(path) - - // Probe whether the file exists before trying to fetch it - _, err := obj.Attrs(ctx) - if err != nil { - return nil, err - } - - return obj.NewReader(ctx) -} - -// renameObject renames an object in the specified Cloud Storage -// bucket. -// -// The Go API for Cloud Storage does not support renaming objects, but -// the HTTP API does. The code below makes the relevant call manually. -func (b *GCSBackend) Move(ctx context.Context, old, new string) error { - creds, err := google.FindDefaultCredentials(ctx, gcsScope) - if err != nil { - return err - } - - token, err := creds.TokenSource.Token() - if err != nil { - return err - } - - // as per https://cloud.google.com/storage/docs/renaming-copying-moving-objects#rename - url := fmt.Sprintf( - "https://www.googleapis.com/storage/v1/b/%s/o/%s/rewriteTo/b/%s/o/%s", - url.PathEscape(b.bucket), url.PathEscape(old), - url.PathEscape(b.bucket), url.PathEscape(new), - ) - - req, err := http.NewRequest("POST", url, nil) - req.Header.Add("Authorization", "Bearer "+token.AccessToken) - _, err = client.Do(req) - if err != nil { - return err - } - - // It seems that 'rewriteTo' copies objects instead of - // renaming/moving them, hence a deletion call afterwards is - // required. - if err = b.handle.Object(old).Delete(ctx); err != nil { - log.WithError(err).WithFields(log.Fields{ - "new": new, - "old": old, - }).Warn("failed to delete renamed object") - - // this error should not break renaming and is not returned - } - - return nil -} - -func (b *GCSBackend) ServeLayer(digest string, r *http.Request, w http.ResponseWriter) error { - url, err := b.constructLayerUrl(digest) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "layer": digest, - "bucket": b.bucket, - }).Error("failed to sign GCS URL") - - return err - } - - log.WithField("layer", digest).Info("redirecting layer request to GCS bucket") - - w.Header().Set("Location", url) - w.WriteHeader(303) - return nil -} - -// Configure GCS URL signing in the presence of a service account key -// (toggled if the user has set GOOGLE_APPLICATION_CREDENTIALS). -func signingOptsFromEnv() (*storage.SignedURLOptions, error) { - path := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") - if path == "" { - // No credentials configured -> no URL signing - return nil, nil - } - - key, err := ioutil.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read service account key: %s", err) - } - - conf, err := google.JWTConfigFromJSON(key) - if err != nil { - return nil, fmt.Errorf("failed to parse service account key: %s", err) - } - - log.WithField("account", conf.Email).Info("GCS URL signing enabled") - - return &storage.SignedURLOptions{ - Scheme: storage.SigningSchemeV4, - GoogleAccessID: conf.Email, - PrivateKey: conf.PrivateKey, - Method: "GET", - }, nil -} - -// layerRedirect constructs the public URL of the layer object in the Cloud -// Storage bucket, signs it and redirects the user there. -// -// Signing the URL allows unauthenticated clients to retrieve objects from the -// bucket. -// -// The Docker client is known to follow redirects, but this might not be true -// for all other registry clients. -func (b *GCSBackend) constructLayerUrl(digest string) (string, error) { - log.WithField("layer", digest).Info("redirecting layer request to bucket") - object := "layers/" + digest - - if b.signing != nil { - opts := *b.signing - opts.Expires = time.Now().Add(5 * time.Minute) - return storage.SignedURL(b.bucket, object, &opts) - } else { - return ("https://storage.googleapis.com/" + b.bucket + "/" + object), nil - } -} diff --git a/tools/nixery/server/storage/storage.go b/tools/nixery/server/storage/storage.go deleted file mode 100644 index c97b5e4fac..0000000000 --- a/tools/nixery/server/storage/storage.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -// Package storage implements an interface that can be implemented by -// storage backends, such as Google Cloud Storage or the local -// filesystem. -package storage - -import ( - "context" - "io" - "net/http" -) - -type Persister = func(io.Writer) (string, int64, error) - -type Backend interface { - // Name returns the name of the storage backend, for use in - // log messages and such. - Name() string - - // Persist provides a user-supplied function with a writer - // that stores data in the storage backend. - // - // It needs to return the SHA256 hash of the data written as - // well as the total number of bytes, as those are required - // for the image manifest. - Persist(context.Context, string, Persister) (string, int64, error) - - // Fetch retrieves data from the storage backend. - Fetch(ctx context.Context, path string) (io.ReadCloser, error) - - // Move renames a path inside the storage backend. This is - // used for staging uploads while calculating their hashes. - Move(ctx context.Context, old, new string) error - - // Serve provides a handler function to serve HTTP requests - // for layers in the storage backend. - ServeLayer(digest string, r *http.Request, w http.ResponseWriter) error -} |