diff options
32 files changed, 194 insertions, 480 deletions
diff --git a/README.md b/README.md index 1bb29f29e0..0934ac9540 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,9 @@ configuration is tracked in `//ops/{modules,machines}`. * [`//nix/readTree`](https://cs.tvl.fyi/depot/-/blob/nix/readTree/README.md) contains the Nix code which automatically registers projects in our Nix attribute hierarchy based on their in-tree location +* [`//tools/nixery`](https://cs.tvl.fyi/depot/-/tree/tools/nixery) + contains the source code of [Nixery][], a container registry that + can build images ad-hoc from Nix packages * `//nix/yants` contains **Y**et **A**nother **N**ix **T**ype **S**ystem, which we use for a variety of things throughout the repository * `//nix/buildGo` implements a Nix library that can build Go software in the @@ -119,3 +122,4 @@ Hackint also provide a [web chat][tvl-webchat]. [hackint-xmpp]: https://hackint.org/transport/xmpp [tvl-xmpp]: xmpp:#tvl@irc.hackint.org?join [tvl-webchat]: https://webirc.hackint.org/#ircs://irc.hackint.org/#tvl +[Nixery]: https://nixery.dev diff --git a/ops/modules/nixery.nix b/ops/modules/nixery.nix index 33f196372d..cb80eff2e5 100644 --- a/ops/modules/nixery.nix +++ b/ops/modules/nixery.nix @@ -28,7 +28,7 @@ in StateDirectory = "nixery"; Restart = "always"; ExecStartPre = "${pkgs.coreutils}/bin/mkdir -p ${storagePath}"; - ExecStart = "${depot.third_party.nixery.nixery-bin}/bin/nixery"; + ExecStart = "${depot.tools.nixery.nixery-bin}/bin/nixery"; }; environment = { diff --git a/third_party/nixery/default.nix b/third_party/nixery/default.nix deleted file mode 100644 index 7a798ca0b0..0000000000 --- a/third_party/nixery/default.nix +++ /dev/null @@ -1,19 +0,0 @@ -# Import the Nixery repository as-is, but pass our own package set -# instead of the pin it has. -{ depot, pkgs, ... }: - -let - inherit (depot.nix.readTree) drvTargets; - - commit = "601cd998077f77f257ad1a40fa488add8464650f"; - src = pkgs.fetchFromGitHub { - owner = "google"; - repo = "nixery"; - rev = commit; - sha256 = "195rz25y3hfxcmniysajzjg7g69qhz7w06lql8fn0dbcdcxsq6g4"; - }; -in -drvTargets (import src { - inherit pkgs; - commitHash = _: commit; -}) diff --git a/tools/depotfmt.nix b/tools/depotfmt.nix index 17c05d0fc4..dbd3a31a0d 100644 --- a/tools/depotfmt.nix +++ b/tools/depotfmt.nix @@ -24,8 +24,7 @@ let includes = [ "*.nix" ] excludes = [ "third_party/nix/tests/*", - "third_party/nix/src/tests/*", - "tools/nixery/*" + "third_party/nix/src/tests/*" ] [formatter.rust] diff --git a/tools/nixery/.gitattributes b/tools/nixery/.gitattributes deleted file mode 100644 index 74464db942..0000000000 --- a/tools/nixery/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -# Ignore stylesheet modifications for the book in Linguist stats -*.css linguist-detectable=false diff --git a/tools/nixery/.github/workflows/build-and-test.yaml b/tools/nixery/.github/workflows/build-and-test.yaml deleted file mode 100644 index d3f258ffaa..0000000000 --- a/tools/nixery/.github/workflows/build-and-test.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Build Nixery, spin up an instance and pull an image from it. -name: "Build and test Nixery" -on: - push: - branches: - - master - pull_request: {} -env: - NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs/archive/4263ba5e133cc3fc699c1152ab5ee46ef668e675.tar.gz" -jobs: - build-and-test: - runs-on: ubuntu-latest - steps: - - name: Install Nix - uses: cachix/install-nix-action@v13 - - name: Checkout - uses: actions/checkout@v2.3.4 - - name: Prepare environment - run: nix-env -f '<nixpkgs>' -iA go - - name: Check formatting - run: "test -z $(gofmt -l .)" - - name: Run `go vet` - run: "go vet ./..." - - name: Build Nixery - run: "nix-build --no-out-link" - - name: Run integration test - run: scripts/integration-test.sh diff --git a/tools/nixery/CONTRIBUTING.md b/tools/nixery/CONTRIBUTING.md deleted file mode 100644 index ecad21b045..0000000000 --- a/tools/nixery/CONTRIBUTING.md +++ /dev/null @@ -1,35 +0,0 @@ -# How to Contribute - -We'd love to accept your patches and contributions to this project. There are -just a few small guidelines you need to follow. - -## Contributor License Agreement - -Contributions to this project must be accompanied by a Contributor License -Agreement. You (or your employer) retain the copyright to your contribution; -this simply gives us permission to use and redistribute your contributions as -part of the project. Head over to <https://cla.developers.google.com/> to see -your current agreements on file or to sign a new one. - -You generally only need to submit a CLA once, so if you've already submitted one -(even if it was for a different project), you probably don't need to do it -again. - -## Commit messages - -Commits in this repository follow the [Angular commit message -guidelines][commits]. - -## Code reviews - -All submissions, including submissions by project members, require review. We -use GitHub pull requests for this purpose. Consult -[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more -information on using pull requests. - -## Community Guidelines - -This project follows [Google's Open Source Community -Guidelines](https://opensource.google.com/conduct/). - -[commits]: https://github.com/angular/angular/blob/master/CONTRIBUTING.md#commit diff --git a/tools/nixery/README.md b/tools/nixery/README.md index cba8ce6b14..03515939a9 100644 --- a/tools/nixery/README.md +++ b/tools/nixery/README.md @@ -4,7 +4,7 @@ ----------------- -[](https://github.com/tazjin/nixery/actions/workflows/build-and-test.yaml) +[](https://buildkite.com/tvl/depot) **Nixery** is a Docker-compatible container registry that is capable of transparently building and serving container images using [Nix][]. @@ -24,6 +24,15 @@ You can watch the NixCon 2019 [talk about Nixery](https://www.youtube.com/watch?v=pOI9H4oeXqA) for more information about the project and its use-cases. +The canonical location of the Nixery source code is +[`//tools/nixery`][depot-link] in the [TVL](https://tvl.fyi) +monorepository. If cloning the entire repository is not desirable, the +Nixery subtree can be cloned like this: + + git clone https://code.tvl.fyi/depot.git:/tools/nixery.git + +The subtree is infrequently mirrored to `tazjin/nixery` on Github. + ## Demo Click the image to see an example in which an image containing an interactive @@ -139,8 +148,9 @@ separate Nix function, which will make it possible to build images directly in Nix builds. [Nix]: https://nixos.org/ -[layering strategy]: https://storage.googleapis.com/nixdoc/nixery-layers.html +[layering strategy]: https://tazj.in/blog/nixery-layers [gist]: https://gist.github.com/tazjin/08f3d37073b3590aacac424303e6f745 [buildLayeredImage]: https://grahamc.com/blog/nix-and-layered-docker-images [public]: https://nixery.dev +[depot-link]: https://cs.tvl.fyi/depot/-/tree/tools/nixery [gcs]: https://cloud.google.com/storage/ diff --git a/tools/nixery/builder/archive.go b/tools/nixery/builder/archive.go index ff822e389a..3bc02ab4d5 100644 --- a/tools/nixery/builder/archive.go +++ b/tools/nixery/builder/archive.go @@ -1,16 +1,5 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. +// Copyright 2022 The TVL Contributors +// SPDX-License-Identifier: Apache-2.0 package builder // This file implements logic for walking through a directory and creating a diff --git a/tools/nixery/builder/builder.go b/tools/nixery/builder/builder.go index 4279cb0a11..37c9b9fcb7 100644 --- a/tools/nixery/builder/builder.go +++ b/tools/nixery/builder/builder.go @@ -1,16 +1,5 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. +// Copyright 2022 The TVL Contributors +// SPDX-License-Identifier: Apache-2.0 // Package builder implements the logic for assembling container // images. It shells out to Nix to retrieve all required Nix-packages diff --git a/tools/nixery/builder/builder_test.go b/tools/nixery/builder/builder_test.go index 3fbe2ab40e..507f3eb15a 100644 --- a/tools/nixery/builder/builder_test.go +++ b/tools/nixery/builder/builder_test.go @@ -1,16 +1,5 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. +// Copyright 2022 The TVL Contributors +// SPDX-License-Identifier: Apache-2.0 package builder import ( diff --git a/tools/nixery/builder/cache.go b/tools/nixery/builder/cache.go index 35b563e524..9e4283c0e5 100644 --- a/tools/nixery/builder/cache.go +++ b/tools/nixery/builder/cache.go @@ -1,16 +1,5 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. +// Copyright 2022 The TVL Contributors +// SPDX-License-Identifier: Apache-2.0 package builder import ( diff --git a/tools/nixery/builder/layers.go b/tools/nixery/builder/layers.go index f769e43c58..5e37e62681 100644 --- a/tools/nixery/builder/layers.go +++ b/tools/nixery/builder/layers.go @@ -1,16 +1,5 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. +// Copyright 2022 The TVL Contributors +// SPDX-License-Identifier: Apache-2.0 // This package reads an export reference graph (i.e. a graph representing the // runtime dependencies of a set of derivations) created by Nix and groups it in diff --git a/tools/nixery/config/config.go b/tools/nixery/config/config.go index 8ea2edc28c..73ff5c8356 100644 --- a/tools/nixery/config/config.go +++ b/tools/nixery/config/config.go @@ -1,16 +1,5 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. +// Copyright 2022 The TVL Contributors +// SPDX-License-Identifier: Apache-2.0 // Package config implements structures to store Nixery's configuration at // runtime as well as the logic for instantiating this configuration from the diff --git a/tools/nixery/config/pkgsource.go b/tools/nixery/config/pkgsource.go index 55007bc806..c7508a4d3a 100644 --- a/tools/nixery/config/pkgsource.go +++ b/tools/nixery/config/pkgsource.go @@ -1,16 +1,5 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. +// Copyright 2022 The TVL Contributors +// SPDX-License-Identifier: Apache-2.0 package config import ( diff --git a/tools/nixery/default.nix b/tools/nixery/default.nix index e39db40a5a..6c0226f2fb 100644 --- a/tools/nixery/default.nix +++ b/tools/nixery/default.nix @@ -1,16 +1,5 @@ -# Copyright 2019-2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Copyright 2022 The TVL Contributors +# SPDX-License-Identifier: Apache-2.0 # This function header aims to provide compatibility between builds of # Nixery taking place inside/outside of the TVL depot. @@ -19,12 +8,13 @@ # build system and this will need some major adaptations to support # that. { depot ? { nix.readTree.drvTargets = x: x; } -, pkgs ? import <nixpkgs> {} +, pkgs ? import <nixpkgs> { } , preLaunch ? "" -, extraPackages ? [] +, extraPackages ? [ ] , maxLayers ? 20 , commitHash ? null -, ... }@args: +, ... +}@args: with pkgs; @@ -54,7 +44,8 @@ let "-ldflags=-s -w -X main.version=${nixery-commit-hash}" ]; }; -in depot.nix.readTree.drvTargets rec { +in +depot.nix.readTree.drvTargets rec { # Implementation of the Nix image building logic nixery-prepare-image = import ./prepare-image { inherit pkgs; }; @@ -79,55 +70,57 @@ in depot.nix.readTree.drvTargets rec { # Container image containing Nixery and Nix itself. This image can # be run on Kubernetes, published on AppEngine or whatever else is # desired. - nixery-image = let - # Wrapper script for the wrapper script (meta!) which configures - # the container environment appropriately. - # - # Most importantly, sandboxing is disabled to avoid privilege - # issues in containers. - nixery-launch-script = writeShellScriptBin "nixery" '' - set -e - export PATH=${coreutils}/bin:$PATH - export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt - mkdir -p /tmp - - # Create the build user/group required by Nix - echo 'nixbld:x:30000:nixbld' >> /etc/group - echo 'nixbld:x:30000:30000:nixbld:/tmp:/bin/bash' >> /etc/passwd - echo 'root:x:0:0:root:/root:/bin/bash' >> /etc/passwd - echo 'root:x:0:' >> /etc/group - - # Disable sandboxing to avoid running into privilege issues - mkdir -p /etc/nix - echo 'sandbox = false' >> /etc/nix/nix.conf - - # In some cases users building their own image might want to - # customise something on the inside (e.g. set up an environment - # for keys or whatever). + nixery-image = + let + # Wrapper script for the wrapper script (meta!) which configures + # the container environment appropriately. # - # This can be achieved by setting a 'preLaunch' script. - ${preLaunch} - - exec ${nixery-bin}/bin/nixery - ''; - in dockerTools.buildLayeredImage { - name = "nixery"; - config.Cmd = [ "${nixery-launch-script}/bin/nixery" ]; - - inherit maxLayers; - contents = [ - bashInteractive - cacert - coreutils - git - gnutar - gzip - iana-etc - nix - nixery-prepare-image - nixery-launch-script - openssh - zlib - ] ++ extraPackages; - }; + # Most importantly, sandboxing is disabled to avoid privilege + # issues in containers. + nixery-launch-script = writeShellScriptBin "nixery" '' + set -e + export PATH=${coreutils}/bin:$PATH + export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt + mkdir -p /tmp + + # Create the build user/group required by Nix + echo 'nixbld:x:30000:nixbld' >> /etc/group + echo 'nixbld:x:30000:30000:nixbld:/tmp:/bin/bash' >> /etc/passwd + echo 'root:x:0:0:root:/root:/bin/bash' >> /etc/passwd + echo 'root:x:0:' >> /etc/group + + # Disable sandboxing to avoid running into privilege issues + mkdir -p /etc/nix + echo 'sandbox = false' >> /etc/nix/nix.conf + + # In some cases users building their own image might want to + # customise something on the inside (e.g. set up an environment + # for keys or whatever). + # + # This can be achieved by setting a 'preLaunch' script. + ${preLaunch} + + exec ${nixery-bin}/bin/nixery + ''; + in + dockerTools.buildLayeredImage { + name = "nixery"; + config.Cmd = [ "${nixery-launch-script}/bin/nixery" ]; + + inherit maxLayers; + contents = [ + bashInteractive + cacert + coreutils + git + gnutar + gzip + iana-etc + nix + nixery-prepare-image + nixery-launch-script + openssh + zlib + ] ++ extraPackages; + }; } diff --git a/tools/nixery/docs/default.nix b/tools/nixery/docs/default.nix index d27cbe5b3e..876a34dcf1 100644 --- a/tools/nixery/docs/default.nix +++ b/tools/nixery/docs/default.nix @@ -1,16 +1,5 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Copyright 2022 The TVL Contributors +# SPDX-License-Identifier: Apache-2.0 # Builds the documentation page using the Rust project's 'mdBook' # tool. @@ -27,7 +16,8 @@ let rev = "9f0baf5e270128d9101ba4446cf6844889e399a2"; sha256 = "1pf9i90gn98vz67h296w5lnwhssk62dc6pij983dff42dbci7lhj"; }; -in runCommand "nixery-book" { } '' +in +runCommand "nixery-book" { } '' mkdir -p $out cp -r ${./.}/* . chmod -R a+w src diff --git a/tools/nixery/docs/src/nixery.md b/tools/nixery/docs/src/nixery.md index 7b78ddf5aa..d9ba179010 100644 --- a/tools/nixery/docs/src/nixery.md +++ b/tools/nixery/docs/src/nixery.md @@ -68,10 +68,6 @@ production project we recommend setting up a private instance. The public Nixery at `nixery.dev` is run on a best-effort basis and we make no guarantees about availability. -### Is this an official Google project? - -**No.** Nixery is not officially supported by Google. - ### Who made this? Nixery was written by [tazjin][], but many people have contributed to Nix over @@ -81,4 +77,4 @@ time, maybe you could become one of them? [Nix]: https://nixos.org/nix [layering strategy]: https://storage.googleapis.com/nixdoc/nixery-layers.html [layers]: https://grahamc.com/blog/nix-and-layered-docker-images -[tazjin]: https://github.com/tazjin +[tazjin]: https://tazj.in diff --git a/tools/nixery/docs/src/run-your-own.md b/tools/nixery/docs/src/run-your-own.md index cf4dc2ce61..7ed8bdd0bc 100644 --- a/tools/nixery/docs/src/run-your-own.md +++ b/tools/nixery/docs/src/run-your-own.md @@ -65,13 +65,17 @@ use it with your own packages. There are three options available: ### 2.1. With a container image -The easiest way to run Nixery is to build a container image. -This section assumes that the container runtime used is Docker, -please modify instructions accordingly if -you are using something else. +The easiest way to run Nixery is to build a container image. This +section assumes that the container runtime used is Docker, please +modify instructions accordingly if you are using something else. -With a working Nix installation, building Nixery is done by invoking `nix-build --A nixery-image` from a checkout of the [Nixery repository][repo]. +With a working Nix installation, you can clone and build the Nixery +image like this: + +``` +git clone https://code.tvl.fyi/depot.git:/tools/nixery.git +nix-build -A nixery-image +``` This will create a `result`-symlink which points to a tarball containing the image. In Docker, this tarball can be loaded by using `docker load -i result`. @@ -184,7 +188,6 @@ If the directory doesn't exist, Nixery will run fine but serve 404. [nixery#4]: https://github.com/tazjin/nixery/issues/4 [Nix]: https://nixos.org/nix [gcs]: https://cloud.google.com/storage/ -[repo]: https://github.com/tazjin/nixery [signed-urls]: under-the-hood.html#5-image-layers-are-requested [ADC]: https://cloud.google.com/docs/authentication/production#finding_credentials_automatically [nixinstall]: https://nixos.org/manual/nix/stable/installation/installing-binary.html diff --git a/tools/nixery/logs/logs.go b/tools/nixery/logs/logs.go index 4c755bc8ab..06adc701ef 100644 --- a/tools/nixery/logs/logs.go +++ b/tools/nixery/logs/logs.go @@ -1,16 +1,5 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. +// Copyright 2022 The TVL Contributors +// SPDX-License-Identifier: Apache-2.0 package logs // This file configures different log formatters via logrus. The diff --git a/tools/nixery/main.go b/tools/nixery/main.go index d94d51b468..2e633e0898 100644 --- a/tools/nixery/main.go +++ b/tools/nixery/main.go @@ -1,16 +1,5 @@ -// Copyright 2019-2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. +// Copyright 2022 The TVL Contributors +// SPDX-License-Identifier: Apache-2.0 // The nixery server implements a container registry that transparently builds // container images based on Nix derivations. diff --git a/tools/nixery/manifest/manifest.go b/tools/nixery/manifest/manifest.go index afe84072ea..d61514d2f6 100644 --- a/tools/nixery/manifest/manifest.go +++ b/tools/nixery/manifest/manifest.go @@ -1,16 +1,5 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. +// Copyright 2022 The TVL Contributors +// SPDX-License-Identifier: Apache-2.0 // Package image implements logic for creating the image metadata // (such as the image manifest and configuration). diff --git a/tools/nixery/popcount/default.nix b/tools/nixery/popcount/default.nix index bd695380cf..4b16768e4e 100644 --- a/tools/nixery/popcount/default.nix +++ b/tools/nixery/popcount/default.nix @@ -1,16 +1,5 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Copyright 2022 The TVL Contributors +# SPDX-License-Identifier: Apache-2.0 { buildGoPackage }: diff --git a/tools/nixery/popcount/popcount.go b/tools/nixery/popcount/popcount.go index dab10aae64..b83ac3ed1a 100644 --- a/tools/nixery/popcount/popcount.go +++ b/tools/nixery/popcount/popcount.go @@ -1,16 +1,5 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. +// Copyright 2022 The TVL Contributors +// SPDX-License-Identifier: Apache-2.0 // Popcount fetches popularity information for each store path in a // given Nix channel from the upstream binary cache. diff --git a/tools/nixery/prepare-image/default.nix b/tools/nixery/prepare-image/default.nix index 60b208f522..efd9ed3404 100644 --- a/tools/nixery/prepare-image/default.nix +++ b/tools/nixery/prepare-image/default.nix @@ -1,16 +1,5 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Copyright 2022 The TVL Contributors +# SPDX-License-Identifier: Apache-2.0 # This file builds a wrapper script called by Nixery to ask for the # content information for a given image. @@ -18,7 +7,7 @@ # The purpose of using a wrapper script is to ensure that the paths to # all required Nix files are set correctly at runtime. -{ pkgs ? import <nixpkgs> {} }: +{ pkgs ? import <nixpkgs> { } }: pkgs.writeShellScriptBin "nixery-prepare-image" '' exec ${pkgs.nix}/bin/nix-build \ diff --git a/tools/nixery/prepare-image/load-pkgs.nix b/tools/nixery/prepare-image/load-pkgs.nix index 4a89dcde3a..7f8ab5479d 100644 --- a/tools/nixery/prepare-image/load-pkgs.nix +++ b/tools/nixery/prepare-image/load-pkgs.nix @@ -1,16 +1,5 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Copyright 2022 The TVL Contributors +# SPDX-License-Identifier: Apache-2.0 # Load a Nix package set from one of the supported source types # (nixpkgs, git, path). @@ -24,7 +13,8 @@ let let url = "https://github.com/NixOS/nixpkgs/archive/${channel}.tar.gz"; - in import (fetchTarball url) importArgs; + in + import (fetchTarball url) importArgs; # If a git repository is requested, it is retrieved via # builtins.fetchGit which defaults to the git configuration of the @@ -35,7 +25,8 @@ let # No special handling is used for paths, so users are expected to pass one # that will work natively with Nix. importPath = path: import (toPath path) importArgs; -in if srcType == "nixpkgs" then +in +if srcType == "nixpkgs" then fetchImportChannel srcArgs else if srcType == "git" then fetchImportGit (fromJSON srcArgs) diff --git a/tools/nixery/prepare-image/prepare-image.nix b/tools/nixery/prepare-image/prepare-image.nix index acd1430548..bb88983cf6 100644 --- a/tools/nixery/prepare-image/prepare-image.nix +++ b/tools/nixery/prepare-image/prepare-image.nix @@ -1,16 +1,5 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Copyright 2022 The TVL Contributors +# SPDX-License-Identifier: Apache-2.0 # This file contains a derivation that outputs structured information # about the runtime dependencies of an image with a given set of @@ -23,13 +12,13 @@ { # Description of the package set to be used (will be loaded by load-pkgs.nix) - srcType ? "nixpkgs", - srcArgs ? "nixos-20.09", - system ? "x86_64-linux", - importArgs ? { }, - # Path to load-pkgs.nix - loadPkgs ? ./load-pkgs.nix, - # Packages to install by name (which must refer to top-level attributes of + srcType ? "nixpkgs" +, srcArgs ? "nixos-20.09" +, system ? "x86_64-linux" +, importArgs ? { } +, # Path to load-pkgs.nix + loadPkgs ? ./load-pkgs.nix +, # Packages to install by name (which must refer to top-level attributes of # nixpkgs). This is passed in as a JSON-array in string form. packages ? "[]" }: @@ -77,24 +66,28 @@ let # `deepFetch haskellpackages.stylish-haskell` retrieves # `haskellPackages.stylish-haskell`. deepFetch = with lib; s: n: - let path = splitString "." n; - err = { error = "not_found"; pkg = n; }; - # The most efficient way I've found to do a lookup against - # case-differing versions of an attribute is to first construct a - # mapping of all lowercased attribute names to their differently cased - # equivalents. - # - # This map is then used for a second lookup if the top-level - # (case-sensitive) one does not yield a result. - hasUpper = str: (match ".*[A-Z].*" str) != null; - allUpperKeys = filter hasUpper (attrNames s); - lowercased = listToAttrs (map (k: { + let + path = splitString "." n; + err = { error = "not_found"; pkg = n; }; + # The most efficient way I've found to do a lookup against + # case-differing versions of an attribute is to first construct a + # mapping of all lowercased attribute names to their differently cased + # equivalents. + # + # This map is then used for a second lookup if the top-level + # (case-sensitive) one does not yield a result. + hasUpper = str: (match ".*[A-Z].*" str) != null; + allUpperKeys = filter hasUpper (attrNames s); + lowercased = listToAttrs (map + (k: { name = toLower k; value = k; - }) allUpperKeys); - caseAmendedPath = map (v: if hasAttr v lowercased then lowercased."${v}" else v) path; - fetchLower = attrByPath caseAmendedPath err s; - in attrByPath path fetchLower s; + }) + allUpperKeys); + caseAmendedPath = map (v: if hasAttr v lowercased then lowercased."${v}" else v) path; + fetchLower = attrByPath caseAmendedPath err s; + in + attrByPath path fetchLower s; # allContents contains all packages successfully retrieved by name # from the package set, as well as any errors encountered while @@ -105,27 +98,30 @@ let # Folds over the results of 'deepFetch' on all requested packages to # separate them into errors and content. This allows the program to # terminate early and return only the errors if any are encountered. - let splitter = attrs: res: - if hasAttr "error" res - then attrs // { errors = attrs.errors ++ [ res ]; } - else attrs // { contents = attrs.contents ++ [ res ]; }; - init = { contents = []; errors = []; }; - fetched = (map (deepFetch pkgs) (fromJSON packages)); - in foldl' splitter init fetched; + let + splitter = attrs: res: + if hasAttr "error" res + then attrs // { errors = attrs.errors ++ [ res ]; } + else attrs // { contents = attrs.contents ++ [ res ]; }; + init = { contents = [ ]; errors = [ ]; }; + fetched = (map (deepFetch pkgs) (fromJSON packages)); + in + foldl' splitter init fetched; # Contains the export references graph of all retrieved packages, # which has information about all runtime dependencies of the image. # # This is used by Nixery to group closures into image layers. - runtimeGraph = runCommand "runtime-graph.json" { - __structuredAttrs = true; - exportReferencesGraph.graph = allContents.contents; - PATH = "${coreutils}/bin"; - builder = toFile "builder" '' - . .attrs.sh - cp .attrs.json ''${outputs[out]} - ''; - } ""; + runtimeGraph = runCommand "runtime-graph.json" + { + __structuredAttrs = true; + exportReferencesGraph.graph = allContents.contents; + PATH = "${coreutils}/bin"; + builder = toFile "builder" '' + . .attrs.sh + cp .attrs.json ''${outputs[out]} + ''; + } ""; # Create a symlink forest into all top-level store paths of the # image contents. @@ -151,7 +147,7 @@ let # Image layer that contains the symlink forest created above. This # must be included in the image to ensure that the filesystem has a # useful layout at runtime. - symlinkLayer = runCommand "symlink-layer.tar" {} '' + symlinkLayer = runCommand "symlink-layer.tar" { } '' cp -r ${contentsEnv}/ ./layer tar --transform='s|^\./||' -C layer --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 -cf $out . ''; @@ -159,9 +155,10 @@ let # Metadata about the symlink layer which is required for serving it. # Two different hashes are computed for different usages (inclusion # in manifest vs. content-checking in the layer cache). - symlinkLayerMeta = fromJSON (readFile (runCommand "symlink-layer-meta.json" { - buildInputs = [ coreutils jq openssl ]; - }'' + symlinkLayerMeta = fromJSON (readFile (runCommand "symlink-layer-meta.json" + { + buildInputs = [ coreutils jq openssl ]; + } '' tarHash=$(sha256sum ${symlinkLayer} | cut -d ' ' -f1) layerSize=$(stat --printf '%s' ${symlinkLayer}) @@ -181,7 +178,8 @@ let error = "not_found"; pkgs = map (err: err.pkg) allContents.errors; }; -in writeText "build-output.json" (if (length allContents.errors) == 0 - then toJSON buildOutput - else toJSON errorOutput +in +writeText "build-output.json" (if (length allContents.errors) == 0 +then toJSON buildOutput +else toJSON errorOutput ) diff --git a/tools/nixery/shell.nix b/tools/nixery/shell.nix index b37caa83ad..b91094722c 100644 --- a/tools/nixery/shell.nix +++ b/tools/nixery/shell.nix @@ -1,16 +1,5 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Copyright 2022 The TVL Contributors +# SPDX-License-Identifier: Apache-2.0 # Configures a shell environment that builds required local packages to # run Nixery. diff --git a/tools/nixery/storage/filesystem.go b/tools/nixery/storage/filesystem.go index 2be3457f32..3df4420f0f 100644 --- a/tools/nixery/storage/filesystem.go +++ b/tools/nixery/storage/filesystem.go @@ -1,16 +1,5 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. +// Copyright 2022 The TVL Contributors +// SPDX-License-Identifier: Apache-2.0 // Filesystem storage backend for Nixery. package storage diff --git a/tools/nixery/storage/gcs.go b/tools/nixery/storage/gcs.go index a4bb4ba31f..752c6bbd82 100644 --- a/tools/nixery/storage/gcs.go +++ b/tools/nixery/storage/gcs.go @@ -1,16 +1,5 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. +// Copyright 2022 The TVL Contributors +// SPDX-License-Identifier: Apache-2.0 // Google Cloud Storage backend for Nixery. package storage diff --git a/tools/nixery/storage/storage.go b/tools/nixery/storage/storage.go index fd496f440a..5500d61640 100644 --- a/tools/nixery/storage/storage.go +++ b/tools/nixery/storage/storage.go @@ -1,16 +1,5 @@ -// Copyright 2019-2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. +// Copyright 2022 The TVL Contributors +// SPDX-License-Identifier: Apache-2.0 // Package storage implements an interface that can be implemented by // storage backends, such as Google Cloud Storage or the local diff --git a/users/tazjin/blog/posts/nixery-layers.md b/users/tazjin/blog/posts/nixery-layers.md index 3f25ceadce..38ca2294a8 100644 --- a/users/tazjin/blog/posts/nixery-layers.md +++ b/users/tazjin/blog/posts/nixery-layers.md @@ -260,13 +260,13 @@ TIP: This is implemented in [popcount][] in Nixery. Hopefully this detailed design review was useful to you. You can also watch [my NixCon talk][talk] about Nixery for a review of some of this, and some demos. -[Nixery]: https://github.com/google/nixery +[Nixery]: https://cs.tvl.fyi/depot/-/tree/tools/nixery [grhmc]: https://grahamc.com/blog/nix-and-layered-docker-images [Nix]: https://nixos.org/nix [registry protocols]: https://github.com/opencontainers/distribution-spec/blob/master/spec.md [nixery.dev]: https://nixery.dev [dominator trees]: https://en.wikipedia.org/wiki/Dominator_(graph_theory) [gonum/graph]: https://godoc.org/gonum.org/v1/gonum/graph -[layers.go]: https://github.com/google/nixery/blob/master/builder/layers.go -[popcount]: https://github.com/google/nixery/tree/master/popcount +[layers.go]: https://cs.tvl.fyi/depot/-/blob/tools/nixery/builder/layers.go +[popcount]: https://cs.tvl.fyi/depot/-/tree/tools/nixery/popcount [talk]: https://www.youtube.com/watch?v=pOI9H4oeXqA |