diff options
| -rw-r--r-- | tvix/Cargo.lock | 12 | ||||
| -rw-r--r-- | tvix/Cargo.nix | 54 | ||||
| -rw-r--r-- | tvix/Cargo.toml | 1 | ||||
| -rw-r--r-- | tvix/build/Cargo.toml | 21 | ||||
| -rw-r--r-- | tvix/build/build.rs | 38 | ||||
| -rw-r--r-- | tvix/build/default.nix | 5 | ||||
| -rw-r--r-- | tvix/build/protos/LICENSE | 21 | ||||
| -rw-r--r-- | tvix/build/protos/build.proto | 121 | ||||
| -rw-r--r-- | tvix/build/protos/default.nix | 35 | ||||
| -rw-r--r-- | tvix/build/protos/rpc_build.proto | 13 | ||||
| -rw-r--r-- | tvix/build/src/lib.rs | 1 | ||||
| -rw-r--r-- | tvix/build/src/proto/mod.rs | 1 | ||||
| -rw-r--r-- | tvix/default.nix | 5 | ||||
| -rw-r--r-- | tvix/proto/default.nix | 2 |
14 files changed, 330 insertions, 0 deletions
diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock index d7f4bba1a5..518845af7b 100644 --- a/tvix/Cargo.lock +++ b/tvix/Cargo.lock @@ -3051,6 +3051,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] +name = "tvix-build" +version = "0.1.0" +dependencies = [ + "prost", + "prost-build", + "tonic", + "tonic-build", + "tonic-reflection", + "tvix-castore", +] + +[[package]] name = "tvix-castore" version = "0.1.0" dependencies = [ diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix index 9553d0c88f..6151fd76d3 100644 --- a/tvix/Cargo.nix +++ b/tvix/Cargo.nix @@ -43,6 +43,16 @@ rec { # File a bug if you depend on any for non-debug work! debug = internal.debugCrate { inherit packageId; }; }; + "tvix-build" = rec { + packageId = "tvix-build"; + build = internal.buildRustCrateWithFeatures { + packageId = "tvix-build"; + }; + + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; "tvix-castore" = rec { packageId = "tvix-castore"; build = internal.buildRustCrateWithFeatures { @@ -9348,6 +9358,50 @@ rec { ]; }; + "tvix-build" = rec { + crateName = "tvix-build"; + version = "0.1.0"; + edition = "2021"; + # We can't filter paths with references in Nix 2.4 + # See https://github.com/NixOS/nix/issues/5410 + src = + if ((lib.versionOlder builtins.nixVersion "2.4pre20211007") || (lib.versionOlder "2.5" builtins.nixVersion)) + then lib.cleanSourceWith { filter = sourceFilter; src = ./build; } + else ./build; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + } + { + name = "tonic"; + packageId = "tonic"; + } + { + name = "tonic-reflection"; + packageId = "tonic-reflection"; + optional = true; + } + { + name = "tvix-castore"; + packageId = "tvix-castore"; + } + ]; + buildDependencies = [ + { + name = "prost-build"; + packageId = "prost-build"; + } + { + name = "tonic-build"; + packageId = "tonic-build"; + } + ]; + features = { + "tonic-reflection" = [ "dep:tonic-reflection" ]; + }; + resolvedDefaultFeatures = [ "default" "tonic-reflection" ]; + }; "tvix-castore" = rec { crateName = "tvix-castore"; version = "0.1.0"; diff --git a/tvix/Cargo.toml b/tvix/Cargo.toml index 0642c32388..44f5a69874 100644 --- a/tvix/Cargo.toml +++ b/tvix/Cargo.toml @@ -19,6 +19,7 @@ resolver = "2" members = [ + "build", "castore", "cli", "eval", diff --git a/tvix/build/Cargo.toml b/tvix/build/Cargo.toml new file mode 100644 index 0000000000..6f9d8a34f2 --- /dev/null +++ b/tvix/build/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "tvix-build" +version = "0.1.0" +edition = "2021" + +[dependencies] +prost = "0.12.1" +tonic = "0.10.2" +tvix-castore = { path = "../castore" } + +[dependencies.tonic-reflection] +optional = true +version = "0.10.2" + +[build-dependencies] +prost-build = "0.12.1" +tonic-build = "0.10.2" + +[features] +default = [] +tonic-reflection = ["dep:tonic-reflection"] diff --git a/tvix/build/build.rs b/tvix/build/build.rs new file mode 100644 index 0000000000..d75ebf7143 --- /dev/null +++ b/tvix/build/build.rs @@ -0,0 +1,38 @@ +use std::io::Result; + +fn main() -> Result<()> { + #[allow(unused_mut)] + let mut builder = tonic_build::configure(); + + #[cfg(feature = "tonic-reflection")] + { + let out_dir = std::path::PathBuf::from(std::env::var("OUT_DIR").unwrap()); + let descriptor_path = out_dir.join("tvix.build.v1.bin"); + + builder = builder.file_descriptor_set_path(descriptor_path); + }; + + // https://github.com/hyperium/tonic/issues/908 + let mut config = prost_build::Config::new(); + config.bytes(["."]); + config.extern_path(".tvix.castore.v1", "::tvix_castore::proto"); + + builder + .build_server(true) + .build_client(true) + .compile_with_config( + config, + &[ + "tvix/build/protos/build.proto", + "tvix/build/protos/rpc_build.proto", + ], + // If we are in running `cargo build` manually, using `../..` works fine, + // but in case we run inside a nix build, we need to instead point PROTO_ROOT + // to a sparseTree containing that structure. + &[match std::env::var_os("PROTO_ROOT") { + Some(proto_root) => proto_root.to_str().unwrap().to_owned(), + None => "../..".to_string(), + }], + )?; + Ok(()) +} diff --git a/tvix/build/default.nix b/tvix/build/default.nix new file mode 100644 index 0000000000..a2a3bea0c5 --- /dev/null +++ b/tvix/build/default.nix @@ -0,0 +1,5 @@ +{ depot, pkgs, ... }: + +depot.tvix.crates.workspaceMembers.tvix-build.build.override { + runTests = true; +} diff --git a/tvix/build/protos/LICENSE b/tvix/build/protos/LICENSE new file mode 100644 index 0000000000..2034ada6fd --- /dev/null +++ b/tvix/build/protos/LICENSE @@ -0,0 +1,21 @@ +Copyright © The Tvix Authors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +“Software”), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/tvix/build/protos/build.proto b/tvix/build/protos/build.proto new file mode 100644 index 0000000000..75146a8bd3 --- /dev/null +++ b/tvix/build/protos/build.proto @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: MIT +// Copyright © 2022 The Tvix Authors + +syntax = "proto3"; + +package tvix.build.v1; + +import "tvix/castore/protos/castore.proto"; + +option go_package = "code.tvl.fyi/tvix/build-go;buildv1"; + +// A BuildRequest describes the request of something to be run on the builder. +// It is distinct from an actual [Build] that has already happened, or might be +// currently ongoing. +// +// A BuildRequest can be seen as a more normalized version of a Derivation +// (parsed from A-Term), "writing out" some of the Nix-internal details about +// how e.g. environment variables in the build are set. +// +// Nix has some impurities when building a Derivation, for example the --cores option +// ends up as an environment variable in the build, that's not part of the ATerm. +// +// As of now, we serialize this into the BuildRequest, so builders can stay dumb. +// This might change in the future. +// +// There's also a big difference when it comes to how inputs are modelled: +// - Nix only uses store path (strings) to describe the inputs. +// As store paths can be input-addressed, a certain store path can contain +// different contents (as not all store paths are binary reproducible). +// This requires that for every input-addressed input, the builder has access +// to either the input's deriver (and needs to build it) or else a trusted +// source for the built input. +// to upload input-addressed paths, requiring the trusted users concept. +// - tvix-build records a list of tvix.castore.v1.Node as inputs. +// These map from the store path base name to their contents, relieving the +// builder from having to "trust" any input-addressed paths, contrary to Nix. +// +// While this approach gives a better hermeticity, it has one downside: +// A BuildRequest can only be sent once the contents of all its inputs are known. +// +// As of now, we're okay to accept this, but it prevents uploading an +// entirely-non-IFD subgraph of BuildRequests eagerly. +// +// FUTUREWORK: We might be introducing another way to refer to inputs, to +// support "send all BuildRequest for a nixpkgs eval to a remote builder and put +// the laptop to sleep" usecases later. +message BuildRequest { + // The command (and its args) executed as the build script. + // In the case of a Nix derivation, this is usually + // ["/path/to/some-bash/bin/bash", "-e", "/path/to/some/builder.sh"]. + repeated string command_args = 1; + + // The list of outputs the build is expected to produce. + // These are basenames inside /nix/store. + // If the path is not produced, the build is considered to have failed. + // Outputs are sorted. + repeated string outputs = 2; + + // The list of environment variables and their values that should be set + // inside the build environment. + // This includes both environment vars set inside the derivation, as well as + // more "ephemeral" ones like NIX_BUILD_CORES, controlled by the `--cores` + // CLI option of `nix-build`. + // For now, we consume this as an option when turning a Derivation into a BuildRequest, + // similar to how Nix has a `--cores` option. + // We don't want to bleed these very nix-specific sandbox impl details into + // (dumber) builders if we don't have to. + // Environment variables are sorted by their keys. + repeated EnvVar environment_vars = 3; + + message EnvVar { + string key = 1; + bytes value = 2; + } + + // The list of all root nodes that should be visible in /nix/store at the + // time of the build. + // As root nodes are content-addressed, no additional signatures are needed + // to substitute / make these available in the build environment. + // Inputs are sorted by their names. + repeated tvix.castore.v1.Node inputs = 4; + + // A set of constraints that need to be satisfied on a build host before a + // Build can be started. + BuildConstraints constraints = 5; + + // BuildConstraints represents certain conditions that must be fulfilled + // inside the build environment to be able to build this. + // Constraints can be things like required architecture and minimum amount of memory. + // The required input paths are *not* represented in here, because it + // wouldn't be hermetic enough - see the comment around inputs too. + message BuildConstraints { + // The system that's needed to execute the build. + string system = 1; + + // The amount of memory required to be available for the build, in bytes. + uint64 min_memory = 2; + + // A list of (absolute) paths that need to be available in the build + // environment. + // TBD, This is probably things like /dev/kvm, but no nix store paths. + repeated string available_ro_paths = 3; + } + + // TODO: allow describing something like "preferLocal", to influence composition? +} + +// A Build is (one possible) outcome of executing a [BuildRequest]. +message Build { + // The orginal build request producing the build. + BuildRequest build_request = 1; // <- TODO: define hashing scheme for BuildRequest, refer to it by hash? + + // The outputs that were produced after successfully building. + // They are sorted by their names. + repeated tvix.castore.v1.Node outputs = 2; + + // TODO: where did this run, how long, logs, … +} + +/// TODO: check remarkable notes on constraints again +/// TODO: https://github.com/adisbladis/go-nix/commit/603df5db86ab97ba29f6f94d74f4e51642c56834 diff --git a/tvix/build/protos/default.nix b/tvix/build/protos/default.nix new file mode 100644 index 0000000000..138bc2fb71 --- /dev/null +++ b/tvix/build/protos/default.nix @@ -0,0 +1,35 @@ +{ depot, pkgs, ... }: { + # Produces the golang bindings. + go-bindings = pkgs.stdenv.mkDerivation { + name = "go-bindings"; + + src = depot.nix.sparseTree { + name = "build-protos"; + root = depot.path.origSrc; + paths = [ + # We need to include castore.proto (only), as it's referred. + ../../castore/protos/castore.proto + ./build.proto + ./rpc_build.proto + ../../../buf.yaml + ../../../buf.gen.yaml + ]; + }; + + nativeBuildInputs = [ + pkgs.buf + pkgs.protoc-gen-go + pkgs.protoc-gen-go-grpc + ]; + + buildPhase = '' + export HOME=$TMPDIR + buf lint + buf format -d --exit-code + buf generate + + mkdir -p $out + cp tvix/build/protos/*.pb.go $out/ + ''; + }; +} diff --git a/tvix/build/protos/rpc_build.proto b/tvix/build/protos/rpc_build.proto new file mode 100644 index 0000000000..73eebf78fe --- /dev/null +++ b/tvix/build/protos/rpc_build.proto @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: MIT +// Copyright © 2022 The Tvix Authors +syntax = "proto3"; + +package tvix.build.v1; + +import "tvix/build/protos/build.proto"; + +option go_package = "code.tvl.fyi/tvix/build-go;buildv1"; + +service BuildService { + rpc DoBuild(BuildRequest) returns (Build); +} diff --git a/tvix/build/src/lib.rs b/tvix/build/src/lib.rs new file mode 100644 index 0000000000..febacec69e --- /dev/null +++ b/tvix/build/src/lib.rs @@ -0,0 +1 @@ +pub mod proto; diff --git a/tvix/build/src/proto/mod.rs b/tvix/build/src/proto/mod.rs new file mode 100644 index 0000000000..b0360dcd4f --- /dev/null +++ b/tvix/build/src/proto/mod.rs @@ -0,0 +1 @@ +tonic::include_proto!("tvix.build.v1"); diff --git a/tvix/default.nix b/tvix/default.nix index a6878f3c15..ac800ce6a1 100644 --- a/tvix/default.nix +++ b/tvix/default.nix @@ -40,6 +40,11 @@ let nativeBuildInputs = protobufDep prev; }; + tvix-build = prev: { + PROTO_ROOT = depot.tvix.proto; + nativeBuildInputs = protobufDep prev; + }; + tvix-castore = prev: { PROTO_ROOT = depot.tvix.proto; nativeBuildInputs = protobufDep prev; diff --git a/tvix/proto/default.nix b/tvix/proto/default.nix index 0ee102e4f9..934a73fe33 100644 --- a/tvix/proto/default.nix +++ b/tvix/proto/default.nix @@ -6,6 +6,8 @@ depot.nix.sparseTree { name = "tvix-protos"; root = depot.path.origSrc; paths = [ + ../build/protos/build.proto + ../build/protos/rpc_build.proto ../castore/protos/castore.proto ../castore/protos/rpc_blobstore.proto ../castore/protos/rpc_directory.proto |