1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
|
// SPDX-License-Identifier: MIT
// Copyright © 2022 The Tvix Authors
syntax = "proto3";
package tvix.build.v1;
import "tvix/castore/protos/castore.proto";
option go_package = "code.tvl.fyi/tvix/build-go;buildv1";
// A BuildRequest describes the request of something to be run on the builder.
// It is distinct from an actual [Build] that has already happened, or might be
// currently ongoing.
//
// A BuildRequest can be seen as a more normalized version of a Derivation
// (parsed from A-Term), "writing out" some of the Nix-internal details about
// how e.g. environment variables in the build are set.
//
// Nix has some impurities when building a Derivation, for example the --cores option
// ends up as an environment variable in the build, that's not part of the ATerm.
//
// As of now, we serialize this into the BuildRequest, so builders can stay dumb.
// This might change in the future.
//
// There's also a big difference when it comes to how inputs are modelled:
// - Nix only uses store path (strings) to describe the inputs.
// As store paths can be input-addressed, a certain store path can contain
// different contents (as not all store paths are binary reproducible).
// This requires that for every input-addressed input, the builder has access
// to either the input's deriver (and needs to build it) or else a trusted
// source for the built input.
// to upload input-addressed paths, requiring the trusted users concept.
// - tvix-build records a list of tvix.castore.v1.Node as inputs.
// These map from the store path base name to their contents, relieving the
// builder from having to "trust" any input-addressed paths, contrary to Nix.
//
// While this approach gives a better hermeticity, it has one downside:
// A BuildRequest can only be sent once the contents of all its inputs are known.
//
// As of now, we're okay to accept this, but it prevents uploading an
// entirely-non-IFD subgraph of BuildRequests eagerly.
//
// FUTUREWORK: We might be introducing another way to refer to inputs, to
// support "send all BuildRequest for a nixpkgs eval to a remote builder and put
// the laptop to sleep" usecases later.
message BuildRequest {
// The list of all root nodes that should be visible in `inputs_dir` at the
// time of the build.
// As all references are content-addressed, no additional signatures are
// needed to substitute / make these available in the build environment.
// Inputs MUST be sorted by their names.
repeated tvix.castore.v1.Node inputs = 1;
// The command (and its args) executed as the build script.
// In the case of a Nix derivation, this is usually
// ["/path/to/some-bash/bin/bash", "-e", "/path/to/some/builder.sh"].
repeated string command_args = 2;
// The working dir of the command, relative to the build root.
// "build", in the case of Nix.
// This MUST be a clean relative path, without any ".", "..", or superfluous
// slashes.
string working_dir = 3;
// A list of "scratch" paths, relative to the build root.
// These will be write-able during the build.
// [build, nix/store] in the case of Nix.
// These MUST be clean relative paths, without any ".", "..", or superfluous
// slashes, and sorted.
repeated string scratch_paths = 4;
// The path where the castore input nodes will be located at,
// "nix/store" in case of Nix.
// Builds might also write into here (Nix builds do that).
// This MUST be a clean relative path, without any ".", "..", or superfluous
// slashes.
string inputs_dir = 5;
// The list of output paths the build is expected to produce,
// relative to the root.
// If the path is not produced, the build is considered to have failed.
// These MUST be clean relative paths, without any ".", "..", or superfluous
// slashes, and sorted.
repeated string outputs = 6;
// The list of environment variables and their values that should be set
// inside the build environment.
// This includes both environment vars set inside the derivation, as well as
// more "ephemeral" ones like NIX_BUILD_CORES, controlled by the `--cores`
// CLI option of `nix-build`.
// For now, we consume this as an option when turning a Derivation into a BuildRequest,
// similar to how Nix has a `--cores` option.
// We don't want to bleed these very nix-specific sandbox impl details into
// (dumber) builders if we don't have to.
// Environment variables are sorted by their keys.
repeated EnvVar environment_vars = 7;
message EnvVar {
// name of the environment variable. Must not contain =.
string key = 1;
bytes value = 2;
}
// A set of constraints that need to be satisfied on a build host before a
// Build can be started.
BuildConstraints constraints = 8;
// BuildConstraints represents certain conditions that must be fulfilled
// inside the build environment to be able to build this.
// Constraints can be things like required architecture and minimum amount of memory.
// The required input paths are *not* represented in here, because it
// wouldn't be hermetic enough - see the comment around inputs too.
message BuildConstraints {
// The system that's needed to execute the build.
// Must not be empty.
string system = 1;
// The amount of memory required to be available for the build, in bytes.
uint64 min_memory = 2;
// A list of (absolute) paths that need to be available in the build
// environment, like `/dev/kvm`.
// This is distinct from the castore nodes in inputs.
// TODO: check if these should be individual constraints instead.
// These MUST be clean absolute paths, without any ".", "..", or superfluous
// slashes, and sorted.
repeated string available_ro_paths = 3;
// Whether the build should be able to access the network,
bool network_access = 4;
// Whether to provide a /bin/sh inside the build environment, usually a static bash.
bool provide_bin_sh = 5;
}
// Additional (small) files and their contents that should be placed into the
// build environment, but outside inputs_dir.
// Used for passAsFile and structuredAttrs in Nix.
repeated AdditionalFile additional_files = 9;
message AdditionalFile {
string path = 1;
bytes contents = 2;
}
// If this is an non-empty list, all paths in `outputs` are scanned for these.
// For Nix, `refscan_needles` would be populated with the nixbase32 hash parts of
// every input store path and output store path. The latter is necessary to scan
// for references between multi-output derivations.
repeated string refscan_needles = 10;
// TODO: allow describing something like "preferLocal", to influence composition?
}
// A Build is (one possible) outcome of executing a [BuildRequest].
message Build {
// The orginal build request producing the build.
BuildRequest build_request = 1; // <- TODO: define hashing scheme for BuildRequest, refer to it by hash?
// The outputs that were produced after successfully building.
// They are sorted by their names.
repeated tvix.castore.v1.Node outputs = 2;
message OutputNeedles {
// The numbers are indexing into `refscan_needles` originally specified in the BuildRequest.
repeated uint64 needles = 1;
}
// Contains the same number of elements as the `outputs` field.
repeated OutputNeedles outputs_needles = 3;
// TODO: where did this run, how long, logs, …
}
/// TODO: check remarkable notes on constraints again
/// TODO: https://github.com/adisbladis/go-nix/commit/603df5db86ab97ba29f6f94d74f4e51642c56834
|
