diff options
Diffstat (limited to 'tvix/store/protos')
-rw-r--r-- | tvix/store/protos/LICENSE | 21 | ||||
-rw-r--r-- | tvix/store/protos/default.nix | 50 | ||||
-rw-r--r-- | tvix/store/protos/pathinfo.proto | 128 | ||||
-rw-r--r-- | tvix/store/protos/rpc_pathinfo.proto | 76 |
4 files changed, 275 insertions, 0 deletions
diff --git a/tvix/store/protos/LICENSE b/tvix/store/protos/LICENSE new file mode 100644 index 000000000000..2034ada6fd9a --- /dev/null +++ b/tvix/store/protos/LICENSE @@ -0,0 +1,21 @@ +Copyright © The Tvix Authors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +“Software”), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/tvix/store/protos/default.nix b/tvix/store/protos/default.nix new file mode 100644 index 000000000000..005a2697e5e9 --- /dev/null +++ b/tvix/store/protos/default.nix @@ -0,0 +1,50 @@ +{ depot, pkgs, lib, ... }: +let + protos = lib.sourceByRegex depot.path.origSrc [ + "buf.yaml" + "buf.gen.yaml" + # We need to include castore.proto (only), as it's referred. + "^tvix(/castore(/protos(/castore\.proto)?)?)?$" + "^tvix(/store(/protos(/.*\.proto)?)?)?$" + ]; +in +depot.nix.readTree.drvTargets { + inherit protos; + + # Lints and ensures formatting of the proto files. + check = pkgs.stdenv.mkDerivation { + name = "proto-check"; + src = protos; + + nativeBuildInputs = [ + pkgs.buf + ]; + + buildPhase = '' + export HOME=$TMPDIR + buf lint + buf format -d --exit-code + touch $out + ''; + }; + + # Produces the golang bindings. + go-bindings = pkgs.stdenv.mkDerivation { + name = "go-bindings"; + src = protos; + + nativeBuildInputs = [ + pkgs.buf + pkgs.protoc-gen-go + pkgs.protoc-gen-go-grpc + ]; + + buildPhase = '' + export HOME=$TMPDIR + buf generate + + mkdir -p $out + cp tvix/store/protos/*.pb.go $out/ + ''; + }; +} diff --git a/tvix/store/protos/pathinfo.proto b/tvix/store/protos/pathinfo.proto new file mode 100644 index 000000000000..b03e7e938e33 --- /dev/null +++ b/tvix/store/protos/pathinfo.proto @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: MIT +// Copyright © 2022 The Tvix Authors +syntax = "proto3"; + +package tvix.store.v1; + +import "tvix/castore/protos/castore.proto"; + +option go_package = "code.tvl.fyi/tvix/store-go;storev1"; + +// PathInfo shows information about a Nix Store Path. +// That's a single element inside /nix/store. +message PathInfo { + // The path can be a directory, file or symlink. + tvix.castore.v1.Node node = 1; + + // List of references (output path hashes) + // This really is the raw *bytes*, after decoding nixbase32, and not a + // base32-encoded string. + repeated bytes references = 2; + + // see below. + NARInfo narinfo = 3; +} + +// Represents a path in the Nix store (a direct child of STORE_DIR). +// It is commonly formatted by a nixbase32-encoding the digest, and +// concatenating the name, separated by a `-`. +message StorePath { + // The string after digest and `-`. + string name = 1; + + // The digest (20 bytes). + bytes digest = 2; +} + +// Nix C++ uses NAR (Nix Archive) as a format to transfer store paths, +// and stores metadata and signatures in NARInfo files. +// Store all these attributes in a separate message. +// +// This is useful to render .narinfo files to clients, or to preserve/validate +// these signatures. +// As verifying these signatures requires the whole NAR file to be synthesized, +// moving to another signature scheme is desired. +// Even then, it still makes sense to hold this data, for old clients. +message NARInfo { + // This represents a (parsed) signature line in a .narinfo file. + message Signature { + string name = 1; + bytes data = 2; + } + + // This size of the NAR file, in bytes. + uint64 nar_size = 1; + + // The sha256 of the NAR file representation. + bytes nar_sha256 = 2; + + // The signatures in a .narinfo file. + repeated Signature signatures = 3; + + // A list of references. To validate .narinfo signatures, a fingerprint needs + // to be constructed. + // This fingerprint doesn't just contain the hashes of the output paths of all + // references (like PathInfo.references), but their whole (base)names, so we + // need to keep them somewhere. + repeated string reference_names = 4; + + // The StorePath of the .drv file producing this output. + // The .drv suffix is omitted in its `name` field. + StorePath deriver = 5; + + // The CA field in the .narinfo. + // Its textual representations seen in the wild are one of the following: + // - `fixed:r:sha256:1gcky5hlf5vqfzpyhihydmm54grhc94mcs8w7xr8613qsqb1v2j6` + // fixed-output derivations using "recursive" `outputHashMode`. + // - `fixed:sha256:19xqkh72crbcba7flwxyi3n293vav6d7qkzkh2v4zfyi4iia8vj8 + // fixed-output derivations using "flat" `outputHashMode` + // - `text:sha256:19xqkh72crbcba7flwxyi3n293vav6d7qkzkh2v4zfyi4iia8vj8` + // Text hashing, used for uploaded .drv files and outputs produced by + // builtins.toFile. + // + // Semantically, they can be split into the following components: + // - "content address prefix". Currently, "fixed" and "text" are supported. + // - "hash mode". Currently, "flat" and "recursive" are supported. + // - "hash type". The underlying hash function used. + // Currently, sha1, md5, sha256, sha512. + // - "digest". The digest itself. + // + // There are some restrictions on the possible combinations. + // For example, `text` and `fixed:recursive` always imply sha256. + // + // We use an enum to encode the possible combinations, and optimize for the + // common case, `fixed:recursive`, identified as `NAR_SHA256`. + CA ca = 6; + + message CA { + enum Hash { + // produced when uploading fixed-output store paths using NAR-based + // hashing (`outputHashMode = "recursive"`). + NAR_SHA256 = 0; + NAR_SHA1 = 1; + NAR_SHA512 = 2; + NAR_MD5 = 3; + + // Produced when uploading .drv files or outputs produced by + // builtins.toFile. + // Produces equivalent digests as FLAT_SHA256, but is a separate + // hashing type in Nix, affecting output path calculation. + TEXT_SHA256 = 4; + + // Produced when using fixed-output derivations with + // `outputHashMode = "flat"`. + FLAT_SHA1 = 5; + FLAT_MD5 = 6; + FLAT_SHA256 = 7; + FLAT_SHA512 = 8; + + // TODO: what happens in Rust if we introduce a new enum kind here? + } + + // The hashing type used. + Hash type = 1; + + // The digest, in raw bytes. + bytes digest = 2; + } +} diff --git a/tvix/store/protos/rpc_pathinfo.proto b/tvix/store/protos/rpc_pathinfo.proto new file mode 100644 index 000000000000..c1c91658ada2 --- /dev/null +++ b/tvix/store/protos/rpc_pathinfo.proto @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: MIT +// Copyright © 2022 The Tvix Authors +syntax = "proto3"; + +package tvix.store.v1; + +import "tvix/castore/protos/castore.proto"; +import "tvix/store/protos/pathinfo.proto"; + +option go_package = "code.tvl.fyi/tvix/store-go;storev1"; + +service PathInfoService { + // Return a PathInfo message matching the criteria specified in the + // GetPathInfoRequest message. + rpc Get(GetPathInfoRequest) returns (PathInfo); + + // Upload a PathInfo object to the remote end. It MUST not return until the + // PathInfo object has been written on the the remote end. + // + // The remote end MAY check if a potential DirectoryNode has already been + // uploaded. + // + // Uploading clients SHOULD obviously not steer other machines to try to + // substitute before from the remote end before having finished uploading + // PathInfo, Directories and Blobs. + // The returned PathInfo object MAY contain additional narinfo signatures, but + // is otherwise left untouched. + rpc Put(PathInfo) returns (PathInfo); + + // Calculate the NAR representation of the contents specified by the + // root_node. The calculation SHOULD be cached server-side for subsequent + // requests. + // + // All references (to blobs or Directory messages) MUST already exist in the + // store. + // + // The method can be used to produce a Nix fixed-output path, which contains + // the (compressed) sha256 of the NAR content representation in the root_node + // name (suffixed with the name). + // + // It can also be used to calculate arbitrary NAR hashes of output paths, in + // case a legacy Nix Binary Cache frontend is provided. + rpc CalculateNAR(tvix.castore.v1.Node) returns (CalculateNARResponse); + + // Return a stream of PathInfo messages matching the criteria specified in + // ListPathInfoRequest. + rpc List(ListPathInfoRequest) returns (stream PathInfo); +} + +// The parameters that can be used to lookup a (single) PathInfo object. +// Currently, only a lookup by output hash is supported. +message GetPathInfoRequest { + oneof by_what { + // The output hash of a nix path (20 bytes). + // This is the nixbase32-decoded portion of a Nix output path, so to substitute + // /nix/store/xm35nga2g20mz5sm5l6n8v3bdm86yj83-cowsay-3.04 + // this field would contain nixbase32dec("xm35nga2g20mz5sm5l6n8v3bdm86yj83"). + bytes by_output_hash = 1; + } +} + +// The parameters that can be used to lookup (multiple) PathInfo objects. +// Currently no filtering is possible, all objects are returned. +message ListPathInfoRequest {} + +// CalculateNARResponse is the response returned by the CalculateNAR request. +// +// It contains the size of the NAR representation (in bytes), and the sha56 +// digest. +message CalculateNARResponse { + // This size of the NAR file, in bytes. + uint64 nar_size = 1; + + // The sha256 of the NAR file representation. + bytes nar_sha256 = 2; +} |