diff options
Diffstat (limited to 'tvix/castore/protos')
-rw-r--r-- | tvix/castore/protos/LICENSE | 21 | ||||
-rw-r--r-- | tvix/castore/protos/castore.proto | 71 | ||||
-rw-r--r-- | tvix/castore/protos/default.nix | 48 | ||||
-rw-r--r-- | tvix/castore/protos/rpc_blobstore.proto | 85 | ||||
-rw-r--r-- | tvix/castore/protos/rpc_directory.proto | 53 |
5 files changed, 278 insertions, 0 deletions
diff --git a/tvix/castore/protos/LICENSE b/tvix/castore/protos/LICENSE new file mode 100644 index 000000000000..2034ada6fd9a --- /dev/null +++ b/tvix/castore/protos/LICENSE @@ -0,0 +1,21 @@ +Copyright © The Tvix Authors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +“Software”), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/tvix/castore/protos/castore.proto b/tvix/castore/protos/castore.proto new file mode 100644 index 000000000000..1ef404404504 --- /dev/null +++ b/tvix/castore/protos/castore.proto @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: edef <edef@unfathomable.blue> +// SPDX-License-Identifier: OSL-3.0 OR MIT OR Apache-2.0 + +syntax = "proto3"; + +package tvix.castore.v1; + +option go_package = "code.tvl.fyi/tvix/castore-go;castorev1"; + +// A Directory can contain Directory, File or Symlink nodes. +// Each of these nodes have a name attribute, which is the basename in that +// directory and node type specific attributes. +// The name attribute: +// - MUST not contain slashes or null bytes +// - MUST not be '.' or '..' +// - MUST be unique across all three lists +// Elements in each list need to be lexicographically ordered by the name +// attribute. +message Directory { + repeated DirectoryNode directories = 1; + repeated FileNode files = 2; + repeated SymlinkNode symlinks = 3; +} + +// A DirectoryNode represents a directory in a Directory. +message DirectoryNode { + // The (base)name of the directory + bytes name = 1; + // The blake3 hash of a Directory message, serialized in protobuf canonical form. + bytes digest = 2; + // Number of child elements in the Directory referred to by `digest`. + // Calculated by summing up the numbers of `directories`, `files` and + // `symlinks`, and for each directory, its size field. Used for inode number + // calculation. + // This field is precisely as verifiable as any other Merkle tree edge. + // Resolve `digest`, and you can compute it incrementally. Resolve the entire + // tree, and you can fully compute it from scratch. + // A credulous implementation won't reject an excessive size, but this is + // harmless: you'll have some ordinals without nodes. Undersizing is obvious + // and easy to reject: you won't have an ordinal for some nodes. + uint64 size = 3; +} + +// A FileNode represents a regular or executable file in a Directory. +message FileNode { + // The (base)name of the file + bytes name = 1; + // The blake3 digest of the file contents + bytes digest = 2; + // The file content size + uint64 size = 3; + // Whether the file is executable + bool executable = 4; +} + +// A SymlinkNode represents a symbolic link in a Directory. +message SymlinkNode { + // The (base)name of the symlink + bytes name = 1; + // The target of the symlink. + bytes target = 2; +} + +// A Node is either a DirectoryNode, FileNode or SymlinkNode. +message Node { + oneof node { + DirectoryNode directory = 1; + FileNode file = 2; + SymlinkNode symlink = 3; + } +} diff --git a/tvix/castore/protos/default.nix b/tvix/castore/protos/default.nix new file mode 100644 index 000000000000..08bb8fcfeef1 --- /dev/null +++ b/tvix/castore/protos/default.nix @@ -0,0 +1,48 @@ +{ depot, pkgs, lib, ... }: +let + protos = lib.sourceByRegex depot.path.origSrc [ + "buf.yaml" + "buf.gen.yaml" + "^tvix(/castore(/protos(/.*\.proto)?)?)?$" + ]; +in +depot.nix.readTree.drvTargets { + inherit protos; + + # Lints and ensures formatting of the proto files. + check = pkgs.stdenv.mkDerivation { + name = "proto-check"; + src = protos; + + nativeBuildInputs = [ + pkgs.buf + ]; + + buildPhase = '' + export HOME=$TMPDIR + buf lint + buf format -d --exit-code + touch $out + ''; + }; + + # Produces the golang bindings. + go-bindings = pkgs.stdenv.mkDerivation { + name = "go-bindings"; + src = protos; + + nativeBuildInputs = [ + pkgs.buf + pkgs.protoc-gen-go + pkgs.protoc-gen-go-grpc + ]; + + buildPhase = '' + export HOME=$TMPDIR + buf generate + + mkdir -p $out + cp tvix/castore/protos/*.pb.go $out/ + ''; + }; +} diff --git a/tvix/castore/protos/rpc_blobstore.proto b/tvix/castore/protos/rpc_blobstore.proto new file mode 100644 index 000000000000..eebe39ace7b3 --- /dev/null +++ b/tvix/castore/protos/rpc_blobstore.proto @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: MIT +// Copyright © 2022 The Tvix Authors +syntax = "proto3"; + +package tvix.castore.v1; + +option go_package = "code.tvl.fyi/tvix/castore-go;castorev1"; + +// BlobService allows reading (or uploading) content-addressed blobs of data. +// BLAKE3 is used as a hashing function for the data. Uploading a blob will +// return the BLAKE3 digest of it, and that's the identifier used to Read/Stat +// them too. +service BlobService { + // Stat can be used to check for the existence of a blob, as well as + // gathering more data about it, like more granular chunking information + // or baos. + // Server implementations are not required to provide more granular chunking + // information, especially if the digest specified in `StatBlobRequest` is + // already a chunk of a blob. + rpc Stat(StatBlobRequest) returns (StatBlobResponse); + + // Read allows reading (all) data of a blob/chunk by the BLAKE3 digest of + // its contents. + // If the backend communicated more granular chunks in the `Stat` request, + // this can also be used to read chunks. + // This request returns a stream of BlobChunk, which is just a container for + // a stream of bytes. + // The server may decide on whatever chunking it may seem fit as a size for + // the individual BlobChunk sent in the response stream, this is mostly to + // keep individual messages at a manageable size. + rpc Read(ReadBlobRequest) returns (stream BlobChunk); + + // Put uploads a Blob, by reading a stream of bytes. + // + // The way the data is chunked up in individual BlobChunk messages sent in + // the stream has no effect on how the server ends up chunking blobs up, if + // it does at all. + rpc Put(stream BlobChunk) returns (PutBlobResponse); +} + +message StatBlobRequest { + // The blake3 digest of the blob requested + bytes digest = 1; + + // Whether the server should reply with a list of more granular chunks. + bool send_chunks = 2; + + // Whether the server should reply with a bao. + bool send_bao = 3; +} + +message StatBlobResponse { + // If `send_chunks` was set to true, this MAY contain a list of more + // granular chunks, which then may be read individually via the `Read` + // method. + repeated ChunkMeta chunks = 2; + + message ChunkMeta { + // Digest of that specific chunk + bytes digest = 1; + + // Length of that chunk, in bytes. + uint64 size = 2; + } + + // If `send_bao` was set to true, this MAY contain a outboard bao. + // The exact format and message types here will still be fleshed out. + bytes bao = 3; +} + +message ReadBlobRequest { + // The blake3 digest of the blob or chunk requested + bytes digest = 1; +} + +// This represents some bytes of a blob. +// Blobs are sent in smaller chunks to keep message sizes manageable. +message BlobChunk { + bytes data = 1; +} + +message PutBlobResponse { + // The blake3 digest of the data that was sent. + bytes digest = 1; +} diff --git a/tvix/castore/protos/rpc_directory.proto b/tvix/castore/protos/rpc_directory.proto new file mode 100644 index 000000000000..f4f41c433a76 --- /dev/null +++ b/tvix/castore/protos/rpc_directory.proto @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: MIT +// Copyright © 2022 The Tvix Authors +syntax = "proto3"; + +package tvix.castore.v1; + +import "tvix/castore/protos/castore.proto"; + +option go_package = "code.tvl.fyi/tvix/castore-go;castorev1"; + +service DirectoryService { + // Get retrieves a stream of Directory messages, by using the lookup + // parameters in GetDirectoryRequest. + // Keep in mind multiple DirectoryNodes in different parts of the graph might + // have the same digest if they have the same underlying contents, + // so sending subsequent ones can be omitted. + // + // It is okay for certain implementations to only allow retrieval of + // Directory digests that are at the "root", aka the last element that's + // sent in a Put. This makes sense for implementations bundling closures of + // directories together in batches. + rpc Get(GetDirectoryRequest) returns (stream Directory); + + // Put uploads a graph of Directory messages. + // Individual Directory messages need to be send in an order walking up + // from the leaves to the root - a Directory message can only refer to + // Directory messages previously sent in the same stream. + // Keep in mind multiple DirectoryNodes in different parts of the graph might + // have the same digest if they have the same underlying contents, + // so sending subsequent ones can be omitted. + // We might add a separate method, allowing to send partial graphs at a later + // time, if requiring to send the full graph turns out to be a problem. + rpc Put(stream Directory) returns (PutDirectoryResponse); +} + +message GetDirectoryRequest { + oneof by_what { + // The blake3 hash of the (root) Directory message, serialized in + // protobuf canonical form. + // Keep in mind this can be a subtree of another root. + bytes digest = 1; + } + + // If set to true, recursively resolve all child Directory messages. + // Directory messages SHOULD be streamed in a recursive breadth-first walk, + // but other orders are also fine, as long as Directory messages are only + // sent after they are referred to from previously sent Directory messages. + bool recursive = 2; +} + +message PutDirectoryResponse { + bytes root_digest = 1; +} |