5 files changed, 284 insertions, 0 deletions
diff --git a/tvix/castore/protos/LICENSE b/tvix/castore/protos/LICENSE
new file mode 100644
index 0000000000..2034ada6fd
--- /dev/null
+++ b/tvix/castore/protos/LICENSE
@@ -0,0 +1,21 @@
+Copyright © The Tvix Authors
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+“Software”), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
diff --git a/tvix/castore/protos/castore.proto b/tvix/castore/protos/castore.proto
new file mode 100644
index 0000000000..1ef4044045
--- /dev/null
+++ b/tvix/castore/protos/castore.proto
@@ -0,0 +1,71 @@
+// SPDX-FileCopyrightText: edef <edef@unfathomable.blue>
+// SPDX-License-Identifier: OSL-3.0 OR MIT OR Apache-2.0
+
+syntax = "proto3";
+
+package tvix.castore.v1;
+
+option go_package = "code.tvl.fyi/tvix/castore-go;castorev1";
+
+// A Directory can contain Directory, File or Symlink nodes.
+// Each of these nodes have a name attribute, which is the basename in that
+// directory and node type specific attributes.
+// The name attribute:
+//  - MUST not contain slashes or null bytes
+//  - MUST not be '.' or '..'
+//  - MUST be unique across all three lists
+// Elements in each list need to be lexicographically ordered by the name
+// attribute.
+message Directory {
+  repeated DirectoryNode directories = 1;
+  repeated FileNode files = 2;
+  repeated SymlinkNode symlinks = 3;
+}
+
+// A DirectoryNode represents a directory in a Directory.
+message DirectoryNode {
+  // The (base)name of the directory
+  bytes name = 1;
+  // The blake3 hash of a Directory message, serialized in protobuf canonical form.
+  bytes digest = 2;
+  // Number of child elements in the Directory referred to by `digest`.
+  // Calculated by summing up the numbers of `directories`, `files` and
+  // `symlinks`, and for each directory, its size field. Used for inode number
+  // calculation.
+  // This field is precisely as verifiable as any other Merkle tree edge.
+  // Resolve `digest`, and you can compute it incrementally. Resolve the entire
+  // tree, and you can fully compute it from scratch.
+  // A credulous implementation won't reject an excessive size, but this is
+  // harmless: you'll have some ordinals without nodes. Undersizing is obvious
+  // and easy to reject: you won't have an ordinal for some nodes.
+  uint64 size = 3;
+}
+
+// A FileNode represents a regular or executable file in a Directory.
+message FileNode {
+  // The (base)name of the file
+  bytes name = 1;
+  // The blake3 digest of the file contents
+  bytes digest = 2;
+  // The file content size
+  uint64 size = 3;
+  // Whether the file is executable
+  bool executable = 4;
+}
+
+// A SymlinkNode represents a symbolic link in a Directory.
+message SymlinkNode {
+  // The (base)name of the symlink
+  bytes name = 1;
+  // The target of the symlink.
+  bytes target = 2;
+}
+
+// A Node is either a DirectoryNode, FileNode or SymlinkNode.
+message Node {
+  oneof node {
+    DirectoryNode directory = 1;
+    FileNode file = 2;
+    SymlinkNode symlink = 3;
+  }
+}
diff --git a/tvix/castore/protos/default.nix b/tvix/castore/protos/default.nix
new file mode 100644
index 0000000000..feef55690f
--- /dev/null
+++ b/tvix/castore/protos/default.nix
@@ -0,0 +1,54 @@
+{ depot, pkgs, ... }:
+let
+  protos = depot.nix.sparseTree {
+    name = "castore-protos";
+    root = depot.path.origSrc;
+    paths = [
+      ./castore.proto
+      ./rpc_blobstore.proto
+      ./rpc_directory.proto
+      ../../../buf.yaml
+      ../../../buf.gen.yaml
+    ];
+  };
+in
+depot.nix.readTree.drvTargets {
+  inherit protos;
+
+  # Lints and ensures formatting of the proto files.
+  check = pkgs.stdenv.mkDerivation {
+    name = "proto-check";
+    src = protos;
+
+    nativeBuildInputs = [
+      pkgs.buf
+    ];
+
+    buildPhase = ''
+      export HOME=$TMPDIR
+      buf lint
+      buf format -d --exit-code
+      touch $out
+    '';
+  };
+
+  # Produces the golang bindings.
+  go-bindings = pkgs.stdenv.mkDerivation {
+    name = "go-bindings";
+    src = protos;
+
+    nativeBuildInputs = [
+      pkgs.buf
+      pkgs.protoc-gen-go
+      pkgs.protoc-gen-go-grpc
+    ];
+
+    buildPhase = ''
+      export HOME=$TMPDIR
+      buf generate
+
+      mkdir -p $out
+      cp tvix/castore/protos/*.pb.go $out/
+    '';
+  };
+}
diff --git a/tvix/castore/protos/rpc_blobstore.proto b/tvix/castore/protos/rpc_blobstore.proto
new file mode 100644
index 0000000000..eebe39ace7
--- /dev/null
+++ b/tvix/castore/protos/rpc_blobstore.proto
@@ -0,0 +1,85 @@
+// SPDX-License-Identifier: MIT
+// Copyright © 2022 The Tvix Authors
+syntax = "proto3";
+
+package tvix.castore.v1;
+
+option go_package = "code.tvl.fyi/tvix/castore-go;castorev1";
+
+// BlobService allows reading (or uploading) content-addressed blobs of data.
+// BLAKE3 is used as a hashing function for the data. Uploading a blob will
+// return the BLAKE3 digest of it, and that's the identifier used to Read/Stat
+// them too.
+service BlobService {
+  // Stat can be used to check for the existence of a blob, as well as
+  // gathering more data about it, like more granular chunking information
+  // or baos.
+  // Server implementations are not required to provide more granular chunking
+  // information, especially if the digest specified in `StatBlobRequest` is
+  // already a chunk of a blob.
+  rpc Stat(StatBlobRequest) returns (StatBlobResponse);
+
+  // Read allows reading (all) data of a blob/chunk by the BLAKE3 digest of
+  // its contents.
+  // If the backend communicated more granular chunks in the `Stat` request,
+  // this can also be used to read chunks.
+  // This request returns a stream of BlobChunk, which is just a container for
+  // a stream of bytes.
+  // The server may decide on whatever chunking it may seem fit as a size for
+  // the individual BlobChunk sent in the response stream, this is mostly to
+  // keep individual messages at a manageable size.
+  rpc Read(ReadBlobRequest) returns (stream BlobChunk);
+
+  // Put uploads a Blob, by reading a stream of bytes.
+  //
+  // The way the data is chunked up in individual BlobChunk messages sent in
+  // the stream has no effect on how the server ends up chunking blobs up, if
+  // it does at all.
+  rpc Put(stream BlobChunk) returns (PutBlobResponse);
+}
+
+message StatBlobRequest {
+  // The blake3 digest of the blob requested
+  bytes digest = 1;
+
+  // Whether the server should reply with a list of more granular chunks.
+  bool send_chunks = 2;
+
+  // Whether the server should reply with a bao.
+  bool send_bao = 3;
+}
+
+message StatBlobResponse {
+  // If `send_chunks` was set to true, this MAY contain a list of more
+  // granular chunks, which then may be read individually via the `Read`
+  // method.
+  repeated ChunkMeta chunks = 2;
+
+  message ChunkMeta {
+    // Digest of that specific chunk
+    bytes digest = 1;
+
+    // Length of that chunk, in bytes.
+    uint64 size = 2;
+  }
+
+  // If `send_bao` was set to true, this MAY contain a outboard bao.
+  // The exact format and message types here will still be fleshed out.
+  bytes bao = 3;
+}
+
+message ReadBlobRequest {
+  // The blake3 digest of the blob or chunk requested
+  bytes digest = 1;
+}
+
+// This represents some bytes of a blob.
+// Blobs are sent in smaller chunks to keep message sizes manageable.
+message BlobChunk {
+  bytes data = 1;
+}
+
+message PutBlobResponse {
+  // The blake3 digest of the data that was sent.
+  bytes digest = 1;
+}
diff --git a/tvix/castore/protos/rpc_directory.proto b/tvix/castore/protos/rpc_directory.proto
new file mode 100644
index 0000000000..f4f41c433a
--- /dev/null
+++ b/tvix/castore/protos/rpc_directory.proto
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: MIT
+// Copyright © 2022 The Tvix Authors
+syntax = "proto3";
+
+package tvix.castore.v1;
+
+import "tvix/castore/protos/castore.proto";
+
+option go_package = "code.tvl.fyi/tvix/castore-go;castorev1";
+
+service DirectoryService {
+  // Get retrieves a stream of Directory messages, by using the lookup
+  // parameters in GetDirectoryRequest.
+  // Keep in mind multiple DirectoryNodes in different parts of the graph might
+  // have the same digest if they have the same underlying contents,
+  // so sending subsequent ones can be omitted.
+  //
+  // It is okay for certain implementations to only allow retrieval of
+  // Directory digests that are at the "root", aka the last element that's
+  // sent in a Put. This makes sense for implementations bundling closures of
+  // directories together in batches.
+  rpc Get(GetDirectoryRequest) returns (stream Directory);
+
+  // Put uploads a graph of Directory messages.
+  // Individual Directory messages need to be send in an order walking up
+  // from the leaves to the root - a Directory message can only refer to
+  // Directory messages previously sent in the same stream.
+  // Keep in mind multiple DirectoryNodes in different parts of the graph might
+  // have the same digest if they have the same underlying contents,
+  // so sending subsequent ones can be omitted.
+  // We might add a separate method, allowing to send partial graphs at a later
+  // time, if requiring to send the full graph turns out to be a problem.
+  rpc Put(stream Directory) returns (PutDirectoryResponse);
+}
+
+message GetDirectoryRequest {
+  oneof by_what {
+    // The blake3 hash of the (root) Directory message, serialized in
+    // protobuf canonical form.
+    // Keep in mind this can be a subtree of another root.
+    bytes digest = 1;
+  }
+
+  // If set to true, recursively resolve all child Directory messages.
+  // Directory messages SHOULD be streamed in a recursive breadth-first walk,
+  // but other orders are also fine, as long as Directory messages are only
+  // sent after they are referred to from previously sent Directory messages.
+  bool recursive = 2;
+}
+
+message PutDirectoryResponse {
+  bytes root_digest = 1;
+}