diff options
| author | Florian Klink <flokli@flokli.de> | 2023-09-21T19·32+0300 |
|---|---|---|
| committer | clbot <clbot@tvl.fyi> | 2023-09-22T12·51+0000 |
| commit | 32f41458c0a0f62bf906021ef096c465ccc45581 (patch) | |
| tree | 3aaab8c453871f39c46fb43f8278aa933b24519d | |
| parent | d8ef0cfb4a859af7e33828b013356412d02532da (diff) | |
refactor(tvix): move castore into tvix-castore crate r/6629
This splits the pure content-addressed layers from tvix-store into a `castore` crate, and only leaves PathInfo related things, as well as the CLI entrypoint in the tvix-store crate. Notable changes: - `fixtures` and `utils` had to be moved out of the `test` cfg, so they can be imported from tvix-store. - Some ad-hoc fixtures in the test were moved to proper fixtures in the same step. - The protos are now created by a (more static) recipe in the protos/ directory. The (now two) golang targets are commented out, as it's not possible to update them properly in the same CL. This will be done by a followup CL once this is merged (and whitby deployed) Bug: https://b.tvl.fyi/issues/301 Change-Id: I8d675d4bf1fb697eb7d479747c1b1e3635718107 Reviewed-on: https://cl.tvl.fyi/c/depot/+/9370 Reviewed-by: tazjin <tazjin@tvl.su> Reviewed-by: flokli <flokli@flokli.de> Autosubmit: flokli <flokli@flokli.de> Tested-by: BuildkiteCI Reviewed-by: Connor Brewster <cbrewster@hey.com>
89 files changed, 2311 insertions, 1832 deletions
diff --git a/nix/bufCheck/default.nix b/nix/bufCheck/default.nix index f247e6839eee..3a645133049a 100644 --- a/nix/bufCheck/default.nix +++ b/nix/bufCheck/default.nix @@ -7,6 +7,7 @@ pkgs.writeShellScriptBin "ci-buf-check" '' (cd $(git rev-parse --show-toplevel) && buf lint .) # Run buf generate, and bail out if generated files are changed. + (cd $(git rev-parse --show-toplevel) && buf generate --path tvix/castore/protos) (cd $(git rev-parse --show-toplevel) && buf generate --path tvix/store/protos) # Check if any files have changed if [[ -n "$(git status --porcelain -unormal)" ]]; then diff --git a/ops/modules/www/code.tvl.fyi.nix b/ops/modules/www/code.tvl.fyi.nix index 5a7a9193ae13..f3edbd3b8730 100644 --- a/ops/modules/www/code.tvl.fyi.nix +++ b/ops/modules/www/code.tvl.fyi.nix @@ -20,6 +20,10 @@ alias ${depot.tvix.docs.svg}/component-flow.svg; } + location = /go-get/tvix/castore/protos { + alias ${pkgs.writeText "go-import-metadata.html" ''<html><meta name="go-import" content="code.tvl.fyi/tvix/castore/protos git https://code.tvl.fyi/depot.git:/tvix/castore/protos.git"></html>''}; + } + location = /go-get/tvix/store/protos { alias ${pkgs.writeText "go-import-metadata.html" ''<html><meta name="go-import" content="code.tvl.fyi/tvix/store/protos git https://code.tvl.fyi/depot.git:/tvix/store/protos.git"></html>''}; } diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock index ba0c4d9af87b..174c68d4d2d8 100644 --- a/tvix/Cargo.lock +++ b/tvix/Cargo.lock @@ -2731,6 +2731,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] +name = "tvix-castore" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-stream", + "blake3", + "bytes", + "data-encoding", + "futures", + "lazy_static", + "pin-project-lite", + "prost", + "prost-build", + "sled", + "tempfile", + "test-case", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "tonic", + "tonic-build", + "tonic-mock", + "tonic-reflection", + "tower", + "tracing", + "url", + "walkdir", +] + +[[package]] name = "tvix-cli" version = "0.1.0" dependencies = [ @@ -2742,6 +2773,7 @@ dependencies = [ "thiserror", "tokio", "tracing", + "tvix-castore", "tvix-eval", "tvix-store", "wu-manber", @@ -2832,6 +2864,7 @@ dependencies = [ "tower", "tracing", "tracing-subscriber", + "tvix-castore", "url", "walkdir", ] diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix index 947f8ab825e1..2f01a9473263 100644 --- a/tvix/Cargo.nix +++ b/tvix/Cargo.nix @@ -53,6 +53,16 @@ rec { # File a bug if you depend on any for non-debug work! debug = internal.debugCrate { inherit packageId; }; }; + "tvix-castore" = rec { + packageId = "tvix-castore"; + build = internal.buildRustCrateWithFeatures { + packageId = "tvix-castore"; + }; + + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; "tvix-cli" = rec { packageId = "tvix-cli"; build = internal.buildRustCrateWithFeatures { @@ -8115,6 +8125,135 @@ rec { ]; }; + "tvix-castore" = rec { + crateName = "tvix-castore"; + version = "0.1.0"; + edition = "2021"; + # We can't filter paths with references in Nix 2.4 + # See https://github.com/NixOS/nix/issues/5410 + src = + if (lib.versionOlder builtins.nixVersion "2.4pre20211007") + then lib.cleanSourceWith { filter = sourceFilter; src = ./castore; } + else ./castore; + dependencies = [ + { + name = "anyhow"; + packageId = "anyhow"; + } + { + name = "async-stream"; + packageId = "async-stream"; + } + { + name = "blake3"; + packageId = "blake3"; + features = [ "rayon" "std" ]; + } + { + name = "bytes"; + packageId = "bytes"; + } + { + name = "data-encoding"; + packageId = "data-encoding"; + } + { + name = "futures"; + packageId = "futures"; + } + { + name = "lazy_static"; + packageId = "lazy_static"; + } + { + name = "pin-project-lite"; + packageId = "pin-project-lite"; + } + { + name = "prost"; + packageId = "prost"; + } + { + name = "sled"; + packageId = "sled"; + features = [ "compression" ]; + } + { + name = "thiserror"; + packageId = "thiserror"; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "fs" "net" "rt-multi-thread" "signal" ]; + } + { + name = "tokio-stream"; + packageId = "tokio-stream"; + features = [ "fs" "net" ]; + } + { + name = "tokio-util"; + packageId = "tokio-util"; + features = [ "io" "io-util" ]; + } + { + name = "tonic"; + packageId = "tonic"; + } + { + name = "tonic-reflection"; + packageId = "tonic-reflection"; + optional = true; + } + { + name = "tower"; + packageId = "tower"; + } + { + name = "tracing"; + packageId = "tracing"; + } + { + name = "url"; + packageId = "url"; + } + { + name = "walkdir"; + packageId = "walkdir"; + } + ]; + buildDependencies = [ + { + name = "prost-build"; + packageId = "prost-build"; + } + { + name = "tonic-build"; + packageId = "tonic-build"; + } + ]; + devDependencies = [ + { + name = "tempfile"; + packageId = "tempfile"; + } + { + name = "test-case"; + packageId = "test-case"; + } + { + name = "tonic-mock"; + packageId = "tonic-mock"; + } + ]; + features = { + "default" = [ "reflection" ]; + "reflection" = [ "tonic-reflection" ]; + "tonic-reflection" = [ "dep:tonic-reflection" ]; + }; + resolvedDefaultFeatures = [ "default" "reflection" "tonic-reflection" ]; + }; "tvix-cli" = rec { crateName = "tvix-cli"; version = "0.1.0"; @@ -8167,6 +8306,10 @@ rec { packageId = "tracing"; } { + name = "tvix-castore"; + packageId = "tvix-castore"; + } + { name = "tvix-eval"; packageId = "tvix-eval"; } @@ -8510,6 +8653,10 @@ rec { features = [ "json" ]; } { + name = "tvix-castore"; + packageId = "tvix-castore"; + } + { name = "url"; packageId = "url"; } diff --git a/tvix/Cargo.toml b/tvix/Cargo.toml index 938364fc26b6..645aac362738 100644 --- a/tvix/Cargo.toml +++ b/tvix/Cargo.toml @@ -19,6 +19,7 @@ resolver = "2" members = [ + "castore", "cli", "eval", "eval/builtin-macros", diff --git a/tvix/castore/Cargo.toml b/tvix/castore/Cargo.toml new file mode 100644 index 000000000000..8bef5bb147bc --- /dev/null +++ b/tvix/castore/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "tvix-castore" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0.68" +async-stream = "0.3.5" +blake3 = { version = "1.3.1", features = ["rayon", "std"] } +bytes = "1.4.0" +data-encoding = "2.3.3" +futures = "0.3.28" +lazy_static = "1.4.0" +pin-project-lite = "0.2.13" +prost = "0.11.2" +sled = { version = "0.34.7", features = ["compression"] } +thiserror = "1.0.38" +tokio-stream = { version = "0.1.14", features = ["fs", "net"] } +tokio-util = { version = "0.7.8", features = ["io", "io-util"] } +tokio = { version = "1.28.0", features = ["fs", "net", "rt-multi-thread", "signal"] } +tonic = "0.8.2" +tower = "0.4.13" +tracing = "0.1.37" +url = "2.4.0" +walkdir = "2.4.0" + +[dependencies.tonic-reflection] +optional = true +version = "0.5.0" + +[build-dependencies] +prost-build = "0.11.2" +tonic-build = "0.8.2" + +[dev-dependencies] +test-case = "2.2.2" +tempfile = "3.3.0" +tonic-mock = { git = "https://github.com/brainrake/tonic-mock", branch = "bump-dependencies" } + +[features] +default = ["reflection"] +reflection = ["tonic-reflection"] \ No newline at end of file diff --git a/tvix/castore/build.rs b/tvix/castore/build.rs new file mode 100644 index 000000000000..9f06c8833319 --- /dev/null +++ b/tvix/castore/build.rs @@ -0,0 +1,38 @@ +use std::io::Result; + +fn main() -> Result<()> { + #[allow(unused_mut)] + let mut builder = tonic_build::configure(); + + #[cfg(feature = "reflection")] + { + let out_dir = std::path::PathBuf::from(std::env::var("OUT_DIR").unwrap()); + let descriptor_path = out_dir.join("tvix.castore.v1.bin"); + + builder = builder.file_descriptor_set_path(descriptor_path); + }; + + // https://github.com/hyperium/tonic/issues/908 + let mut config = prost_build::Config::new(); + config.bytes(["."]); + + builder + .build_server(true) + .build_client(true) + .compile_with_config( + config, + &[ + "tvix/castore/protos/castore.proto", + "tvix/castore/protos/rpc_blobstore.proto", + "tvix/castore/protos/rpc_directory.proto", + ], + // If we are in running `cargo build` manually, using `../..` works fine, + // but in case we run inside a nix build, we need to instead point PROTO_ROOT + // to a sparseTree containing that structure. + &[match std::env::var_os("PROTO_ROOT") { + Some(proto_root) => proto_root.to_str().unwrap().to_owned(), + None => "../..".to_string(), + }], + )?; + Ok(()) +} diff --git a/tvix/castore/default.nix b/tvix/castore/default.nix new file mode 100644 index 000000000000..09f6c0bea343 --- /dev/null +++ b/tvix/castore/default.nix @@ -0,0 +1,5 @@ +{ depot, pkgs, ... }: + +depot.tvix.crates.workspaceMembers.tvix-castore.build.override { + runTests = true; +} diff --git a/tvix/castore/protos/LICENSE b/tvix/castore/protos/LICENSE new file mode 100644 index 000000000000..2034ada6fd9a --- /dev/null +++ b/tvix/castore/protos/LICENSE @@ -0,0 +1,21 @@ +Copyright © The Tvix Authors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +“Software”), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/tvix/store/protos/castore.go b/tvix/castore/protos/castore.go index 4ab7ab42887a..102ba4bff75d 100644 --- a/tvix/store/protos/castore.go +++ b/tvix/castore/protos/castore.go @@ -1,4 +1,4 @@ -package storev1 +package castorev1 import ( "bytes" diff --git a/tvix/castore/protos/castore.pb.go b/tvix/castore/protos/castore.pb.go new file mode 100644 index 000000000000..5323d6c923a8 --- /dev/null +++ b/tvix/castore/protos/castore.pb.go @@ -0,0 +1,580 @@ +// SPDX-FileCopyrightText: edef <edef@unfathomable.blue> +// SPDX-License-Identifier: OSL-3.0 OR MIT OR Apache-2.0 + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc (unknown) +// source: tvix/castore/protos/castore.proto + +package castorev1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// A Directory can contain Directory, File or Symlink nodes. +// Each of these nodes have a name attribute, which is the basename in that directory +// and node type specific attributes. +// The name attribute: +// - MUST not contain slashes or null bytes +// - MUST not be '.' or '..' +// - MUST be unique across all three lists +// +// Elements in each list need to be lexicographically ordered by the name +// attribute. +type Directory struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Directories []*DirectoryNode `protobuf:"bytes,1,rep,name=directories,proto3" json:"directories,omitempty"` + Files []*FileNode `protobuf:"bytes,2,rep,name=files,proto3" json:"files,omitempty"` + Symlinks []*SymlinkNode `protobuf:"bytes,3,rep,name=symlinks,proto3" json:"symlinks,omitempty"` +} + +func (x *Directory) Reset() { + *x = Directory{} + if protoimpl.UnsafeEnabled { + mi := &file_tvix_castore_protos_castore_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Directory) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Directory) ProtoMessage() {} + +func (x *Directory) ProtoReflect() protoreflect.Message { + mi := &file_tvix_castore_protos_castore_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Directory.ProtoReflect.Descriptor instead. +func (*Directory) Descriptor() ([]byte, []int) { + return file_tvix_castore_protos_castore_proto_rawDescGZIP(), []int{0} +} + +func (x *Directory) GetDirectories() []*DirectoryNode { + if x != nil { + return x.Directories + } + return nil +} + +func (x *Directory) GetFiles() []*FileNode { + if x != nil { + return x.Files + } + return nil +} + +func (x *Directory) GetSymlinks() []*SymlinkNode { + if x != nil { + return x.Symlinks + } + return nil +} + +// A DirectoryNode represents a directory in a Directory. +type DirectoryNode struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The (base)name of the directory + Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // The blake3 hash of a Directory message, serialized in protobuf canonical form. + Digest []byte `protobuf:"bytes,2,opt,name=digest,proto3" json:"digest,omitempty"` + // Number of child elements in the Directory referred to by `digest`. + // Calculated by summing up the numbers of `directories`, `files` and + // `symlinks`, and for each directory, its size field. Used for inode + // number calculation. + // This field is precisely as verifiable as any other Merkle tree edge. + // Resolve `digest`, and you can compute it incrementally. Resolve the + // entire tree, and you can fully compute it from scratch. + // A credulous implementation won't reject an excessive size, but this is + // harmless: you'll have some ordinals without nodes. Undersizing is + // obvious and easy to reject: you won't have an ordinal for some nodes. + Size uint32 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"` +} + +func (x *DirectoryNode) Reset() { + *x = DirectoryNode{} + if protoimpl.UnsafeEnabled { + mi := &file_tvix_castore_protos_castore_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DirectoryNode) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DirectoryNode) ProtoMessage() {} + +func (x *DirectoryNode) ProtoReflect() protoreflect.Message { + mi := &file_tvix_castore_protos_castore_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DirectoryNode.ProtoReflect.Descriptor instead. +func (*DirectoryNode) Descriptor() ([]byte, []int) { + return file_tvix_castore_protos_castore_proto_rawDescGZIP(), []int{1} +} + +func (x *DirectoryNode) GetName() []byte { + if x != nil { + return x.Name + } + return nil +} + +func (x *DirectoryNode) GetDigest() []byte { + if x != nil { + return x.Digest + } + return nil +} + +func (x *DirectoryNode) GetSize() uint32 { + if x != nil { + return x.Size + } + return 0 +} + +// A FileNode represents a regular or executable file in a Directory. +type FileNode struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The (base)name of the file + Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // The blake3 digest of the file contents + Digest []byte `protobuf:"bytes,2,opt,name=digest,proto3" json:"digest,omitempty"` + // The file content size + Size uint32 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"` + // Whether the file is executable + Executable bool `protobuf:"varint,4,opt,name=executable,proto3" json:"executable,omitempty"` +} + +func (x *FileNode) Reset() { + *x = FileNode{} + if protoimpl.UnsafeEnabled { + mi := &file_tvix_castore_protos_castore_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FileNode) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileNode) ProtoMessage() {} + +func (x *FileNode) ProtoReflect() protoreflect.Message { + mi := &file_tvix_castore_protos_castore_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileNode.ProtoReflect.Descriptor instead. +func (*FileNode) Descriptor() ([]byte, []int) { + return file_tvix_castore_protos_castore_proto_rawDescGZIP(), []int{2} +} + +func (x *FileNode) GetName() []byte { + if x != nil { + return x.Name + } + return nil +} + +func (x *FileNode) GetDigest() []byte { + if x != nil { + return x.Digest + } + return nil +} + +func (x *FileNode) GetSize() uint32 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *FileNode) GetExecutable() bool { + if x != nil { + return x.Executable + } + return false +} + +// A SymlinkNode represents a symbolic link in a Directory. +type SymlinkNode struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The (base)name of the symlink + Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // The target of the symlink. + Target []byte `protobuf:"bytes,2,opt,name=target,proto3" json:"target,omitempty"` +} + +func (x *SymlinkNode) Reset() { + *x = SymlinkNode{} + if protoimpl.UnsafeEnabled { + mi := &file_tvix_castore_protos_castore_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SymlinkNode) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SymlinkNode) ProtoMessage() {} + +func (x *SymlinkNode) ProtoReflect() protoreflect.Message { + mi := &file_tvix_castore_protos_castore_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SymlinkNode.ProtoReflect.Descriptor instead. +func (*SymlinkNode) Descriptor() ([]byte, []int) { + return file_tvix_castore_protos_castore_proto_rawDescGZIP(), []int{3} +} + +func (x *SymlinkNode) GetName() []byte { + if x != nil { + return x.Name + } + return nil +} + +func (x *SymlinkNode) GetTarget() []byte { + if x != nil { + return x.Target + } + return nil +} + +// A Node is either a DirectoryNode, FileNode or SymlinkNode. +type Node struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to Node: + // + // *Node_Directory + // *Node_File + // *Node_Symlink + Node isNode_Node `protobuf_oneof:"node"` +} + +func (x *Node) Reset() { + *x = Node{} + if protoimpl.UnsafeEnabled { + mi := &file_tvix_castore_protos_castore_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Node) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Node) ProtoMessage() {} + +func (x *Node) ProtoReflect() protoreflect.Message { + mi := &file_tvix_castore_protos_castore_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Node.ProtoReflect.Descriptor instead. +func (*Node) Descriptor() ([]byte, []int) { + return file_tvix_castore_protos_castore_proto_rawDescGZIP(), []int{4} +} + +func (m *Node) GetNode() isNode_Node { + if m != nil { + return m.Node + } + return nil +} + +func (x *Node) GetDirectory() *DirectoryNode { + if x, ok := x.GetNode().(*Node_Directory); ok { + return x.Directory + } + return nil +} + +func (x *Node) GetFile() *FileNode { + if x, ok := x.GetNode().(*Node_File); ok { + return x.File + } + return nil +} + +func (x *Node) GetSymlink() *SymlinkNode { + if x, ok := x.GetNode().(*Node_Symlink); ok { + return x.Symlink + } + return nil +} + +type isNode_Node interface { + isNode_Node() +} + +type Node_Directory struct { + Directory *DirectoryNode `protobuf:"bytes,1,opt,name=directory,proto3,oneof"` +} + +type Node_File struct { + File *FileNode `protobuf:"bytes,2,opt,name=file,proto3,oneof"` +} + +type Node_Symlink struct { + Symlink *SymlinkNode `protobuf:"bytes,3,opt,name=symlink,proto3,oneof"` +} + +func (*Node_Directory) isNode_Node() {} + +func (*Node_File) isNode_Node() {} + +func (*Node_Symlink) isNode_Node() {} + +var File_tvix_castore_protos_castore_proto protoreflect.FileDescriptor + +var file_tvix_castore_protos_castore_proto_rawDesc = []byte{ + 0x0a, 0x21, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x0f, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, + 0x65, 0x2e, 0x76, 0x31, 0x22, 0xb8, 0x01, 0x0a, 0x09, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x79, 0x12, 0x40, 0x0a, 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, + 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x79, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x69, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, + 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x05, + 0x66, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x08, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, + 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, + 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, + 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x08, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x22, + 0x4f, 0x0a, 0x0d, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x4e, 0x6f, 0x64, 0x65, + 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, + 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, + 0x22, 0x6a, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1e, 0x0a, 0x0a, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x22, 0x39, 0x0a, 0x0b, + 0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, + 0x16, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0xb9, 0x01, 0x0a, 0x04, 0x4e, 0x6f, 0x64, 0x65, + 0x12, 0x3e, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, + 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x4e, + 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, + 0x12, 0x2f, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, + 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, + 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x04, 0x66, 0x69, 0x6c, + 0x65, 0x12, 0x38, 0x0a, 0x07, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x4e, 0x6f, 0x64, 0x65, + 0x48, 0x00, 0x52, 0x07, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x42, 0x06, 0x0a, 0x04, 0x6e, + 0x6f, 0x64, 0x65, 0x42, 0x2c, 0x5a, 0x2a, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, + 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, + 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_tvix_castore_protos_castore_proto_rawDescOnce sync.Once + file_tvix_castore_protos_castore_proto_rawDescData = file_tvix_castore_protos_castore_proto_rawDesc +) + +func file_tvix_castore_protos_castore_proto_rawDescGZIP() []byte { + file_tvix_castore_protos_castore_proto_rawDescOnce.Do(func() { + file_tvix_castore_protos_castore_proto_rawDescData = protoimpl.X.CompressGZIP(file_tvix_castore_protos_castore_proto_rawDescData) + }) + return file_tvix_castore_protos_castore_proto_rawDescData +} + +var file_tvix_castore_protos_castore_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_tvix_castore_protos_castore_proto_goTypes = []interface{}{ + (*Directory)(nil), // 0: tvix.castore.v1.Directory + (*DirectoryNode)(nil), // 1: tvix.castore.v1.DirectoryNode + (*FileNode)(nil), // 2: tvix.castore.v1.FileNode + (*SymlinkNode)(nil), // 3: tvix.castore.v1.SymlinkNode + (*Node)(nil), // 4: tvix.castore.v1.Node +} +var file_tvix_castore_protos_castore_proto_depIdxs = []int32{ + 1, // 0: tvix.castore.v1.Directory.directories:type_name -> tvix.castore.v1.DirectoryNode + 2, // 1: tvix.castore.v1.Directory.files:type_name -> tvix.castore.v1.FileNode + 3, // 2: tvix.castore.v1.Directory.symlinks:type_name -> tvix.castore.v1.SymlinkNode + 1, // 3: tvix.castore.v1.Node.directory:type_name -> tvix.castore.v1.DirectoryNode + 2, // 4: tvix.castore.v1.Node.file:type_name -> tvix.castore.v1.FileNode + 3, // 5: tvix.castore.v1.Node.symlink:type_name -> tvix.castore.v1.SymlinkNode + 6, // [6:6] is the sub-list for method output_type + 6, // [6:6] is the sub-list for method input_type + 6, // [6:6] is the sub-list for extension type_name + 6, // [6:6] is the sub-list for extension extendee + 0, // [0:6] is the sub-list for field type_name +} + +func init() { file_tvix_castore_protos_castore_proto_init() } +func file_tvix_castore_protos_castore_proto_init() { + if File_tvix_castore_protos_castore_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_tvix_castore_protos_castore_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Directory); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_tvix_castore_protos_castore_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DirectoryNode); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_tvix_castore_protos_castore_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FileNode); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_tvix_castore_protos_castore_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SymlinkNode); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_tvix_castore_protos_castore_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Node); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_tvix_castore_protos_castore_proto_msgTypes[4].OneofWrappers = []interface{}{ + (*Node_Directory)(nil), + (*Node_File)(nil), + (*Node_Symlink)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_tvix_castore_protos_castore_proto_rawDesc, + NumEnums: 0, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_tvix_castore_protos_castore_proto_goTypes, + DependencyIndexes: file_tvix_castore_protos_castore_proto_depIdxs, + MessageInfos: file_tvix_castore_protos_castore_proto_msgTypes, + }.Build() + File_tvix_castore_protos_castore_proto = out.File + file_tvix_castore_protos_castore_proto_rawDesc = nil + file_tvix_castore_protos_castore_proto_goTypes = nil + file_tvix_castore_protos_castore_proto_depIdxs = nil +} diff --git a/tvix/store/protos/castore.proto b/tvix/castore/protos/castore.proto index 347815107198..d99df43857ac 100644 --- a/tvix/store/protos/castore.proto +++ b/tvix/castore/protos/castore.proto @@ -3,9 +3,9 @@ syntax = "proto3"; -package tvix.store.v1; +package tvix.castore.v1; -option go_package = "code.tvl.fyi/tvix/store/protos;storev1"; +option go_package = "code.tvl.fyi/tvix/castore/protos;castorev1"; // A Directory can contain Directory, File or Symlink nodes. // Each of these nodes have a name attribute, which is the basename in that directory @@ -60,3 +60,13 @@ message SymlinkNode { // The target of the symlink. bytes target = 2; } + +// A Node is either a DirectoryNode, FileNode or SymlinkNode. +message Node { + oneof node { + DirectoryNode directory = 1; + FileNode file = 2; + SymlinkNode symlink = 3; + } +} + diff --git a/tvix/store/protos/castore_test.go b/tvix/castore/protos/castore_test.go index 15a2554bbb57..958d399d76cc 100644 --- a/tvix/store/protos/castore_test.go +++ b/tvix/castore/protos/castore_test.go @@ -1,9 +1,9 @@ -package storev1_test +package castorev1_test import ( "testing" - storev1pb "code.tvl.fyi/tvix/store/protos" + castorev1pb "code.tvl.fyi/tvix/castore/protos" "github.com/stretchr/testify/assert" ) @@ -17,63 +17,63 @@ var ( func TestDirectorySize(t *testing.T) { t.Run("empty", func(t *testing.T) { - d := storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } assert.Equal(t, uint32(0), d.Size()) }) t.Run("containing single empty directory", func(t *testing.T) { - d := storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{{ + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ Name: []byte([]byte("foo")), Digest: dummyDigest, Size: 0, }}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } assert.Equal(t, uint32(1), d.Size()) }) t.Run("containing single non-empty directory", func(t *testing.T) { - d := storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{{ + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ Name: []byte("foo"), Digest: dummyDigest, Size: 4, }}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } assert.Equal(t, uint32(5), d.Size()) }) t.Run("containing single file", func(t *testing.T) { - d := storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{{ + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{{ Name: []byte("foo"), Digest: dummyDigest, Size: 42, Executable: false, }}, - Symlinks: []*storev1pb.SymlinkNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } assert.Equal(t, uint32(1), d.Size()) }) t.Run("containing single symlink", func(t *testing.T) { - d := storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{{ + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{{ Name: []byte("foo"), Target: []byte("bar"), }}, @@ -84,10 +84,10 @@ func TestDirectorySize(t *testing.T) { } func TestDirectoryDigest(t *testing.T) { - d := storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } dgst, err := d.Digest() @@ -101,10 +101,10 @@ func TestDirectoryDigest(t *testing.T) { func TestDirectoryValidate(t *testing.T) { t.Run("empty", func(t *testing.T) { - d := storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } assert.NoError(t, d.Validate()) @@ -112,50 +112,50 @@ func TestDirectoryValidate(t *testing.T) { t.Run("invalid names", func(t *testing.T) { { - d := storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{{ + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ Name: []byte{}, Digest: dummyDigest, Size: 42, }}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } assert.ErrorContains(t, d.Validate(), "invalid name") } { - d := storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{{ + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ Name: []byte("."), Digest: dummyDigest, Size: 42, }}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } assert.ErrorContains(t, d.Validate(), "invalid name") } { - d := storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{{ + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{{ Name: []byte(".."), Digest: dummyDigest, Size: 42, Executable: false, }}, - Symlinks: []*storev1pb.SymlinkNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } assert.ErrorContains(t, d.Validate(), "invalid name") } { - d := storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{{ + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{{ Name: []byte("\x00"), Target: []byte("foo"), }}, @@ -164,10 +164,10 @@ func TestDirectoryValidate(t *testing.T) { assert.ErrorContains(t, d.Validate(), "invalid name") } { - d := storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{{ + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{{ Name: []byte("foo/bar"), Target: []byte("foo"), }}, @@ -178,14 +178,14 @@ func TestDirectoryValidate(t *testing.T) { }) t.Run("invalid digest", func(t *testing.T) { - d := storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{{ + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ Name: []byte("foo"), Digest: nil, Size: 42, }}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } assert.ErrorContains(t, d.Validate(), "invalid digest length") @@ -194,8 +194,8 @@ func TestDirectoryValidate(t *testing.T) { t.Run("sorting", func(t *testing.T) { // "b" comes before "a", bad. { - d := storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{{ + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ Name: []byte("b"), Digest: dummyDigest, Size: 42, @@ -204,35 +204,35 @@ func TestDirectoryValidate(t *testing.T) { Digest: dummyDigest, Size: 42, }}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } assert.ErrorContains(t, d.Validate(), "is not in sorted order") } // "a" exists twice, bad. { - d := storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{{ + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ Name: []byte("a"), Digest: dummyDigest, Size: 42, }}, - Files: []*storev1pb.FileNode{{ + Files: []*castorev1pb.FileNode{{ Name: []byte("a"), Digest: dummyDigest, Size: 42, Executable: false, }}, - Symlinks: []*storev1pb.SymlinkNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } assert.ErrorContains(t, d.Validate(), "duplicate name") } // "a" comes before "b", all good. { - d := storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{{ + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ Name: []byte("a"), Digest: dummyDigest, Size: 42, @@ -241,16 +241,16 @@ func TestDirectoryValidate(t *testing.T) { Digest: dummyDigest, Size: 42, }}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{}, + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{}, } assert.NoError(t, d.Validate(), "shouldn't error") } // [b, c] and [a] are both properly sorted. { - d := storev1pb.Directory{ - Directories: []*storev1pb.DirectoryNode{{ + d := castorev1pb.Directory{ + Directories: []*castorev1pb.DirectoryNode{{ Name: []byte("b"), Digest: dummyDigest, Size: 42, @@ -259,8 +259,8 @@ func TestDirectoryValidate(t *testing.T) { Digest: dummyDigest, Size: 42, }}, - Files: []*storev1pb.FileNode{}, - Symlinks: []*storev1pb.SymlinkNode{{ + Files: []*castorev1pb.FileNode{}, + Symlinks: []*castorev1pb.SymlinkNode{{ Name: []byte("a"), Target: []byte("foo"), }}, diff --git a/tvix/castore/protos/go.mod b/tvix/castore/protos/go.mod new file mode 100644 index 000000000000..35219aba5c1b --- /dev/null +++ b/tvix/castore/protos/go.mod @@ -0,0 +1,19 @@ +module code.tvl.fyi/tvix/castore/protos + +go 1.19 + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/testify v1.8.1 // indirect + golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect + golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect + golang.org/x/text v0.4.0 // indirect + google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect + google.golang.org/grpc v1.51.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + lukechampine.com/blake3 v1.1.7 // indirect +) diff --git a/tvix/castore/protos/go.sum b/tvix/castore/protos/go.sum new file mode 100644 index 000000000000..7a603cdb120d --- /dev/null +++ b/tvix/castore/protos/go.sum @@ -0,0 +1,96 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= +lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= diff --git a/tvix/store/protos/rpc_blobstore.pb.go b/tvix/castore/protos/rpc_blobstore.pb.go index 987d75e3b997..1afc82674451 100644 --- a/tvix/store/protos/rpc_blobstore.pb.go +++ b/tvix/castore/protos/rpc_blobstore.pb.go @@ -5,9 +5,9 @@ // versions: // protoc-gen-go v1.31.0 // protoc (unknown) -// source: tvix/store/protos/rpc_blobstore.proto +// source: tvix/castore/protos/rpc_blobstore.proto -package storev1 +package castorev1 import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" @@ -35,7 +35,7 @@ type StatBlobRequest struct { func (x *StatBlobRequest) Reset() { *x = StatBlobRequest{} if protoimpl.UnsafeEnabled { - mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[0] + mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -48,7 +48,7 @@ func (x *StatBlobRequest) String() string { func (*StatBlobRequest) ProtoMessage() {} func (x *StatBlobRequest) ProtoReflect() protoreflect.Message { - mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[0] + mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -61,7 +61,7 @@ func (x *StatBlobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StatBlobRequest.ProtoReflect.Descriptor instead. func (*StatBlobRequest) Descriptor() ([]byte, []int) { - return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{0} + return file_tvix_castore_protos_rpc_blobstore_proto_rawDescGZIP(), []int{0} } func (x *StatBlobRequest) GetDigest() []byte { @@ -80,7 +80,7 @@ type BlobMeta struct { func (x *BlobMeta) Reset() { *x = BlobMeta{} if protoimpl.UnsafeEnabled { - mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1] + mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -93,7 +93,7 @@ func (x *BlobMeta) String() string { func (*BlobMeta) ProtoMessage() {} func (x *BlobMeta) ProtoReflect() protoreflect.Message { - mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1] + mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -106,7 +106,7 @@ func (x *BlobMeta) ProtoReflect() protoreflect.Message { // Deprecated: Use BlobMeta.ProtoReflect.Descriptor instead. func (*BlobMeta) Descriptor() ([]byte, []int) { - return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{1} + return file_tvix_castore_protos_rpc_blobstore_proto_rawDescGZIP(), []int{1} } type ReadBlobRequest struct { @@ -121,7 +121,7 @@ type ReadBlobRequest struct { func (x *ReadBlobRequest) Reset() { *x = ReadBlobRequest{} if protoimpl.UnsafeEnabled { - mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2] + mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -134,7 +134,7 @@ func (x *ReadBlobRequest) String() string { func (*ReadBlobRequest) ProtoMessage() {} func (x *ReadBlobRequest) ProtoReflect() protoreflect.Message { - mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2] + mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -147,7 +147,7 @@ func (x *ReadBlobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadBlobRequest.ProtoReflect.Descriptor instead. func (*ReadBlobRequest) Descriptor() ([]byte, []int) { - return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{2} + return file_tvix_castore_protos_rpc_blobstore_proto_rawDescGZIP(), []int{2} } func (x *ReadBlobRequest) GetDigest() []byte { @@ -170,7 +170,7 @@ type BlobChunk struct { func (x *BlobChunk) Reset() { *x = BlobChunk{} if protoimpl.UnsafeEnabled { - mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[3] + mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -183,7 +183,7 @@ func (x *BlobChunk) String() string { func (*BlobChunk) ProtoMessage() {} func (x *BlobChunk) ProtoReflect() protoreflect.Message { - mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[3] + mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -196,7 +196,7 @@ func (x *BlobChunk) ProtoReflect() protoreflect.Message { // Deprecated: Use BlobChunk.ProtoReflect.Descriptor instead. func (*BlobChunk) Descriptor() ([]byte, []int) { - return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{3} + return file_tvix_castore_protos_rpc_blobstore_proto_rawDescGZIP(), []int{3} } func (x *BlobChunk) GetData() []byte { @@ -218,7 +218,7 @@ type PutBlobResponse struct { func (x *PutBlobResponse) Reset() { *x = PutBlobResponse{} if protoimpl.UnsafeEnabled { - mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[4] + mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -231,7 +231,7 @@ func (x *PutBlobResponse) String() string { func (*PutBlobResponse) ProtoMessage() {} func (x *PutBlobResponse) ProtoReflect() protoreflect.Message { - mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[4] + mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -244,7 +244,7 @@ func (x *PutBlobResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PutBlobResponse.ProtoReflect.Descriptor instead. func (*PutBlobResponse) Descriptor() ([]byte, []int) { - return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{4} + return file_tvix_castore_protos_rpc_blobstore_proto_rawDescGZIP(), []int{4} } func (x *PutBlobResponse) GetDigest() []byte { @@ -254,69 +254,71 @@ func (x *PutBlobResponse) GetDigest() []byte { return nil } -var File_tvix_store_protos_rpc_blobstore_proto protoreflect.FileDescriptor - -var file_tvix_store_protos_rpc_blobstore_proto_rawDesc = []byte{ - 0x0a, 0x25, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x5f, 0x62, 0x6c, 0x6f, 0x62, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x22, 0x29, 0x0a, 0x0f, 0x53, 0x74, 0x61, 0x74, 0x42, 0x6c, - 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, - 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, - 0x74, 0x22, 0x0a, 0x0a, 0x08, 0x42, 0x6c, 0x6f, 0x62, 0x4d, 0x65, 0x74, 0x61, 0x22, 0x29, 0x0a, - 0x0f, 0x52, 0x65, 0x61, 0x64, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, +var File_tvix_castore_protos_rpc_blobstore_proto protoreflect.FileDescriptor + +var file_tvix_castore_protos_rpc_blobstore_proto_rawDesc = []byte{ + 0x0a, 0x27, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x5f, 0x62, 0x6c, 0x6f, 0x62, 0x73, 0x74, + 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0f, 0x74, 0x76, 0x69, 0x78, 0x2e, + 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x22, 0x29, 0x0a, 0x0f, 0x53, 0x74, + 0x61, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, + 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, + 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0x0a, 0x0a, 0x08, 0x42, 0x6c, 0x6f, 0x62, 0x4d, 0x65, 0x74, + 0x61, 0x22, 0x29, 0x0a, 0x0f, 0x52, 0x65, 0x61, 0x64, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0x1f, 0x0a, 0x09, + 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, + 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x29, 0x0a, + 0x0f, 0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0x1f, 0x0a, 0x09, 0x42, 0x6c, 0x6f, 0x62, - 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x29, 0x0a, 0x0f, 0x50, 0x75, 0x74, - 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, - 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, - 0x67, 0x65, 0x73, 0x74, 0x32, 0xd5, 0x01, 0x0a, 0x0b, 0x42, 0x6c, 0x6f, 0x62, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x53, 0x74, 0x61, 0x74, 0x12, 0x1e, 0x2e, 0x74, - 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, - 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x74, - 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, - 0x62, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x42, 0x0a, 0x04, 0x52, 0x65, 0x61, 0x64, 0x12, 0x1e, 0x2e, - 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, - 0x61, 0x64, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, - 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, - 0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x03, 0x50, 0x75, 0x74, - 0x12, 0x18, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x1a, 0x1e, 0x2e, 0x74, 0x76, 0x69, - 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x74, 0x42, 0x6c, - 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x42, 0x28, 0x5a, 0x26, + 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x32, 0xe1, 0x01, 0x0a, 0x0b, 0x42, 0x6c, 0x6f, + 0x62, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x04, 0x53, 0x74, 0x61, 0x74, + 0x12, 0x20, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, + 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x46, 0x0a, + 0x04, 0x52, 0x65, 0x61, 0x64, 0x12, 0x20, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, + 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x42, 0x6c, 0x6f, 0x62, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, + 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x68, + 0x75, 0x6e, 0x6b, 0x30, 0x01, 0x12, 0x45, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x1a, 0x2e, 0x74, + 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, + 0x6c, 0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x1a, 0x20, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, + 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x74, 0x42, 0x6c, + 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x42, 0x2c, 0x5a, 0x2a, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, - 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x73, - 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, + 0x3b, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( - file_tvix_store_protos_rpc_blobstore_proto_rawDescOnce sync.Once - file_tvix_store_protos_rpc_blobstore_proto_rawDescData = file_tvix_store_protos_rpc_blobstore_proto_rawDesc + file_tvix_castore_protos_rpc_blobstore_proto_rawDescOnce sync.Once + file_tvix_castore_protos_rpc_blobstore_proto_rawDescData = file_tvix_castore_protos_rpc_blobstore_proto_rawDesc ) -func file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP() []byte { - file_tvix_store_protos_rpc_blobstore_proto_rawDescOnce.Do(func() { - file_tvix_store_protos_rpc_blobstore_proto_rawDescData = protoimpl.X.CompressGZIP(file_tvix_store_protos_rpc_blobstore_proto_rawDescData) +func file_tvix_castore_protos_rpc_blobstore_proto_rawDescGZIP() []byte { + file_tvix_castore_protos_rpc_blobstore_proto_rawDescOnce.Do(func() { + file_tvix_castore_protos_rpc_blobstore_proto_rawDescData = protoimpl.X.CompressGZIP(file_tvix_castore_protos_rpc_blobstore_proto_rawDescData) }) - return file_tvix_store_protos_rpc_blobstore_proto_rawDescData -} - -var file_tvix_store_protos_rpc_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 5) -var file_tvix_store_protos_rpc_blobstore_proto_goTypes = []interface{}{ - (*StatBlobRequest)(nil), // 0: tvix.store.v1.StatBlobRequest - (*BlobMeta)(nil), // 1: tvix.store.v1.BlobMeta - (*ReadBlobRequest)(nil), // 2: tvix.store.v1.ReadBlobRequest - (*BlobChunk)(nil), // 3: tvix.store.v1.BlobChunk - (*PutBlobResponse)(nil), // 4: tvix.store.v1.PutBlobResponse -} -var file_tvix_store_protos_rpc_blobstore_proto_depIdxs = []int32{ - 0, // 0: tvix.store.v1.BlobService.Stat:input_type -> tvix.store.v1.StatBlobRequest - 2, // 1: tvix.store.v1.BlobService.Read:input_type -> tvix.store.v1.ReadBlobRequest - 3, // 2: tvix.store.v1.BlobService.Put:input_type -> tvix.store.v1.BlobChunk - 1, // 3: tvix.store.v1.BlobService.Stat:output_type -> tvix.store.v1.BlobMeta - 3, // 4: tvix.store.v1.BlobService.Read:output_type -> tvix.store.v1.BlobChunk - 4, // 5: tvix.store.v1.BlobService.Put:output_type -> tvix.store.v1.PutBlobResponse + return file_tvix_castore_protos_rpc_blobstore_proto_rawDescData +} + +var file_tvix_castore_protos_rpc_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_tvix_castore_protos_rpc_blobstore_proto_goTypes = []interface{}{ + (*StatBlobRequest)(nil), // 0: tvix.castore.v1.StatBlobRequest + (*BlobMeta)(nil), // 1: tvix.castore.v1.BlobMeta + (*ReadBlobRequest)(nil), // 2: tvix.castore.v1.ReadBlobRequest + (*BlobChunk)(nil), // 3: tvix.castore.v1.BlobChunk + (*PutBlobResponse)(nil), // 4: tvix.castore.v1.PutBlobResponse +} +var file_tvix_castore_protos_rpc_blobstore_proto_depIdxs = []int32{ + 0, // 0: tvix.castore.v1.BlobService.Stat:input_type -> tvix.castore.v1.StatBlobRequest + 2, // 1: tvix.castore.v1.BlobService.Read:input_type -> tvix.castore.v1.ReadBlobRequest + 3, // 2: tvix.castore.v1.BlobService.Put:input_type -> tvix.castore.v1.BlobChunk + 1, // 3: tvix.castore.v1.BlobService.Stat:output_type -> tvix.castore.v1.BlobMeta + 3, // 4: tvix.castore.v1.BlobService.Read:output_type -> tvix.castore.v1.BlobChunk + 4, // 5: tvix.castore.v1.BlobService.Put:output_type -> tvix.castore.v1.PutBlobResponse 3, // [3:6] is the sub-list for method output_type 0, // [0:3] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name @@ -324,13 +326,13 @@ var file_tvix_store_protos_rpc_blobstore_proto_depIdxs = []int32{ 0, // [0:0] is the sub-list for field type_name } -func init() { file_tvix_store_protos_rpc_blobstore_proto_init() } -func file_tvix_store_protos_rpc_blobstore_proto_init() { - if File_tvix_store_protos_rpc_blobstore_proto != nil { +func init() { file_tvix_castore_protos_rpc_blobstore_proto_init() } +func file_tvix_castore_protos_rpc_blobstore_proto_init() { + if File_tvix_castore_protos_rpc_blobstore_proto != nil { return } if !protoimpl.UnsafeEnabled { - file_tvix_store_protos_rpc_blobstore_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*StatBlobRequest); i { case 0: return &v.state @@ -342,7 +344,7 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() { return nil } } - file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*BlobMeta); i { case 0: return &v.state @@ -354,7 +356,7 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() { return nil } } - file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ReadBlobRequest); i { case 0: return &v.state @@ -366,7 +368,7 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() { return nil } } - file_tvix_store_protos_rpc_blobstore_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*BlobChunk); i { case 0: return &v.state @@ -378,7 +380,7 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() { return nil } } - file_tvix_store_protos_rpc_blobstore_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*PutBlobResponse); i { case 0: return &v.state @@ -395,18 +397,18 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() { out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_tvix_store_protos_rpc_blobstore_proto_rawDesc, + RawDescriptor: file_tvix_castore_protos_rpc_blobstore_proto_rawDesc, NumEnums: 0, NumMessages: 5, NumExtensions: 0, NumServices: 1, }, - GoTypes: file_tvix_store_protos_rpc_blobstore_proto_goTypes, - DependencyIndexes: file_tvix_store_protos_rpc_blobstore_proto_depIdxs, - MessageInfos: file_tvix_store_protos_rpc_blobstore_proto_msgTypes, + GoTypes: file_tvix_castore_protos_rpc_blobstore_proto_goTypes, + DependencyIndexes: file_tvix_castore_protos_rpc_blobstore_proto_depIdxs, + MessageInfos: file_tvix_castore_protos_rpc_blobstore_proto_msgTypes, }.Build() - File_tvix_store_protos_rpc_blobstore_proto = out.File - file_tvix_store_protos_rpc_blobstore_proto_rawDesc = nil - file_tvix_store_protos_rpc_blobstore_proto_goTypes = nil - file_tvix_store_protos_rpc_blobstore_proto_depIdxs = nil + File_tvix_castore_protos_rpc_blobstore_proto = out.File + file_tvix_castore_protos_rpc_blobstore_proto_rawDesc = nil + file_tvix_castore_protos_rpc_blobstore_proto_goTypes = nil + file_tvix_castore_protos_rpc_blobstore_proto_depIdxs = nil } diff --git a/tvix/store/protos/rpc_blobstore.proto b/tvix/castore/protos/rpc_blobstore.proto index 2ca3df60a81b..6ee9a80f0afc 100644 --- a/tvix/store/protos/rpc_blobstore.proto +++ b/tvix/castore/protos/rpc_blobstore.proto @@ -2,9 +2,9 @@ // Copyright © 2022 The Tvix Authors syntax = "proto3"; -package tvix.store.v1; +package tvix.castore.v1; -option go_package = "code.tvl.fyi/tvix/store/protos;storev1"; +option go_package = "code.tvl.fyi/tvix/castore/protos;castorev1"; service BlobService { // In the future, Stat will expose more metadata about a given blob, diff --git a/tvix/store/protos/rpc_blobstore_grpc.pb.go b/tvix/castore/protos/rpc_blobstore_grpc.pb.go index 531ebcd2c637..0876bcc4e95a 100644 --- a/tvix/store/protos/rpc_blobstore_grpc.pb.go +++ b/tvix/castore/protos/rpc_blobstore_grpc.pb.go @@ -5,9 +5,9 @@ // versions: // - protoc-gen-go-grpc v1.3.0 // - protoc (unknown) -// source: tvix/store/protos/rpc_blobstore.proto +// source: tvix/castore/protos/rpc_blobstore.proto -package storev1 +package castorev1 import ( context "context" @@ -22,9 +22,9 @@ import ( const _ = grpc.SupportPackageIsVersion7 const ( - BlobService_Stat_FullMethodName = "/tvix.store.v1.BlobService/Stat" - BlobService_Read_FullMethodName = "/tvix.store.v1.BlobService/Read" - BlobService_Put_FullMethodName = "/tvix.store.v1.BlobService/Put" + BlobService_Stat_FullMethodName = "/tvix.castore.v1.BlobService/Stat" + BlobService_Read_FullMethodName = "/tvix.castore.v1.BlobService/Read" + BlobService_Put_FullMethodName = "/tvix.castore.v1.BlobService/Put" ) // BlobServiceClient is the client API for BlobService service. @@ -250,7 +250,7 @@ func (x *blobServicePutServer) Recv() (*BlobChunk, error) { // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var BlobService_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "tvix.store.v1.BlobService", + ServiceName: "tvix.castore.v1.BlobService", HandlerType: (*BlobServiceServer)(nil), Methods: []grpc.MethodDesc{ { @@ -270,5 +270,5 @@ var BlobService_ServiceDesc = grpc.ServiceDesc{ ClientStreams: true, }, }, - Metadata: "tvix/store/protos/rpc_blobstore.proto", + Metadata: "tvix/castore/protos/rpc_blobstore.proto", } diff --git a/tvix/castore/protos/rpc_directory.pb.go b/tvix/castore/protos/rpc_directory.pb.go new file mode 100644 index 000000000000..f658c6b60cc0 --- /dev/null +++ b/tvix/castore/protos/rpc_directory.pb.go @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: MIT +// Copyright © 2022 The Tvix Authors + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc (unknown) +// source: tvix/castore/protos/rpc_directory.proto + +package castorev1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type GetDirectoryRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to ByWhat: + // + // *GetDirectoryRequest_Digest + ByWhat isGetDirectoryRequest_ByWhat `protobuf_oneof:"by_what"` + // If set to true, recursively resolve all child Directory messages. + // Directory messages SHOULD be streamed in a recursive breadth-first walk, + // but other orders are also fine, as long as Directory messages are only + // sent after they are referred to from previously sent Directory messages. + Recursive bool `protobuf:"varint,2,opt,name=recursive,proto3" json:"recursive,omitempty"` +} + +func (x *GetDirectoryRequest) Reset() { + *x = GetDirectoryRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_tvix_castore_protos_rpc_directory_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetDirectoryRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetDirectoryRequest) ProtoMessage() {} + +func (x *GetDirectoryRequest) ProtoReflect() protoreflect.Message { + mi := &file_tvix_castore_protos_rpc_directory_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetDirectoryRequest.ProtoReflect.Descriptor instead. +func (*GetDirectoryRequest) Descriptor() ([]byte, []int) { + return file_tvix_castore_protos_rpc_directory_proto_rawDescGZIP(), []int{0} +} + +func (m *GetDirectoryRequest) GetByWhat() isGetDirectoryRequest_ByWhat { + if m != nil { + return m.ByWhat + } + return nil +} + +func (x *GetDirectoryRequest) GetDigest() []byte { + if x, ok := x.GetByWhat().(*GetDirectoryRequest_Digest); ok { + return x.Digest + } + return nil +} + +func (x *GetDirectoryRequest) GetRecursive() bool { + if x != nil { + return x.Recursive + } + return false +} + +type isGetDirectoryRequest_ByWhat interface { + isGetDirectoryRequest_ByWhat() +} + +type GetDirectoryRequest_Digest struct { + // The blake3 hash of the (root) Directory message, serialized in + // protobuf canonical form. + // Keep in mind this can be a subtree of another root. + Digest []byte `protobuf:"bytes,1,opt,name=digest,proto3,oneof"` +} + +func (*GetDirectoryRequest_Digest) isGetDirectoryRequest_ByWhat() {} + +type PutDirectoryResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + RootDigest []byte `protobuf:"bytes,1,opt,name=root_digest,json=rootDigest,proto3" json:"root_digest,omitempty"` +} + +func (x *PutDirectoryResponse) Reset() { + *x = PutDirectoryResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_tvix_castore_protos_rpc_directory_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PutDirectoryResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PutDirectoryResponse) ProtoMessage() {} + +func (x *PutDirectoryResponse) ProtoReflect() protoreflect.Message { + mi := &file_tvix_castore_protos_rpc_directory_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PutDirectoryResponse.ProtoReflect.Descriptor instead. +func (*PutDirectoryResponse) Descriptor() ([]byte, []int) { + return file_tvix_castore_protos_rpc_directory_proto_rawDescGZIP(), []int{1} +} + +func (x *PutDirectoryResponse) GetRootDigest() []byte { + if x != nil { + return x.RootDigest + } + return nil +} + +var File_tvix_castore_protos_rpc_directory_proto protoreflect.FileDescriptor + +var file_tvix_castore_protos_rpc_directory_proto_rawDesc = []byte{ + 0x0a, 0x27, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0f, 0x74, 0x76, 0x69, 0x78, 0x2e, + 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x21, 0x74, 0x76, 0x69, 0x78, + 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, + 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x58, 0x0a, + 0x13, 0x47, 0x65, 0x74, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x1c, + 0x0a, 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x42, 0x09, 0x0a, 0x07, + 0x62, 0x79, 0x5f, 0x77, 0x68, 0x61, 0x74, 0x22, 0x37, 0x0a, 0x14, 0x50, 0x75, 0x74, 0x44, 0x69, + 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x1f, 0x0a, 0x0b, 0x72, 0x6f, 0x6f, 0x74, 0x5f, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x72, 0x6f, 0x6f, 0x74, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, + 0x32, 0xa9, 0x01, 0x0a, 0x10, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x49, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x24, 0x2e, 0x74, + 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, + 0x65, 0x74, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x30, 0x01, + 0x12, 0x4a, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x1a, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, + 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x79, 0x1a, 0x25, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, + 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x74, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x42, 0x2c, 0x5a, 0x2a, + 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, + 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, + 0x3b, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, +} + +var ( + file_tvix_castore_protos_rpc_directory_proto_rawDescOnce sync.Once + file_tvix_castore_protos_rpc_directory_proto_rawDescData = file_tvix_castore_protos_rpc_directory_proto_rawDesc +) + +func file_tvix_castore_protos_rpc_directory_proto_rawDescGZIP() []byte { + file_tvix_castore_protos_rpc_directory_proto_rawDescOnce.Do(func() { + file_tvix_castore_protos_rpc_directory_proto_rawDescData = protoimpl.X.CompressGZIP(file_tvix_castore_protos_rpc_directory_proto_rawDescData) + }) + return file_tvix_castore_protos_rpc_directory_proto_rawDescData +} + +var file_tvix_castore_protos_rpc_directory_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_tvix_castore_protos_rpc_directory_proto_goTypes = []interface{}{ + (*GetDirectoryRequest)(nil), // 0: tvix.castore.v1.GetDirectoryRequest + (*PutDirectoryResponse)(nil), // 1: tvix.castore.v1.PutDirectoryResponse + (*Directory)(nil), // 2: tvix.castore.v1.Directory +} +var file_tvix_castore_protos_rpc_directory_proto_depIdxs = []int32{ + 0, // 0: tvix.castore.v1.DirectoryService.Get:input_type -> tvix.castore.v1.GetDirectoryRequest + 2, // 1: tvix.castore.v1.DirectoryService.Put:input_type -> tvix.castore.v1.Directory + 2, // 2: tvix.castore.v1.DirectoryService.Get:output_type -> tvix.castore.v1.Directory + 1, // 3: tvix.castore.v1.DirectoryService.Put:output_type -> tvix.castore.v1.PutDirectoryResponse + 2, // [2:4] is the sub-list for method output_type + 0, // [0:2] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_tvix_castore_protos_rpc_directory_proto_init() } +func file_tvix_castore_protos_rpc_directory_proto_init() { + if File_tvix_castore_protos_rpc_directory_proto != nil { + return + } + file_tvix_castore_protos_castore_proto_init() + if !protoimpl.UnsafeEnabled { + file_tvix_castore_protos_rpc_directory_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetDirectoryRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_tvix_castore_protos_rpc_directory_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutDirectoryResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_tvix_castore_protos_rpc_directory_proto_msgTypes[0].OneofWrappers = []interface{}{ + (*GetDirectoryRequest_Digest)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_tvix_castore_protos_rpc_directory_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_tvix_castore_protos_rpc_directory_proto_goTypes, + DependencyIndexes: file_tvix_castore_protos_rpc_directory_proto_depIdxs, + MessageInfos: file_tvix_castore_protos_rpc_directory_proto_msgTypes, + }.Build() + File_tvix_castore_protos_rpc_directory_proto = out.File + file_tvix_castore_protos_rpc_directory_proto_rawDesc = nil + file_tvix_castore_protos_rpc_directory_proto_goTypes = nil + file_tvix_castore_protos_rpc_directory_proto_depIdxs = nil +} diff --git a/tvix/store/protos/rpc_directory.proto b/tvix/castore/protos/rpc_directory.proto index 0aeed5c3c0e1..8d4c22828547 100644 --- a/tvix/store/protos/rpc_directory.proto +++ b/tvix/castore/protos/rpc_directory.proto @@ -2,11 +2,11 @@ // Copyright © 2022 The Tvix Authors syntax = "proto3"; -package tvix.store.v1; +package tvix.castore.v1; -import "tvix/store/protos/castore.proto"; +import "tvix/castore/protos/castore.proto"; -option go_package = "code.tvl.fyi/tvix/store/protos;storev1"; +option go_package = "code.tvl.fyi/tvix/castore/protos;castorev1"; service DirectoryService { // Get retrieves a stream of Directory messages, by using the lookup diff --git a/tvix/store/protos/rpc_directory_grpc.pb.go b/tvix/castore/protos/rpc_directory_grpc.pb.go index a578dbd89d39..f19e457d867b 100644 --- a/tvix/store/protos/rpc_directory_grpc.pb.go +++ b/tvix/castore/protos/rpc_directory_grpc.pb.go @@ -5,9 +5,9 @@ // versions: // - protoc-gen-go-grpc v1.3.0 // - protoc (unknown) -// source: tvix/store/protos/rpc_directory.proto +// source: tvix/castore/protos/rpc_directory.proto -package storev1 +package castorev1 import ( context "context" @@ -22,8 +22,8 @@ import ( const _ = grpc.SupportPackageIsVersion7 const ( - DirectoryService_Get_FullMethodName = "/tvix.store.v1.DirectoryService/Get" - DirectoryService_Put_FullMethodName = "/tvix.store.v1.DirectoryService/Put" + DirectoryService_Get_FullMethodName = "/tvix.castore.v1.DirectoryService/Get" + DirectoryService_Put_FullMethodName = "/tvix.castore.v1.DirectoryService/Put" ) // DirectoryServiceClient is the client API for DirectoryService service. @@ -219,7 +219,7 @@ func (x *directoryServicePutServer) Recv() (*Directory, error) { // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var DirectoryService_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "tvix.store.v1.DirectoryService", + ServiceName: "tvix.castore.v1.DirectoryService", HandlerType: (*DirectoryServiceServer)(nil), Methods: []grpc.MethodDesc{}, Streams: []grpc.StreamDesc{ @@ -234,5 +234,5 @@ var DirectoryService_ServiceDesc = grpc.ServiceDesc{ ClientStreams: true, }, }, - Metadata: "tvix/store/protos/rpc_directory.proto", + Metadata: "tvix/castore/protos/rpc_directory.proto", } diff --git a/tvix/store/src/blobservice/from_addr.rs b/tvix/castore/src/blobservice/from_addr.rs index 2e0a30697d75..2e0a30697d75 100644 --- a/tvix/store/src/blobservice/from_addr.rs +++ b/tvix/castore/src/blobservice/from_addr.rs diff --git a/tvix/store/src/blobservice/grpc.rs b/tvix/castore/src/blobservice/grpc.rs index ae84f4ce0ff7..db9d4a9c00f0 100644 --- a/tvix/store/src/blobservice/grpc.rs +++ b/tvix/castore/src/blobservice/grpc.rs @@ -289,8 +289,8 @@ mod tests { use tokio_stream::wrappers::UnixListenerStream; use crate::blobservice::MemoryBlobService; + use crate::fixtures; use crate::proto::GRPCBlobServiceWrapper; - use crate::tests::fixtures; use super::BlobService; use super::GRPCBlobService; diff --git a/tvix/store/src/blobservice/memory.rs b/tvix/castore/src/blobservice/memory.rs index 383127344a17..383127344a17 100644 --- a/tvix/store/src/blobservice/memory.rs +++ b/tvix/castore/src/blobservice/memory.rs diff --git a/tvix/store/src/blobservice/mod.rs b/tvix/castore/src/blobservice/mod.rs index 5ecf25ac1337..5ecf25ac1337 100644 --- a/tvix/store/src/blobservice/mod.rs +++ b/tvix/castore/src/blobservice/mod.rs diff --git a/tvix/store/src/blobservice/naive_seeker.rs b/tvix/castore/src/blobservice/naive_seeker.rs index e65a82c7f45a..e65a82c7f45a 100644 --- a/tvix/store/src/blobservice/naive_seeker.rs +++ b/tvix/castore/src/blobservice/naive_seeker.rs diff --git a/tvix/store/src/blobservice/sled.rs b/tvix/castore/src/blobservice/sled.rs index 209f0b76fc7a..209f0b76fc7a 100644 --- a/tvix/store/src/blobservice/sled.rs +++ b/tvix/castore/src/blobservice/sled.rs diff --git a/tvix/store/src/blobservice/tests.rs b/tvix/castore/src/blobservice/tests.rs index 501270780cf4..fe390b537eb8 100644 --- a/tvix/store/src/blobservice/tests.rs +++ b/tvix/castore/src/blobservice/tests.rs @@ -9,7 +9,7 @@ use super::B3Digest; use super::BlobService; use super::MemoryBlobService; use super::SledBlobService; -use crate::tests::fixtures; +use crate::fixtures; // TODO: avoid having to define all different services we test against for all functions. // maybe something like rstest can be used? diff --git a/tvix/store/src/digests.rs b/tvix/castore/src/digests.rs index 4df11b389e93..4df11b389e93 100644 --- a/tvix/store/src/digests.rs +++ b/tvix/castore/src/digests.rs diff --git a/tvix/store/src/directoryservice/from_addr.rs b/tvix/castore/src/directoryservice/from_addr.rs index 776cf061096c..776cf061096c 100644 --- a/tvix/store/src/directoryservice/from_addr.rs +++ b/tvix/castore/src/directoryservice/from_addr.rs diff --git a/tvix/store/src/directoryservice/grpc.rs b/tvix/castore/src/directoryservice/grpc.rs index 6257a8e81485..0f0305341265 100644 --- a/tvix/store/src/directoryservice/grpc.rs +++ b/tvix/castore/src/directoryservice/grpc.rs @@ -348,12 +348,10 @@ mod tests { use crate::{ directoryservice::DirectoryService, + fixtures::{DIRECTORY_A, DIRECTORY_B}, proto, proto::{directory_service_server::DirectoryServiceServer, GRPCDirectoryServiceWrapper}, - tests::{ - fixtures::{DIRECTORY_A, DIRECTORY_B}, - utils::gen_directory_service, - }, + utils::gen_directory_service, }; #[test] diff --git a/tvix/store/src/directoryservice/memory.rs b/tvix/castore/src/directoryservice/memory.rs index ac67c999d01b..ac67c999d01b 100644 --- a/tvix/store/src/directoryservice/memory.rs +++ b/tvix/castore/src/directoryservice/memory.rs diff --git a/tvix/store/src/directoryservice/mod.rs b/tvix/castore/src/directoryservice/mod.rs index 3b26f4baf79b..3b26f4baf79b 100644 --- a/tvix/store/src/directoryservice/mod.rs +++ b/tvix/castore/src/directoryservice/mod.rs diff --git a/tvix/store/src/directoryservice/sled.rs b/tvix/castore/src/directoryservice/sled.rs index 0dc5496803cb..0dc5496803cb 100644 --- a/tvix/store/src/directoryservice/sled.rs +++ b/tvix/castore/src/directoryservice/sled.rs diff --git a/tvix/store/src/directoryservice/traverse.rs b/tvix/castore/src/directoryservice/traverse.rs index 5043439e9de5..4f011c963c06 100644 --- a/tvix/store/src/directoryservice/traverse.rs +++ b/tvix/castore/src/directoryservice/traverse.rs @@ -84,10 +84,8 @@ pub async fn descend_to( mod tests { use std::path::PathBuf; - use crate::tests::{ - fixtures::{DIRECTORY_COMPLICATED, DIRECTORY_WITH_KEEP}, - utils::gen_directory_service, - }; + use crate::fixtures::{DIRECTORY_COMPLICATED, DIRECTORY_WITH_KEEP}; + use crate::utils::gen_directory_service; use super::descend_to; diff --git a/tvix/store/src/directoryservice/utils.rs b/tvix/castore/src/directoryservice/utils.rs index 4c5e7cfde37c..4c5e7cfde37c 100644 --- a/tvix/store/src/directoryservice/utils.rs +++ b/tvix/castore/src/directoryservice/utils.rs diff --git a/tvix/store/src/errors.rs b/tvix/castore/src/errors.rs index 3b23f972b045..3b23f972b045 100644 --- a/tvix/store/src/errors.rs +++ b/tvix/castore/src/errors.rs diff --git a/tvix/castore/src/fixtures.rs b/tvix/castore/src/fixtures.rs new file mode 100644 index 000000000000..ed3d1ca6e855 --- /dev/null +++ b/tvix/castore/src/fixtures.rs @@ -0,0 +1,95 @@ +use crate::{ + proto::{self, Directory, DirectoryNode, FileNode, SymlinkNode}, + B3Digest, +}; +use lazy_static::lazy_static; + +pub const HELLOWORLD_BLOB_CONTENTS: &[u8] = b"Hello World!"; +pub const EMPTY_BLOB_CONTENTS: &[u8] = b""; + +lazy_static! { + pub static ref DUMMY_DIGEST: B3Digest = { + let u: &[u8; 32] = &[ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, + ]; + u.into() + }; + pub static ref DUMMY_DIGEST_2: B3Digest = { + let u: &[u8; 32] = &[ + 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ]; + u.into() + }; + pub static ref DUMMY_DATA_1: bytes::Bytes = vec![0x01, 0x02, 0x03].into(); + pub static ref DUMMY_DATA_2: bytes::Bytes = vec![0x04, 0x05].into(); + + pub static ref HELLOWORLD_BLOB_DIGEST: B3Digest = + blake3::hash(HELLOWORLD_BLOB_CONTENTS).as_bytes().into(); + pub static ref EMPTY_BLOB_DIGEST: B3Digest = + blake3::hash(EMPTY_BLOB_CONTENTS).as_bytes().into(); + + // 2 bytes + pub static ref BLOB_A: bytes::Bytes = vec![0x00, 0x01].into(); + pub static ref BLOB_A_DIGEST: B3Digest = blake3::hash(&BLOB_A).as_bytes().into(); + + // 1MB + pub static ref BLOB_B: bytes::Bytes = (0..255).collect::<Vec<u8>>().repeat(4 * 1024).into(); + pub static ref BLOB_B_DIGEST: B3Digest = blake3::hash(&BLOB_B).as_bytes().into(); + + // Directories + pub static ref DIRECTORY_WITH_KEEP: proto::Directory = proto::Directory { + directories: vec![], + files: vec![FileNode { + name: b".keep".to_vec().into(), + digest: EMPTY_BLOB_DIGEST.clone().into(), + size: 0, + executable: false, + }], + symlinks: vec![], + }; + pub static ref DIRECTORY_COMPLICATED: proto::Directory = proto::Directory { + directories: vec![DirectoryNode { + name: b"keep".to_vec().into(), + digest: DIRECTORY_WITH_KEEP.digest().into(), + size: DIRECTORY_WITH_KEEP.size(), + }], + files: vec![FileNode { + name: b".keep".to_vec().into(), + digest: EMPTY_BLOB_DIGEST.clone().into(), + size: 0, + executable: false, + }], + symlinks: vec![SymlinkNode { + name: b"aa".to_vec().into(), + target: b"/nix/store/somewhereelse".to_vec().into(), + }], + }; + pub static ref DIRECTORY_A: Directory = Directory::default(); + pub static ref DIRECTORY_B: Directory = Directory { + directories: vec![DirectoryNode { + name: b"a".to_vec().into(), + digest: DIRECTORY_A.digest().into(), + size: DIRECTORY_A.size(), + }], + ..Default::default() + }; + pub static ref DIRECTORY_C: Directory = Directory { + directories: vec![ + DirectoryNode { + name: b"a".to_vec().into(), + digest: DIRECTORY_A.digest().into(), + size: DIRECTORY_A.size(), + }, + DirectoryNode { + name: b"a'".to_vec().into(), + digest: DIRECTORY_A.digest().into(), + size: DIRECTORY_A.size(), + } + ], + ..Default::default() + }; +} diff --git a/tvix/store/src/import.rs b/tvix/castore/src/import.rs index 6eebe500d275..92f792bfe704 100644 --- a/tvix/store/src/import.rs +++ b/tvix/castore/src/import.rs @@ -1,6 +1,12 @@ use crate::blobservice::BlobService; +use crate::directoryservice::DirectoryPutter; use crate::directoryservice::DirectoryService; -use crate::{directoryservice::DirectoryPutter, proto}; +use crate::proto::node::Node; +use crate::proto::Directory; +use crate::proto::DirectoryNode; +use crate::proto::FileNode; +use crate::proto::SymlinkNode; +use crate::Error as CastoreError; use std::os::unix::ffi::OsStrExt; use std::sync::Arc; use std::{ @@ -15,7 +21,7 @@ use walkdir::WalkDir; #[derive(Debug, thiserror::Error)] pub enum Error { #[error("failed to upload directory at {0}: {1}")] - UploadDirectoryError(PathBuf, crate::Error), + UploadDirectoryError(PathBuf, CastoreError), #[error("invalid encoding encountered for entry {0:?}")] InvalidEncoding(PathBuf), @@ -30,11 +36,11 @@ pub enum Error { UnableToRead(PathBuf, std::io::Error), } -impl From<super::Error> for Error { - fn from(value: super::Error) -> Self { +impl From<CastoreError> for Error { + fn from(value: CastoreError) -> Self { match value { - crate::Error::InvalidRequest(_) => panic!("tvix bug"), - crate::Error::StorageError(_) => panic!("error"), + CastoreError::InvalidRequest(_) => panic!("tvix bug"), + CastoreError::StorageError(_) => panic!("error"), } } } @@ -59,8 +65,8 @@ async fn process_entry( blob_service: Arc<dyn BlobService>, directory_putter: &mut Box<dyn DirectoryPutter>, entry: &walkdir::DirEntry, - maybe_directory: Option<proto::Directory>, -) -> Result<proto::node::Node, Error> { + maybe_directory: Option<Directory>, +) -> Result<Node, Error> { let file_type = entry.file_type(); if file_type.is_dir() { @@ -75,7 +81,7 @@ async fn process_entry( .await .map_err(|e| Error::UploadDirectoryError(entry.path().to_path_buf(), e))?; - return Ok(proto::node::Node::Directory(proto::DirectoryNode { + return Ok(Node::Directory(DirectoryNode { name: entry.file_name().as_bytes().to_owned().into(), digest: directory_digest.into(), size: directory_size, @@ -90,7 +96,7 @@ async fn process_entry( .to_owned() .into(); - return Ok(proto::node::Node::Symlink(proto::SymlinkNode { + return Ok(Node::Symlink(SymlinkNode { name: entry.file_name().as_bytes().to_owned().into(), target, })); @@ -113,7 +119,7 @@ async fn process_entry( let digest = writer.close().await?; - return Ok(proto::node::Node::File(proto::FileNode { + return Ok(Node::File(FileNode { name: entry.file_name().as_bytes().to_vec().into(), digest: digest.into(), size: metadata.len() as u32, @@ -132,17 +138,17 @@ async fn process_entry( /// It does not follow symlinks at the root, they will be ingested as actual /// symlinks. /// -/// It's not interacting with a -/// [PathInfoService](crate::pathinfoservice::PathInfoService), it's up to the -/// caller to possibly register it somewhere (and potentially rename it based on -/// some naming scheme. +/// It's not interacting with a PathInfoService (from tvix-store), or anything +/// else giving it a "non-content-addressed name". +/// It's up to the caller to possibly register it somewhere (and potentially +/// rename it based on some naming scheme) #[instrument(skip(blob_service, directory_service), fields(path=?p))] pub async fn ingest_path<P: AsRef<Path> + Debug>( blob_service: Arc<dyn BlobService>, directory_service: Arc<dyn DirectoryService>, p: P, -) -> Result<proto::node::Node, Error> { - let mut directories: HashMap<PathBuf, proto::Directory> = HashMap::default(); +) -> Result<Node, Error> { + let mut directories: HashMap<PathBuf, Directory> = HashMap::default(); // TODO: pass this one instead? let mut directory_putter = directory_service.put_multiple_start(); @@ -157,7 +163,7 @@ pub async fn ingest_path<P: AsRef<Path> + Debug>( // process_entry wants an Option<Directory> in case the entry points to a directory. // make sure to provide it. - let maybe_directory: Option<proto::Directory> = { + let maybe_directory: Option<Directory> = { if entry.file_type().is_dir() { Some( directories @@ -188,9 +194,9 @@ pub async fn ingest_path<P: AsRef<Path> + Debug>( // record node in parent directory, creating a new [proto:Directory] if not there yet. let parent_directory = directories.entry(parent_path).or_default(); match node { - proto::node::Node::Directory(e) => parent_directory.directories.push(e), - proto::node::Node::File(e) => parent_directory.files.push(e), - proto::node::Node::Symlink(e) => parent_directory.symlinks.push(e), + Node::Directory(e) => parent_directory.directories.push(e), + Node::File(e) => parent_directory.files.push(e), + Node::Symlink(e) => parent_directory.symlinks.push(e), } } } diff --git a/tvix/castore/src/lib.rs b/tvix/castore/src/lib.rs new file mode 100644 index 000000000000..76490fb7e6ef --- /dev/null +++ b/tvix/castore/src/lib.rs @@ -0,0 +1,15 @@ +mod digests; +mod errors; + +pub mod blobservice; +pub mod directoryservice; +pub mod fixtures; +pub mod import; +pub mod proto; +pub mod utils; + +pub use digests::B3Digest; +pub use errors::Error; + +#[cfg(test)] +mod tests; diff --git a/tvix/store/src/proto/grpc_blobservice_wrapper.rs b/tvix/castore/src/proto/grpc_blobservice_wrapper.rs index 93db1deef69a..93db1deef69a 100644 --- a/tvix/store/src/proto/grpc_blobservice_wrapper.rs +++ b/tvix/castore/src/proto/grpc_blobservice_wrapper.rs diff --git a/tvix/store/src/proto/grpc_directoryservice_wrapper.rs b/tvix/castore/src/proto/grpc_directoryservice_wrapper.rs index 5e143a7bd7a8..5e143a7bd7a8 100644 --- a/tvix/store/src/proto/grpc_directoryservice_wrapper.rs +++ b/tvix/castore/src/proto/grpc_directoryservice_wrapper.rs diff --git a/tvix/castore/src/proto/mod.rs b/tvix/castore/src/proto/mod.rs new file mode 100644 index 000000000000..2a44383fdd85 --- /dev/null +++ b/tvix/castore/src/proto/mod.rs @@ -0,0 +1,279 @@ +#![allow(clippy::derive_partial_eq_without_eq, non_snake_case)] +// https://github.com/hyperium/tonic/issues/1056 +use data_encoding::BASE64; +use std::{collections::HashSet, iter::Peekable}; +use thiserror::Error; + +use prost::Message; + +mod grpc_blobservice_wrapper; +mod grpc_directoryservice_wrapper; + +pub use grpc_blobservice_wrapper::GRPCBlobServiceWrapper; +pub use grpc_directoryservice_wrapper::GRPCDirectoryServiceWrapper; + +use crate::B3Digest; + +tonic::include_proto!("tvix.castore.v1"); + +#[cfg(feature = "reflection")] +/// Compiled file descriptors for implementing [gRPC +/// reflection](https://github.com/grpc/grpc/blob/master/doc/server-reflection.md) with e.g. +/// [`tonic_reflection`](https://docs.rs/tonic-reflection). +pub const FILE_DESCRIPTOR_SET: &[u8] = tonic::include_file_descriptor_set!("tvix.castore.v1"); + +#[cfg(test)] +mod tests; + +/// Errors that can occur during the validation of Directory messages. +#[derive(Debug, PartialEq, Eq, Error)] +pub enum ValidateDirectoryError { + /// Elements are not in sorted order + #[error("{} is not sorted", std::str::from_utf8(.0).unwrap_or(&BASE64.encode(.0)))] + WrongSorting(Vec<u8>), + /// Multiple elements with the same name encountered + #[error("{0:?} is a duplicate name")] + DuplicateName(Vec<u8>), + /// Invalid name encountered + #[error("Invalid name in {0:?}")] + InvalidName(Vec<u8>), + /// Invalid digest length encountered + #[error("Invalid Digest length: {0}")] + InvalidDigestLen(usize), +} + +/// Checks a Node name for validity as an intermediate node, and returns an +/// error that's generated from the supplied constructor. +/// +/// We disallow slashes, null bytes, '.', '..' and the empty string. +fn validate_node_name<E>(name: &[u8], err: fn(Vec<u8>) -> E) -> Result<(), E> { + if name.is_empty() + || name == b".." + || name == b"." + || name.contains(&0x00) + || name.contains(&b'/') + { + return Err(err(name.to_vec())); + } + Ok(()) +} + +/// NamedNode is implemented for [FileNode], [DirectoryNode] and [SymlinkNode] +/// and [node::Node], so we can ask all of them for the name easily. +pub trait NamedNode { + fn get_name(&self) -> &[u8]; +} + +impl NamedNode for &FileNode { + fn get_name(&self) -> &[u8] { + &self.name + } +} + +impl NamedNode for &DirectoryNode { + fn get_name(&self) -> &[u8] { + &self.name + } +} + +impl NamedNode for &SymlinkNode { + fn get_name(&self) -> &[u8] { + &self.name + } +} + +impl NamedNode for node::Node { + fn get_name(&self) -> &[u8] { + match self { + node::Node::File(node_file) => &node_file.name, + node::Node::Directory(node_directory) => &node_directory.name, + node::Node::Symlink(node_symlink) => &node_symlink.name, + } + } +} + +impl node::Node { + /// Returns the node with a new name. + pub fn rename(self, name: bytes::Bytes) -> Self { + match self { + node::Node::Directory(n) => node::Node::Directory(DirectoryNode { name, ..n }), + node::Node::File(n) => node::Node::File(FileNode { name, ..n }), + node::Node::Symlink(n) => node::Node::Symlink(SymlinkNode { name, ..n }), + } + } +} + +/// Accepts a name, and a mutable reference to the previous name. +/// If the passed name is larger than the previous one, the reference is updated. +/// If it's not, an error is returned. +fn update_if_lt_prev<'n>( + prev_name: &mut &'n [u8], + name: &'n [u8], +) -> Result<(), ValidateDirectoryError> { + if *name < **prev_name { + return Err(ValidateDirectoryError::WrongSorting(name.to_vec())); + } + *prev_name = name; + Ok(()) +} + +/// Inserts the given name into a HashSet if it's not already in there. +/// If it is, an error is returned. +fn insert_once<'n>( + seen_names: &mut HashSet<&'n [u8]>, + name: &'n [u8], +) -> Result<(), ValidateDirectoryError> { + if seen_names.get(name).is_some() { + return Err(ValidateDirectoryError::DuplicateName(name.to_vec())); + } + seen_names.insert(name); + Ok(()) +} + +impl Directory { + /// The size of a directory is the number of all regular and symlink elements, + /// the number of directory elements, and their size fields. + pub fn size(&self) -> u32 { + self.files.len() as u32 + + self.symlinks.len() as u32 + + self + .directories + .iter() + .fold(0, |acc: u32, e| (acc + 1 + e.size)) + } + + /// Calculates the digest of a Directory, which is the blake3 hash of a + /// Directory protobuf message, serialized in protobuf canonical form. + pub fn digest(&self) -> B3Digest { + let mut hasher = blake3::Hasher::new(); + + hasher + .update(&self.encode_to_vec()) + .finalize() + .as_bytes() + .into() + } + + /// validate checks the directory for invalid data, such as: + /// - violations of name restrictions + /// - invalid digest lengths + /// - not properly sorted lists + /// - duplicate names in the three lists + pub fn validate(&self) -> Result<(), ValidateDirectoryError> { + let mut seen_names: HashSet<&[u8]> = HashSet::new(); + + let mut last_directory_name: &[u8] = b""; + let mut last_file_name: &[u8] = b""; + let mut last_symlink_name: &[u8] = b""; + + // check directories + for directory_node in &self.directories { + validate_node_name(&directory_node.name, ValidateDirectoryError::InvalidName)?; + // ensure the digest has the appropriate size. + if TryInto::<B3Digest>::try_into(directory_node.digest.clone()).is_err() { + return Err(ValidateDirectoryError::InvalidDigestLen( + directory_node.digest.len(), + )); + } + + update_if_lt_prev(&mut last_directory_name, &directory_node.name)?; + insert_once(&mut seen_names, &directory_node.name)?; + } + + // check files + for file_node in &self.files { + validate_node_name(&file_node.name, ValidateDirectoryError::InvalidName)?; + if TryInto::<B3Digest>::try_into(file_node.digest.clone()).is_err() { + return Err(ValidateDirectoryError::InvalidDigestLen( + file_node.digest.len(), + )); + } + + update_if_lt_prev(&mut last_file_name, &file_node.name)?; + insert_once(&mut seen_names, &file_node.name)?; + } + + // check symlinks + for symlink_node in &self.symlinks { + validate_node_name(&symlink_node.name, ValidateDirectoryError::InvalidName)?; + + update_if_lt_prev(&mut last_symlink_name, &symlink_node.name)?; + insert_once(&mut seen_names, &symlink_node.name)?; + } + + Ok(()) + } + + /// Allows iterating over all three nodes ([DirectoryNode], [FileNode], + /// [SymlinkNode]) in an ordered fashion, as long as the individual lists + /// are sorted (which can be checked by the [Directory::validate]). + pub fn nodes(&self) -> DirectoryNodesIterator { + return DirectoryNodesIterator { + i_directories: self.directories.iter().peekable(), + i_files: self.files.iter().peekable(), + i_symlinks: self.symlinks.iter().peekable(), + }; + } +} + +/// Struct to hold the state of an iterator over all nodes of a Directory. +/// +/// Internally, this keeps peekable Iterators over all three lists of a +/// Directory message. +pub struct DirectoryNodesIterator<'a> { + // directory: &Directory, + i_directories: Peekable<std::slice::Iter<'a, DirectoryNode>>, + i_files: Peekable<std::slice::Iter<'a, FileNode>>, + i_symlinks: Peekable<std::slice::Iter<'a, SymlinkNode>>, +} + +/// looks at two elements implementing NamedNode, and returns true if "left +/// is smaller / comes first". +/// +/// Some(_) is preferred over None. +fn left_name_lt_right<A: NamedNode, B: NamedNode>(left: Option<&A>, right: Option<&B>) -> bool { + match left { + // if left is None, right always wins + None => false, + Some(left_inner) => { + // left is Some. + match right { + // left is Some, right is None - left wins. + None => true, + Some(right_inner) => { + // both are Some - compare the name. + return left_inner.get_name() < right_inner.get_name(); + } + } + } + } +} + +impl Iterator for DirectoryNodesIterator<'_> { + type Item = node::Node; + + // next returns the next node in the Directory. + // we peek at all three internal iterators, and pick the one with the + // smallest name, to ensure lexicographical ordering. + // The individual lists are already known to be sorted. + fn next(&mut self) -> Option<Self::Item> { + if left_name_lt_right(self.i_directories.peek(), self.i_files.peek()) { + // i_directories is still in the game, compare with symlinks + if left_name_lt_right(self.i_directories.peek(), self.i_symlinks.peek()) { + self.i_directories + .next() + .cloned() + .map(node::Node::Directory) + } else { + self.i_symlinks.next().cloned().map(node::Node::Symlink) + } + } else { + // i_files is still in the game, compare with symlinks + if left_name_lt_right(self.i_files.peek(), self.i_symlinks.peek()) { + self.i_files.next().cloned().map(node::Node::File) + } else { + self.i_symlinks.next().cloned().map(node::Node::Symlink) + } + } + } +} diff --git a/tvix/store/src/proto/tests/directory.rs b/tvix/castore/src/proto/tests/directory.rs index eed49b2b593c..eed49b2b593c 100644 --- a/tvix/store/src/proto/tests/directory.rs +++ b/tvix/castore/src/proto/tests/directory.rs diff --git a/tvix/store/src/proto/tests/directory_nodes_iterator.rs b/tvix/castore/src/proto/tests/directory_nodes_iterator.rs index 68f147a33210..68f147a33210 100644 --- a/tvix/store/src/proto/tests/directory_nodes_iterator.rs +++ b/tvix/castore/src/proto/tests/directory_nodes_iterator.rs diff --git a/tvix/store/src/proto/tests/grpc_blobservice.rs b/tvix/castore/src/proto/tests/grpc_blobservice.rs index 497893f03dd7..0d7b340b4409 100644 --- a/tvix/store/src/proto/tests/grpc_blobservice.rs +++ b/tvix/castore/src/proto/tests/grpc_blobservice.rs @@ -1,7 +1,7 @@ +use crate::fixtures::{BLOB_A, BLOB_A_DIGEST}; use crate::proto::blob_service_server::BlobService as GRPCBlobService; use crate::proto::{BlobChunk, GRPCBlobServiceWrapper, ReadBlobRequest, StatBlobRequest}; -use crate::tests::fixtures::{BLOB_A, BLOB_A_DIGEST}; -use crate::tests::utils::gen_blob_service; +use crate::utils::gen_blob_service; use tokio_stream::StreamExt; fn gen_grpc_blob_service() -> GRPCBlobServiceWrapper { diff --git a/tvix/store/src/proto/tests/grpc_directoryservice.rs b/tvix/castore/src/proto/tests/grpc_directoryservice.rs index a5300039fb9f..6e8cf1e4a7a4 100644 --- a/tvix/store/src/proto/tests/grpc_directoryservice.rs +++ b/tvix/castore/src/proto/tests/grpc_directoryservice.rs @@ -1,9 +1,9 @@ +use crate::fixtures::{DIRECTORY_A, DIRECTORY_B, DIRECTORY_C}; use crate::proto::directory_service_server::DirectoryService as GRPCDirectoryService; use crate::proto::get_directory_request::ByWhat; use crate::proto::{Directory, DirectoryNode, SymlinkNode}; use crate::proto::{GRPCDirectoryServiceWrapper, GetDirectoryRequest}; -use crate::tests::fixtures::{DIRECTORY_A, DIRECTORY_B, DIRECTORY_C}; -use crate::tests::utils::gen_directory_service; +use crate::utils::gen_directory_service; use tokio_stream::StreamExt; use tonic::Status; diff --git a/tvix/castore/src/proto/tests/mod.rs b/tvix/castore/src/proto/tests/mod.rs new file mode 100644 index 000000000000..8b62fadeb5a6 --- /dev/null +++ b/tvix/castore/src/proto/tests/mod.rs @@ -0,0 +1,4 @@ +mod directory; +mod directory_nodes_iterator; +mod grpc_blobservice; +mod grpc_directoryservice; diff --git a/tvix/store/src/tests/import.rs b/tvix/castore/src/tests/import.rs index 3f7f7dff9db1..77ed6d21c07a 100644 --- a/tvix/store/src/tests/import.rs +++ b/tvix/castore/src/tests/import.rs @@ -1,8 +1,7 @@ -use super::utils::{gen_blob_service, gen_directory_service}; +use crate::fixtures::*; use crate::import::ingest_path; use crate::proto; -use crate::tests::fixtures::DIRECTORY_COMPLICATED; -use crate::tests::fixtures::*; +use crate::utils::{gen_blob_service, gen_directory_service}; use tempfile::TempDir; #[cfg(target_family = "unix")] @@ -29,7 +28,7 @@ async fn symlink() { .expect("must succeed"); assert_eq!( - crate::proto::node::Node::Symlink(proto::SymlinkNode { + proto::node::Node::Symlink(proto::SymlinkNode { name: "doesntmatter".into(), target: "/nix/store/somewhereelse".into(), }), @@ -54,7 +53,7 @@ async fn single_file() { .expect("must succeed"); assert_eq!( - crate::proto::node::Node::File(proto::FileNode { + proto::node::Node::File(proto::FileNode { name: "root".into(), digest: HELLOWORLD_BLOB_DIGEST.clone().into(), size: HELLOWORLD_BLOB_CONTENTS.len() as u32, @@ -94,7 +93,7 @@ async fn complicated() { // ensure root_node matched expectations assert_eq!( - crate::proto::node::Node::Directory(proto::DirectoryNode { + proto::node::Node::Directory(proto::DirectoryNode { name: tmpdir .path() .file_name() diff --git a/tvix/castore/src/tests/mod.rs b/tvix/castore/src/tests/mod.rs new file mode 100644 index 000000000000..d016f3e0aa55 --- /dev/null +++ b/tvix/castore/src/tests/mod.rs @@ -0,0 +1 @@ +mod import; diff --git a/tvix/castore/src/utils.rs b/tvix/castore/src/utils.rs new file mode 100644 index 000000000000..660cb2f4b084 --- /dev/null +++ b/tvix/castore/src/utils.rs @@ -0,0 +1,19 @@ +//! A crate containing constructors to provide instances of a BlobService and +//! DirectoryService. +//! Only used for testing purposes, but across crates. +//! Should be removed once we have a better concept of a "Service registry". + +use std::sync::Arc; + +use crate::{ + blobservice::{BlobService, MemoryBlobService}, + directoryservice::{DirectoryService, MemoryDirectoryService}, +}; + +pub fn gen_blob_service() -> Arc<dyn BlobService> { + Arc::new(MemoryBlobService::default()) +} + +pub fn gen_directory_service() -> Arc<dyn DirectoryService> { + Arc::new(MemoryDirectoryService::default()) +} diff --git a/tvix/cli/Cargo.toml b/tvix/cli/Cargo.toml index cb795976a971..577ee8bb9e87 100644 --- a/tvix/cli/Cargo.toml +++ b/tvix/cli/Cargo.toml @@ -9,6 +9,7 @@ path = "src/main.rs" [dependencies] nix-compat = { path = "../nix-compat" } +tvix-castore = { path = "../castore" } tvix-store = { path = "../store", features = []} tvix-eval = { path = "../eval" } bytes = "1.4.0" diff --git a/tvix/cli/src/main.rs b/tvix/cli/src/main.rs index 65970e1d1cd8..ebcfe4b800b8 100644 --- a/tvix/cli/src/main.rs +++ b/tvix/cli/src/main.rs @@ -13,10 +13,10 @@ use std::{fs, path::PathBuf}; use clap::Parser; use known_paths::KnownPaths; use rustyline::{error::ReadlineError, Editor}; +use tvix_castore::blobservice::MemoryBlobService; +use tvix_castore::directoryservice::MemoryDirectoryService; use tvix_eval::observer::{DisassemblingObserver, TracingObserver}; use tvix_eval::Value; -use tvix_store::blobservice::MemoryBlobService; -use tvix_store::directoryservice::MemoryDirectoryService; use tvix_store::pathinfoservice::MemoryPathInfoService; use tvix_store_io::TvixStoreIO; diff --git a/tvix/cli/src/tvix_store_io.rs b/tvix/cli/src/tvix_store_io.rs index ef112858b1bc..cc69357282fd 100644 --- a/tvix/cli/src/tvix_store_io.rs +++ b/tvix/cli/src/tvix_store_io.rs @@ -6,14 +6,17 @@ use tokio::io::AsyncReadExt; use tracing::{error, instrument, warn}; use tvix_eval::{EvalIO, FileType, StdIO}; -use tvix_store::{ +use tvix_castore::{ blobservice::BlobService, directoryservice::{self, DirectoryService}, import, + proto::{node::Node, NamedNode}, + B3Digest, +}; +use tvix_store::{ nar::calculate_size_and_sha256, pathinfoservice::PathInfoService, - proto::{node::Node, NamedNode, NarInfo, PathInfo}, - B3Digest, + proto::{NarInfo, PathInfo}, }; /// Implements [EvalIO], asking given [PathInfoService], [DirectoryService] @@ -330,7 +333,7 @@ async fn import_path_with_pathinfo( // assemble the [PathInfo] object. let path_info = PathInfo { - node: Some(tvix_store::proto::Node { + node: Some(tvix_castore::proto::Node { node: Some(root_node), }), // There's no reference scanning on path contents ingested like this. diff --git a/tvix/default.nix b/tvix/default.nix index 8082a13155c8..84d9730d1907 100644 --- a/tvix/default.nix +++ b/tvix/default.nix @@ -36,8 +36,13 @@ let nativeBuildInputs = protobufDep prev; }; + tvix-castore = prev: { + PROTO_ROOT = depot.tvix.proto; + nativeBuildInputs = protobufDep prev; + }; + tvix-store = prev: { - PROTO_ROOT = depot.tvix.store.protos; + PROTO_ROOT = depot.tvix.proto; nativeBuildInputs = protobufDep prev; }; }; @@ -103,20 +108,26 @@ in ] ++ iconvDarwinDep; }; - # Builds and tests the code in store/protos. - store-protos-go = pkgs.buildGoModule { - name = "store-golang"; - src = depot.third_party.gitignoreSource ./store/protos; + # Builds and tests the code in castore/protos. + # castore-protos-go = pkgs.buildGoModule { + # name = "castore-golang"; + # src = depot.third_party.gitignoreSource ./store/protos; + # vendorHash = "sha256-00000000000000000000000000000000000000000000"; + # }; - vendorHash = "sha256-7xfXBBU3xJz7ifjk7Owm/byTfCQ8oaZtqXzBKhLqo00="; - }; + # Builds and tests the code in store/protos. + # store-protos-go = pkgs.buildGoModule { + # name = "store-golang"; + # src = depot.third_party.gitignoreSource ./store/protos; + # vendorHash = "sha256-00000000000000000000000000000000000000000000"; + # }; # Build the Rust documentation for publishing on docs.tvix.dev. rust-docs = pkgs.stdenv.mkDerivation { inherit cargoDeps; name = "tvix-rust-docs"; src = depot.third_party.gitignoreSource ./.; - PROTO_ROOT = depot.tvix.store.protos; + PROTO_ROOT = depot.tvix.proto; buildInputs = [ pkgs.fuse @@ -135,5 +146,10 @@ in ''; }; - meta.ci.targets = [ "store-protos-go" "shell" "rust-docs" ]; + meta.ci.targets = [ + # "castore-protos-go" + # "store-protos-go" + "shell" + "rust-docs" + ]; } diff --git a/tvix/proto/default.nix b/tvix/proto/default.nix index 35e2eba7fed4..0ee102e4f958 100644 --- a/tvix/proto/default.nix +++ b/tvix/proto/default.nix @@ -1,9 +1,15 @@ -# Build protocol buffer definitions to ensure that protos are valid in -# CI. Note that the output of this build target is not actually used -# anywhere, it just functions as a CI check for now. -{ pkgs, ... }: +# Target containing just the proto files used in tvix -pkgs.runCommand "tvix-cc-proto" { } '' - mkdir $out - ${pkgs.protobuf}/bin/protoc -I ${./.} evaluator.proto --cpp_out=$out -'' +{ depot, lib, ... }: + +depot.nix.sparseTree { + name = "tvix-protos"; + root = depot.path.origSrc; + paths = [ + ../castore/protos/castore.proto + ../castore/protos/rpc_blobstore.proto + ../castore/protos/rpc_directory.proto + ../store/protos/pathinfo.proto + ../store/protos/rpc_pathinfo.proto + ]; +} diff --git a/tvix/store/Cargo.toml b/tvix/store/Cargo.toml index 81f9d7e75b48..a2e143de7014 100644 --- a/tvix/store/Cargo.toml +++ b/tvix/store/Cargo.toml @@ -7,28 +7,29 @@ edition = "2021" anyhow = "1.0.68" async-stream = "0.3.5" blake3 = { version = "1.3.1", features = ["rayon", "std"] } +bytes = "1.4.0" clap = { version = "4.0", features = ["derive", "env"] } count-write = "0.1.0" data-encoding = "2.3.3" +futures = "0.3.28" lazy_static = "1.4.0" nix-compat = { path = "../nix-compat" } parking_lot = "0.12.1" +pin-project-lite = "0.2.13" prost = "0.11.2" sha2 = "0.10.6" sled = { version = "0.34.7", features = ["compression"] } thiserror = "1.0.38" tokio-stream = { version = "0.1.14", features = ["fs"] } +tokio-util = { version = "0.7.8", features = ["io", "io-util"] } tokio = { version = "1.28.0", features = ["fs", "net", "rt-multi-thread", "signal"] } tonic = "0.8.2" +tower = "0.4.13" tracing = "0.1.37" tracing-subscriber = { version = "0.3.16", features = ["json"] } -walkdir = "2.4.0" -tokio-util = { version = "0.7.8", features = ["io", "io-util"] } -tower = "0.4.13" -futures = "0.3.28" -bytes = "1.4.0" +tvix-castore = { path = "../castore" } url = "2.4.0" -pin-project-lite = "0.2.13" +walkdir = "2.4.0" [dependencies.fuse-backend-rs] optional = true diff --git a/tvix/store/README.md b/tvix/store/README.md index ca64cd249c1e..8fa4cf5a67c5 100644 --- a/tvix/store/README.md +++ b/tvix/store/README.md @@ -14,9 +14,12 @@ However, enough information is preserved to still be able to render NAR and NARInfo when needed. ## More Information -Check the `protos/` subfolder for the definition of the exact RPC methods and -messages. +The store consists out of two different gRPC services, `tvix.castore.v1` for +the low-level content-addressed bits, and `tvix.store.v1` for the Nix and +`StorePath`-specific bits. +Check the `protos/` subfolder both here and in `castore` for the definition of +the exact RPC methods and messages. ## Interacting with the GRPC service manually The shell environment in `//tvix` provides `evans`, which is an interactive @@ -37,15 +40,16 @@ $ evans --host localhost --port 8000 -r repl more expressive universal gRPC client -tvix.store.v1@localhost:8000> service BlobService +localhost:8000> package tvix.castore.v1 +tvix.castore.v1@localhost:8000> service BlobService -tvix.store.v1.BlobService@localhost:8000> call Put --bytes-from-file +tvix.castore.v1.BlobService@localhost:8000> call Put --bytes-from-file data (TYPE_BYTES) => /run/current-system/system { "digest": "KOM3/IHEx7YfInAnlJpAElYezq0Sxn9fRz7xuClwNfA=" } -tvix.store.v1.BlobService@localhost:8000> call Get --bytes-as-base64 +tvix.castore.v1.BlobService@localhost:8000> call Get --bytes-as-base64 digest (TYPE_BYTES) => KOM3/IHEx7YfInAnlJpAElYezq0Sxn9fRz7xuClwNfA= { "data": "eDg2XzY0LWxpbnV4" diff --git a/tvix/store/build.rs b/tvix/store/build.rs index a021dc328af6..9a7356f84eec 100644 --- a/tvix/store/build.rs +++ b/tvix/store/build.rs @@ -15,6 +15,7 @@ fn main() -> Result<()> { // https://github.com/hyperium/tonic/issues/908 let mut config = prost_build::Config::new(); config.bytes(["."]); + config.extern_path(".tvix.castore.v1", "::tvix_castore::proto"); builder .build_server(true) @@ -22,10 +23,7 @@ fn main() -> Result<()> { .compile_with_config( config, &[ - "tvix/store/protos/castore.proto", "tvix/store/protos/pathinfo.proto", - "tvix/store/protos/rpc_blobstore.proto", - "tvix/store/protos/rpc_directory.proto", "tvix/store/protos/rpc_pathinfo.proto", ], // If we are in running `cargo build` manually, using `../..` works fine, diff --git a/tvix/store/protos/castore.pb.go b/tvix/store/protos/castore.pb.go deleted file mode 100644 index 074b39d548e3..000000000000 --- a/tvix/store/protos/castore.pb.go +++ /dev/null @@ -1,450 +0,0 @@ -// SPDX-FileCopyrightText: edef <edef@unfathomable.blue> -// SPDX-License-Identifier: OSL-3.0 OR MIT OR Apache-2.0 - -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.31.0 -// protoc (unknown) -// source: tvix/store/protos/castore.proto - -package storev1 - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -// A Directory can contain Directory, File or Symlink nodes. -// Each of these nodes have a name attribute, which is the basename in that directory -// and node type specific attributes. -// The name attribute: -// - MUST not contain slashes or null bytes -// - MUST not be '.' or '..' -// - MUST be unique across all three lists -// -// Elements in each list need to be lexicographically ordered by the name -// attribute. -type Directory struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Directories []*DirectoryNode `protobuf:"bytes,1,rep,name=directories,proto3" json:"directories,omitempty"` - Files []*FileNode `protobuf:"bytes,2,rep,name=files,proto3" json:"files,omitempty"` - Symlinks []*SymlinkNode `protobuf:"bytes,3,rep,name=symlinks,proto3" json:"symlinks,omitempty"` -} - -func (x *Directory) Reset() { - *x = Directory{} - if protoimpl.UnsafeEnabled { - mi := &file_tvix_store_protos_castore_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Directory) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Directory) ProtoMessage() {} - -func (x *Directory) ProtoReflect() protoreflect.Message { - mi := &file_tvix_store_protos_castore_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Directory.ProtoReflect.Descriptor instead. -func (*Directory) Descriptor() ([]byte, []int) { - return file_tvix_store_protos_castore_proto_rawDescGZIP(), []int{0} -} - -func (x *Directory) GetDirectories() []*DirectoryNode { - if x != nil { - return x.Directories - } - return nil -} - -func (x *Directory) GetFiles() []*FileNode { - if x != nil { - return x.Files - } - return nil -} - -func (x *Directory) GetSymlinks() []*SymlinkNode { - if x != nil { - return x.Symlinks - } - return nil -} - -// A DirectoryNode represents a directory in a Directory. -type DirectoryNode struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // The (base)name of the directory - Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - // The blake3 hash of a Directory message, serialized in protobuf canonical form. - Digest []byte `protobuf:"bytes,2,opt,name=digest,proto3" json:"digest,omitempty"` - // Number of child elements in the Directory referred to by `digest`. - // Calculated by summing up the numbers of `directories`, `files` and - // `symlinks`, and for each directory, its size field. Used for inode - // number calculation. - // This field is precisely as verifiable as any other Merkle tree edge. - // Resolve `digest`, and you can compute it incrementally. Resolve the - // entire tree, and you can fully compute it from scratch. - // A credulous implementation won't reject an excessive size, but this is - // harmless: you'll have some ordinals without nodes. Undersizing is - // obvious and easy to reject: you won't have an ordinal for some nodes. - Size uint32 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"` -} - -func (x *DirectoryNode) Reset() { - *x = DirectoryNode{} - if protoimpl.UnsafeEnabled { - mi := &file_tvix_store_protos_castore_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DirectoryNode) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DirectoryNode) ProtoMessage() {} - -func (x *DirectoryNode) ProtoReflect() protoreflect.Message { - mi := &file_tvix_store_protos_castore_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DirectoryNode.ProtoReflect.Descriptor instead. -func (*DirectoryNode) Descriptor() ([]byte, []int) { - return file_tvix_store_protos_castore_proto_rawDescGZIP(), []int{1} -} - -func (x *DirectoryNode) GetName() []byte { - if x != nil { - return x.Name - } - return nil -} - -func (x *DirectoryNode) GetDigest() []byte { - if x != nil { - return x.Digest - } - return nil -} - -func (x *DirectoryNode) GetSize() uint32 { - if x != nil { - return x.Size - } - return 0 -} - -// A FileNode represents a regular or executable file in a Directory. -type FileNode struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // The (base)name of the file - Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - // The blake3 digest of the file contents - Digest []byte `protobuf:"bytes,2,opt,name=digest,proto3" json:"digest,omitempty"` - // The file content size - Size uint32 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"` - // Whether the file is executable - Executable bool `protobuf:"varint,4,opt,name=executable,proto3" json:"executable,omitempty"` -} - -func (x *FileNode) Reset() { - *x = FileNode{} - if protoimpl.UnsafeEnabled { - mi := &file_tvix_store_protos_castore_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *FileNode) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*FileNode) ProtoMessage() {} - -func (x *FileNode) ProtoReflect() protoreflect.Message { - mi := &file_tvix_store_protos_castore_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use FileNode.ProtoReflect.Descriptor instead. -func (*FileNode) Descriptor() ([]byte, []int) { - return file_tvix_store_protos_castore_proto_rawDescGZIP(), []int{2} -} - -func (x *FileNode) GetName() []byte { - if x != nil { - return x.Name - } - return nil -} - -func (x *FileNode) GetDigest() []byte { - if x != nil { - return x.Digest - } - return nil -} - -func (x *FileNode) GetSize() uint32 { - if x != nil { - return x.Size - } - return 0 -} - -func (x *FileNode) GetExecutable() bool { - if x != nil { - return x.Executable - } - return false -} - -// A SymlinkNode represents a symbolic link in a Directory. -type SymlinkNode struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // The (base)name of the symlink - Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - // The target of the symlink. - Target []byte `protobuf:"bytes,2,opt,name=target,proto3" json:"target,omitempty"` -} - -func (x *SymlinkNode) Reset() { - *x = SymlinkNode{} - if protoimpl.UnsafeEnabled { - mi := &file_tvix_store_protos_castore_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SymlinkNode) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SymlinkNode) ProtoMessage() {} - -func (x *SymlinkNode) ProtoReflect() protoreflect.Message { - mi := &file_tvix_store_protos_castore_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SymlinkNode.ProtoReflect.Descriptor instead. -func (*SymlinkNode) Descriptor() ([]byte, []int) { - return file_tvix_store_protos_castore_proto_rawDescGZIP(), []int{3} -} - -func (x *SymlinkNode) GetName() []byte { - if x != nil { - return x.Name - } - return nil -} - -func (x *SymlinkNode) GetTarget() []byte { - if x != nil { - return x.Target - } - return nil -} - -var File_tvix_store_protos_castore_proto protoreflect.FileDescriptor - -var file_tvix_store_protos_castore_proto_rawDesc = []byte{ - 0x0a, 0x1f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x12, 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, - 0x22, 0xb2, 0x01, 0x0a, 0x09, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x3e, - 0x0a, 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x4e, 0x6f, 0x64, - 0x65, 0x52, 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x2d, - 0x0a, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, - 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, - 0x6c, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x36, 0x0a, - 0x08, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x08, 0x73, 0x79, 0x6d, - 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x22, 0x4f, 0x0a, 0x0d, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x79, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, - 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, - 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x22, 0x6a, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x6f, - 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x12, - 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x73, 0x69, - 0x7a, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, - 0x6c, 0x65, 0x22, 0x39, 0x0a, 0x0b, 0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x4e, 0x6f, 0x64, - 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x42, 0x28, 0x5a, - 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, - 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, - 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_tvix_store_protos_castore_proto_rawDescOnce sync.Once - file_tvix_store_protos_castore_proto_rawDescData = file_tvix_store_protos_castore_proto_rawDesc -) - -func file_tvix_store_protos_castore_proto_rawDescGZIP() []byte { - file_tvix_store_protos_castore_proto_rawDescOnce.Do(func() { - file_tvix_store_protos_castore_proto_rawDescData = protoimpl.X.CompressGZIP(file_tvix_store_protos_castore_proto_rawDescData) - }) - return file_tvix_store_protos_castore_proto_rawDescData -} - -var file_tvix_store_protos_castore_proto_msgTypes = make([]protoimpl.MessageInfo, 4) -var file_tvix_store_protos_castore_proto_goTypes = []interface{}{ - (*Directory)(nil), // 0: tvix.store.v1.Directory - (*DirectoryNode)(nil), // 1: tvix.store.v1.DirectoryNode - (*FileNode)(nil), // 2: tvix.store.v1.FileNode - (*SymlinkNode)(nil), // 3: tvix.store.v1.SymlinkNode -} -var file_tvix_store_protos_castore_proto_depIdxs = []int32{ - 1, // 0: tvix.store.v1.Directory.directories:type_name -> tvix.store.v1.DirectoryNode - 2, // 1: tvix.store.v1.Directory.files:type_name -> tvix.store.v1.FileNode - 3, // 2: tvix.store.v1.Directory.symlinks:type_name -> tvix.store.v1.SymlinkNode - 3, // [3:3] is the sub-list for method output_type - 3, // [3:3] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name -} - -func init() { file_tvix_store_protos_castore_proto_init() } -func file_tvix_store_protos_castore_proto_init() { - if File_tvix_store_protos_castore_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_tvix_store_protos_castore_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Directory); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_tvix_store_protos_castore_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DirectoryNode); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_tvix_store_protos_castore_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FileNode); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_tvix_store_protos_castore_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SymlinkNode); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_tvix_store_protos_castore_proto_rawDesc, - NumEnums: 0, - NumMessages: 4, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_tvix_store_protos_castore_proto_goTypes, - DependencyIndexes: file_tvix_store_protos_castore_proto_depIdxs, - MessageInfos: file_tvix_store_protos_castore_proto_msgTypes, - }.Build() - File_tvix_store_protos_castore_proto = out.File - file_tvix_store_protos_castore_proto_rawDesc = nil - file_tvix_store_protos_castore_proto_goTypes = nil - file_tvix_store_protos_castore_proto_depIdxs = nil -} diff --git a/tvix/store/protos/default.nix b/tvix/store/protos/default.nix deleted file mode 100644 index d5c44842229d..000000000000 --- a/tvix/store/protos/default.nix +++ /dev/null @@ -1,16 +0,0 @@ -# Target containing just the proto files. - -{ depot, lib, ... }: - -let - inherit (lib.strings) hasSuffix; - inherit (builtins) attrNames filter readDir; - - protoFileNames = filter (hasSuffix ".proto") (attrNames (readDir ./.)); - protoFiles = map (f: ./. + ("/" + f)) protoFileNames; -in -depot.nix.sparseTree { - name = "tvix-store-protos"; - root = depot.path.origSrc; - paths = protoFiles; -} diff --git a/tvix/store/protos/pathinfo.pb.go b/tvix/store/protos/pathinfo.pb.go index 126fc34af27d..1e5479ac8f75 100644 --- a/tvix/store/protos/pathinfo.pb.go +++ b/tvix/store/protos/pathinfo.pb.go @@ -10,6 +10,7 @@ package storev1 import ( + protos "code.tvl.fyi/tvix/castore/protos" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -31,7 +32,7 @@ type PathInfo struct { unknownFields protoimpl.UnknownFields // The path can be a directory, file or symlink. - Node *Node `protobuf:"bytes,1,opt,name=node,proto3" json:"node,omitempty"` + Node *protos.Node `protobuf:"bytes,1,opt,name=node,proto3" json:"node,omitempty"` // List of references (output path hashes) // This really is the raw *bytes*, after decoding nixbase32, and not a // base32-encoded string. @@ -72,7 +73,7 @@ func (*PathInfo) Descriptor() ([]byte, []int) { return file_tvix_store_protos_pathinfo_proto_rawDescGZIP(), []int{0} } -func (x *PathInfo) GetNode() *Node { +func (x *PathInfo) GetNode() *protos.Node { if x != nil { return x.Node } @@ -93,101 +94,6 @@ func (x *PathInfo) GetNarinfo() *NARInfo { return nil } -type Node struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Types that are assignable to Node: - // - // *Node_Directory - // *Node_File - // *Node_Symlink - Node isNode_Node `protobuf_oneof:"node"` -} - -func (x *Node) Reset() { - *x = Node{} - if protoimpl.UnsafeEnabled { - mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Node) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Node) ProtoMessage() {} - -func (x *Node) ProtoReflect() protoreflect.Message { - mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Node.ProtoReflect.Descriptor instead. -func (*Node) Descriptor() ([]byte, []int) { - return file_tvix_store_protos_pathinfo_proto_rawDescGZIP(), []int{1} -} - -func (m *Node) GetNode() isNode_Node { - if m != nil { - return m.Node - } - return nil -} - -func (x *Node) GetDirectory() *DirectoryNode { - if x, ok := x.GetNode().(*Node_Directory); ok { - return x.Directory - } - return nil -} - -func (x *Node) GetFile() *FileNode { - if x, ok := x.GetNode().(*Node_File); ok { - return x.File - } - return nil -} - -func (x *Node) GetSymlink() *SymlinkNode { - if x, ok := x.GetNode().(*Node_Symlink); ok { - return x.Symlink - } - return nil -} - -type isNode_Node interface { - isNode_Node() -} - -type Node_Directory struct { - Directory *DirectoryNode `protobuf:"bytes,1,opt,name=directory,proto3,oneof"` -} - -type Node_File struct { - File *FileNode `protobuf:"bytes,2,opt,name=file,proto3,oneof"` -} - -type Node_Symlink struct { - Symlink *SymlinkNode `protobuf:"bytes,3,opt,name=symlink,proto3,oneof"` -} - -func (*Node_Directory) isNode_Node() {} - -func (*Node_File) isNode_Node() {} - -func (*Node_Symlink) isNode_Node() {} - // Nix C++ uses NAR (Nix Archive) as a format to transfer store paths, // and stores metadata and signatures in NARInfo files. // Store all these attributes in a separate message. @@ -219,7 +125,7 @@ type NARInfo struct { func (x *NARInfo) Reset() { *x = NARInfo{} if protoimpl.UnsafeEnabled { - mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[2] + mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -232,7 +138,7 @@ func (x *NARInfo) String() string { func (*NARInfo) ProtoMessage() {} func (x *NARInfo) ProtoReflect() protoreflect.Message { - mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[2] + mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -245,7 +151,7 @@ func (x *NARInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use NARInfo.ProtoReflect.Descriptor instead. func (*NARInfo) Descriptor() ([]byte, []int) { - return file_tvix_store_protos_pathinfo_proto_rawDescGZIP(), []int{2} + return file_tvix_store_protos_pathinfo_proto_rawDescGZIP(), []int{1} } func (x *NARInfo) GetNarSize() uint64 { @@ -289,7 +195,7 @@ type NARInfo_Signature struct { func (x *NARInfo_Signature) Reset() { *x = NARInfo_Signature{} if protoimpl.UnsafeEnabled { - mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[3] + mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -302,7 +208,7 @@ func (x *NARInfo_Signature) String() string { func (*NARInfo_Signature) ProtoMessage() {} func (x *NARInfo_Signature) ProtoReflect() protoreflect.Message { - mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[3] + mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -315,7 +221,7 @@ func (x *NARInfo_Signature) ProtoReflect() protoreflect.Message { // Deprecated: Use NARInfo_Signature.ProtoReflect.Descriptor instead. func (*NARInfo_Signature) Descriptor() ([]byte, []int) { - return file_tvix_store_protos_pathinfo_proto_rawDescGZIP(), []int{2, 0} + return file_tvix_store_protos_pathinfo_proto_rawDescGZIP(), []int{1, 0} } func (x *NARInfo_Signature) GetName() string { @@ -338,46 +244,35 @@ var file_tvix_store_protos_pathinfo_proto_rawDesc = []byte{ 0x0a, 0x20, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x70, 0x61, 0x74, 0x68, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, - 0x31, 0x1a, 0x1f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x22, 0x85, 0x01, 0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x12, - 0x27, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, - 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x6f, - 0x64, 0x65, 0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x66, 0x65, - 0x72, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0a, 0x72, 0x65, - 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x30, 0x0a, 0x07, 0x6e, 0x61, 0x72, 0x69, - 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x74, 0x76, 0x69, 0x78, - 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, - 0x6f, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0xb3, 0x01, 0x0a, 0x04, 0x4e, - 0x6f, 0x64, 0x65, 0x12, 0x3c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, - 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x79, 0x12, 0x2d, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, - 0x12, 0x36, 0x0a, 0x07, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, - 0x07, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x42, 0x06, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, - 0x22, 0xe3, 0x01, 0x0a, 0x07, 0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, - 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, - 0x6e, 0x61, 0x72, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x61, 0x72, 0x5f, 0x73, - 0x68, 0x61, 0x32, 0x35, 0x36, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6e, 0x61, 0x72, - 0x53, 0x68, 0x61, 0x32, 0x35, 0x36, 0x12, 0x40, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x76, 0x69, - 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x41, 0x52, 0x49, 0x6e, - 0x66, 0x6f, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x0a, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x65, - 0x72, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x0e, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, - 0x73, 0x1a, 0x33, 0x0a, 0x09, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x12, - 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, - 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x31, 0x1a, 0x21, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x87, 0x01, 0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, + 0x6f, 0x12, 0x29, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x15, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x1e, 0x0a, 0x0a, + 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, + 0x52, 0x0a, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x30, 0x0a, 0x07, + 0x6e, 0x61, 0x72, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, + 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x41, + 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0xe3, + 0x01, 0x0a, 0x07, 0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x61, + 0x72, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x6e, 0x61, + 0x72, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x68, 0x61, + 0x32, 0x35, 0x36, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6e, 0x61, 0x72, 0x53, 0x68, + 0x61, 0x32, 0x35, 0x36, 0x12, 0x40, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, + 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, 0x6f, + 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, + 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x0e, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x1a, + 0x33, 0x0a, 0x09, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, + 0x64, 0x61, 0x74, 0x61, 0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, + 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -392,28 +287,22 @@ func file_tvix_store_protos_pathinfo_proto_rawDescGZIP() []byte { return file_tvix_store_protos_pathinfo_proto_rawDescData } -var file_tvix_store_protos_pathinfo_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_tvix_store_protos_pathinfo_proto_msgTypes = make([]protoimpl.MessageInfo, 3) var file_tvix_store_protos_pathinfo_proto_goTypes = []interface{}{ (*PathInfo)(nil), // 0: tvix.store.v1.PathInfo - (*Node)(nil), // 1: tvix.store.v1.Node - (*NARInfo)(nil), // 2: tvix.store.v1.NARInfo - (*NARInfo_Signature)(nil), // 3: tvix.store.v1.NARInfo.Signature - (*DirectoryNode)(nil), // 4: tvix.store.v1.DirectoryNode - (*FileNode)(nil), // 5: tvix.store.v1.FileNode - (*SymlinkNode)(nil), // 6: tvix.store.v1.SymlinkNode + (*NARInfo)(nil), // 1: tvix.store.v1.NARInfo + (*NARInfo_Signature)(nil), // 2: tvix.store.v1.NARInfo.Signature + (*protos.Node)(nil), // 3: tvix.castore.v1.Node } var file_tvix_store_protos_pathinfo_proto_depIdxs = []int32{ - 1, // 0: tvix.store.v1.PathInfo.node:type_name -> tvix.store.v1.Node - 2, // 1: tvix.store.v1.PathInfo.narinfo:type_name -> tvix.store.v1.NARInfo - 4, // 2: tvix.store.v1.Node.directory:type_name -> tvix.store.v1.DirectoryNode - 5, // 3: tvix.store.v1.Node.file:type_name -> tvix.store.v1.FileNode - 6, // 4: tvix.store.v1.Node.symlink:type_name -> tvix.store.v1.SymlinkNode - 3, // 5: tvix.store.v1.NARInfo.signatures:type_name -> tvix.store.v1.NARInfo.Signature - 6, // [6:6] is the sub-list for method output_type - 6, // [6:6] is the sub-list for method input_type - 6, // [6:6] is the sub-list for extension type_name - 6, // [6:6] is the sub-list for extension extendee - 0, // [0:6] is the sub-list for field type_name + 3, // 0: tvix.store.v1.PathInfo.node:type_name -> tvix.castore.v1.Node + 1, // 1: tvix.store.v1.PathInfo.narinfo:type_name -> tvix.store.v1.NARInfo + 2, // 2: tvix.store.v1.NARInfo.signatures:type_name -> tvix.store.v1.NARInfo.Signature + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_tvix_store_protos_pathinfo_proto_init() } @@ -421,7 +310,6 @@ func file_tvix_store_protos_pathinfo_proto_init() { if File_tvix_store_protos_pathinfo_proto != nil { return } - file_tvix_store_protos_castore_proto_init() if !protoimpl.UnsafeEnabled { file_tvix_store_protos_pathinfo_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*PathInfo); i { @@ -436,18 +324,6 @@ func file_tvix_store_protos_pathinfo_proto_init() { } } file_tvix_store_protos_pathinfo_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Node); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_tvix_store_protos_pathinfo_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*NARInfo); i { case 0: return &v.state @@ -459,7 +335,7 @@ func file_tvix_store_protos_pathinfo_proto_init() { return nil } } - file_tvix_store_protos_pathinfo_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_tvix_store_protos_pathinfo_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*NARInfo_Signature); i { case 0: return &v.state @@ -472,18 +348,13 @@ func file_tvix_store_protos_pathinfo_proto_init() { } } } - file_tvix_store_protos_pathinfo_proto_msgTypes[1].OneofWrappers = []interface{}{ - (*Node_Directory)(nil), - (*Node_File)(nil), - (*Node_Symlink)(nil), - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_tvix_store_protos_pathinfo_proto_rawDesc, NumEnums: 0, - NumMessages: 4, + NumMessages: 3, NumExtensions: 0, NumServices: 0, }, diff --git a/tvix/store/protos/pathinfo.proto b/tvix/store/protos/pathinfo.proto index 896d4aa225ac..aa98c6df9a2d 100644 --- a/tvix/store/protos/pathinfo.proto +++ b/tvix/store/protos/pathinfo.proto @@ -4,7 +4,7 @@ syntax = "proto3"; package tvix.store.v1; -import "tvix/store/protos/castore.proto"; +import "tvix/castore/protos/castore.proto"; option go_package = "code.tvl.fyi/tvix/store/protos;storev1"; @@ -12,7 +12,7 @@ option go_package = "code.tvl.fyi/tvix/store/protos;storev1"; // That's a single element inside /nix/store. message PathInfo { // The path can be a directory, file or symlink. - Node node = 1; + tvix.castore.v1.Node node = 1; // List of references (output path hashes) // This really is the raw *bytes*, after decoding nixbase32, and not a @@ -23,14 +23,6 @@ message PathInfo { NARInfo narinfo = 3; } -message Node { - oneof node { - DirectoryNode directory = 1; - FileNode file = 2; - SymlinkNode symlink = 3; - } -} - // Nix C++ uses NAR (Nix Archive) as a format to transfer store paths, // and stores metadata and signatures in NARInfo files. // Store all these attributes in a separate message. diff --git a/tvix/store/protos/rpc_directory.pb.go b/tvix/store/protos/rpc_directory.pb.go deleted file mode 100644 index ac5384677b1e..000000000000 --- a/tvix/store/protos/rpc_directory.pb.go +++ /dev/null @@ -1,271 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright © 2022 The Tvix Authors - -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.31.0 -// protoc (unknown) -// source: tvix/store/protos/rpc_directory.proto - -package storev1 - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type GetDirectoryRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Types that are assignable to ByWhat: - // - // *GetDirectoryRequest_Digest - ByWhat isGetDirectoryRequest_ByWhat `protobuf_oneof:"by_what"` - // If set to true, recursively resolve all child Directory messages. - // Directory messages SHOULD be streamed in a recursive breadth-first walk, - // but other orders are also fine, as long as Directory messages are only - // sent after they are referred to from previously sent Directory messages. - Recursive bool `protobuf:"varint,2,opt,name=recursive,proto3" json:"recursive,omitempty"` -} - -func (x *GetDirectoryRequest) Reset() { - *x = GetDirectoryRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_tvix_store_protos_rpc_directory_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetDirectoryRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetDirectoryRequest) ProtoMessage() {} - -func (x *GetDirectoryRequest) ProtoReflect() protoreflect.Message { - mi := &file_tvix_store_protos_rpc_directory_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetDirectoryRequest.ProtoReflect.Descriptor instead. -func (*GetDirectoryRequest) Descriptor() ([]byte, []int) { - return file_tvix_store_protos_rpc_directory_proto_rawDescGZIP(), []int{0} -} - -func (m *GetDirectoryRequest) GetByWhat() isGetDirectoryRequest_ByWhat { - if m != nil { - return m.ByWhat - } - return nil -} - -func (x *GetDirectoryRequest) GetDigest() []byte { - if x, ok := x.GetByWhat().(*GetDirectoryRequest_Digest); ok { - return x.Digest - } - return nil -} - -func (x *GetDirectoryRequest) GetRecursive() bool { - if x != nil { - return x.Recursive - } - return false -} - -type isGetDirectoryRequest_ByWhat interface { - isGetDirectoryRequest_ByWhat() -} - -type GetDirectoryRequest_Digest struct { - // The blake3 hash of the (root) Directory message, serialized in - // protobuf canonical form. - // Keep in mind this can be a subtree of another root. - Digest []byte `protobuf:"bytes,1,opt,name=digest,proto3,oneof"` -} - -func (*GetDirectoryRequest_Digest) isGetDirectoryRequest_ByWhat() {} - -type PutDirectoryResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - RootDigest []byte `protobuf:"bytes,1,opt,name=root_digest,json=rootDigest,proto3" json:"root_digest,omitempty"` -} - -func (x *PutDirectoryResponse) Reset() { - *x = PutDirectoryResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_tvix_store_protos_rpc_directory_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *PutDirectoryResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*PutDirectoryResponse) ProtoMessage() {} - -func (x *PutDirectoryResponse) ProtoReflect() protoreflect.Message { - mi := &file_tvix_store_protos_rpc_directory_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use PutDirectoryResponse.ProtoReflect.Descriptor instead. -func (*PutDirectoryResponse) Descriptor() ([]byte, []int) { - return file_tvix_store_protos_rpc_directory_proto_rawDescGZIP(), []int{1} -} - -func (x *PutDirectoryResponse) GetRootDigest() []byte { - if x != nil { - return x.RootDigest - } - return nil -} - -var File_tvix_store_protos_rpc_directory_proto protoreflect.FileDescriptor - -var file_tvix_store_protos_rpc_directory_proto_rawDesc = []byte{ - 0x0a, 0x25, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x1f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, - 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x58, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x44, 0x69, - 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, - 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, - 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x63, 0x75, - 0x72, 0x73, 0x69, 0x76, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x72, 0x65, 0x63, - 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x62, 0x79, 0x5f, 0x77, 0x68, 0x61, - 0x74, 0x22, 0x37, 0x0a, 0x14, 0x50, 0x75, 0x74, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x6f, 0x6f, - 0x74, 0x5f, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, - 0x72, 0x6f, 0x6f, 0x74, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x32, 0xa1, 0x01, 0x0a, 0x10, 0x44, - 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, - 0x45, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x22, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x74, 0x76, 0x69, - 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x79, 0x30, 0x01, 0x12, 0x46, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x18, 0x2e, - 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, - 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x1a, 0x23, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, - 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x74, 0x44, 0x69, 0x72, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x42, 0x28, - 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, - 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, - 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_tvix_store_protos_rpc_directory_proto_rawDescOnce sync.Once - file_tvix_store_protos_rpc_directory_proto_rawDescData = file_tvix_store_protos_rpc_directory_proto_rawDesc -) - -func file_tvix_store_protos_rpc_directory_proto_rawDescGZIP() []byte { - file_tvix_store_protos_rpc_directory_proto_rawDescOnce.Do(func() { - file_tvix_store_protos_rpc_directory_proto_rawDescData = protoimpl.X.CompressGZIP(file_tvix_store_protos_rpc_directory_proto_rawDescData) - }) - return file_tvix_store_protos_rpc_directory_proto_rawDescData -} - -var file_tvix_store_protos_rpc_directory_proto_msgTypes = make([]protoimpl.MessageInfo, 2) -var file_tvix_store_protos_rpc_directory_proto_goTypes = []interface{}{ - (*GetDirectoryRequest)(nil), // 0: tvix.store.v1.GetDirectoryRequest - (*PutDirectoryResponse)(nil), // 1: tvix.store.v1.PutDirectoryResponse - (*Directory)(nil), // 2: tvix.store.v1.Directory -} -var file_tvix_store_protos_rpc_directory_proto_depIdxs = []int32{ - 0, // 0: tvix.store.v1.DirectoryService.Get:input_type -> tvix.store.v1.GetDirectoryRequest - 2, // 1: tvix.store.v1.DirectoryService.Put:input_type -> tvix.store.v1.Directory - 2, // 2: tvix.store.v1.DirectoryService.Get:output_type -> tvix.store.v1.Directory - 1, // 3: tvix.store.v1.DirectoryService.Put:output_type -> tvix.store.v1.PutDirectoryResponse - 2, // [2:4] is the sub-list for method output_type - 0, // [0:2] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_tvix_store_protos_rpc_directory_proto_init() } -func file_tvix_store_protos_rpc_directory_proto_init() { - if File_tvix_store_protos_rpc_directory_proto != nil { - return - } - file_tvix_store_protos_castore_proto_init() - if !protoimpl.UnsafeEnabled { - file_tvix_store_protos_rpc_directory_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetDirectoryRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_tvix_store_protos_rpc_directory_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutDirectoryResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - file_tvix_store_protos_rpc_directory_proto_msgTypes[0].OneofWrappers = []interface{}{ - (*GetDirectoryRequest_Digest)(nil), - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_tvix_store_protos_rpc_directory_proto_rawDesc, - NumEnums: 0, - NumMessages: 2, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_tvix_store_protos_rpc_directory_proto_goTypes, - DependencyIndexes: file_tvix_store_protos_rpc_directory_proto_depIdxs, - MessageInfos: file_tvix_store_protos_rpc_directory_proto_msgTypes, - }.Build() - File_tvix_store_protos_rpc_directory_proto = out.File - file_tvix_store_protos_rpc_directory_proto_rawDesc = nil - file_tvix_store_protos_rpc_directory_proto_goTypes = nil - file_tvix_store_protos_rpc_directory_proto_depIdxs = nil -} diff --git a/tvix/store/protos/rpc_pathinfo.pb.go b/tvix/store/protos/rpc_pathinfo.pb.go index 293cb5a7c321..8a3c10a82101 100644 --- a/tvix/store/protos/rpc_pathinfo.pb.go +++ b/tvix/store/protos/rpc_pathinfo.pb.go @@ -10,6 +10,7 @@ package storev1 import ( + protos "code.tvl.fyi/tvix/castore/protos" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -205,39 +206,42 @@ var file_tvix_store_protos_rpc_pathinfo_proto_rawDesc = []byte{ 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x20, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x70, 0x61, 0x74, 0x68, 0x69, 0x6e, 0x66, - 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x47, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x50, 0x61, - 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, - 0x0e, 0x62, 0x79, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x0c, 0x62, 0x79, 0x4f, 0x75, 0x74, 0x70, 0x75, - 0x74, 0x48, 0x61, 0x73, 0x68, 0x42, 0x09, 0x0a, 0x07, 0x62, 0x79, 0x5f, 0x77, 0x68, 0x61, 0x74, - 0x22, 0x15, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x50, 0x0a, 0x14, 0x43, 0x61, 0x6c, 0x63, 0x75, - 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x41, 0x52, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x19, 0x0a, 0x08, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x61, - 0x72, 0x5f, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, - 0x6e, 0x61, 0x72, 0x53, 0x68, 0x61, 0x32, 0x35, 0x36, 0x32, 0x9e, 0x02, 0x0a, 0x0f, 0x50, 0x61, - 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x41, 0x0a, - 0x03, 0x47, 0x65, 0x74, 0x12, 0x21, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, - 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, - 0x12, 0x37, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, - 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, - 0x1a, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x48, 0x0a, 0x0c, 0x43, 0x61, 0x6c, - 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x41, 0x52, 0x12, 0x13, 0x2e, 0x74, 0x76, 0x69, 0x78, - 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x1a, 0x23, - 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, - 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x41, 0x52, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x22, 0x2e, 0x74, 0x76, - 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x30, 0x01, 0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f, - 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, - 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f, - 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x21, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, + 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, + 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x47, 0x0a, 0x12, 0x47, 0x65, + 0x74, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x26, 0x0a, 0x0e, 0x62, 0x79, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x68, 0x61, + 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x0c, 0x62, 0x79, 0x4f, 0x75, + 0x74, 0x70, 0x75, 0x74, 0x48, 0x61, 0x73, 0x68, 0x42, 0x09, 0x0a, 0x07, 0x62, 0x79, 0x5f, 0x77, + 0x68, 0x61, 0x74, 0x22, 0x15, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x50, 0x0a, 0x14, 0x43, 0x61, + 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x41, 0x52, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, + 0x0a, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x09, 0x6e, 0x61, 0x72, 0x53, 0x68, 0x61, 0x32, 0x35, 0x36, 0x32, 0xa0, 0x02, 0x0a, + 0x0f, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x41, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x21, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, + 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x61, 0x74, 0x68, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x74, 0x76, 0x69, + 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, + 0x6e, 0x66, 0x6f, 0x12, 0x37, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x17, 0x2e, 0x74, 0x76, 0x69, + 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, + 0x6e, 0x66, 0x6f, 0x1a, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x4a, 0x0a, 0x0c, + 0x43, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x41, 0x52, 0x12, 0x15, 0x2e, 0x74, + 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, + 0x6f, 0x64, 0x65, 0x1a, 0x23, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x41, 0x52, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, + 0x12, 0x22, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x30, 0x01, 0x42, + 0x28, 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, + 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x73, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( @@ -258,12 +262,12 @@ var file_tvix_store_protos_rpc_pathinfo_proto_goTypes = []interface{}{ (*ListPathInfoRequest)(nil), // 1: tvix.store.v1.ListPathInfoRequest (*CalculateNARResponse)(nil), // 2: tvix.store.v1.CalculateNARResponse (*PathInfo)(nil), // 3: tvix.store.v1.PathInfo - (*Node)(nil), // 4: tvix.store.v1.Node + (*protos.Node)(nil), // 4: tvix.castore.v1.Node } var file_tvix_store_protos_rpc_pathinfo_proto_depIdxs = []int32{ 0, // 0: tvix.store.v1.PathInfoService.Get:input_type -> tvix.store.v1.GetPathInfoRequest 3, // 1: tvix.store.v1.PathInfoService.Put:input_type -> tvix.store.v1.PathInfo - 4, // 2: tvix.store.v1.PathInfoService.CalculateNAR:input_type -> tvix.store.v1.Node + 4, // 2: tvix.store.v1.PathInfoService.CalculateNAR:input_type -> tvix.castore.v1.Node 1, // 3: tvix.store.v1.PathInfoService.List:input_type -> tvix.store.v1.ListPathInfoRequest 3, // 4: tvix.store.v1.PathInfoService.Get:output_type -> tvix.store.v1.PathInfo 3, // 5: tvix.store.v1.PathInfoService.Put:output_type -> tvix.store.v1.PathInfo diff --git a/tvix/store/protos/rpc_pathinfo.proto b/tvix/store/protos/rpc_pathinfo.proto index e1d6cd774144..1930e87de004 100644 --- a/tvix/store/protos/rpc_pathinfo.proto +++ b/tvix/store/protos/rpc_pathinfo.proto @@ -5,6 +5,7 @@ syntax = "proto3"; package tvix.store.v1; import "tvix/store/protos/pathinfo.proto"; +import "tvix/castore/protos/castore.proto"; option go_package = "code.tvl.fyi/tvix/store/protos;storev1"; @@ -40,7 +41,7 @@ service PathInfoService { // // It can also be used to calculate arbitrary NAR hashes of output paths, // in case a legacy Nix Binary Cache frontend is provided. - rpc CalculateNAR(Node) returns (CalculateNARResponse); + rpc CalculateNAR(tvix.castore.v1.Node) returns (CalculateNARResponse); // Return a stream of PathInfo messages matching the criteria specified in // ListPathInfoRequest. diff --git a/tvix/store/protos/rpc_pathinfo_grpc.pb.go b/tvix/store/protos/rpc_pathinfo_grpc.pb.go index d7b6711c0310..10d8a7ffa49c 100644 --- a/tvix/store/protos/rpc_pathinfo_grpc.pb.go +++ b/tvix/store/protos/rpc_pathinfo_grpc.pb.go @@ -10,6 +10,7 @@ package storev1 import ( + protos "code.tvl.fyi/tvix/castore/protos" context "context" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" @@ -60,7 +61,7 @@ type PathInfoServiceClient interface { // // It can also be used to calculate arbitrary NAR hashes of output paths, // in case a legacy Nix Binary Cache frontend is provided. - CalculateNAR(ctx context.Context, in *Node, opts ...grpc.CallOption) (*CalculateNARResponse, error) + CalculateNAR(ctx context.Context, in *protos.Node, opts ...grpc.CallOption) (*CalculateNARResponse, error) // Return a stream of PathInfo messages matching the criteria specified in // ListPathInfoRequest. List(ctx context.Context, in *ListPathInfoRequest, opts ...grpc.CallOption) (PathInfoService_ListClient, error) @@ -92,7 +93,7 @@ func (c *pathInfoServiceClient) Put(ctx context.Context, in *PathInfo, opts ...g return out, nil } -func (c *pathInfoServiceClient) CalculateNAR(ctx context.Context, in *Node, opts ...grpc.CallOption) (*CalculateNARResponse, error) { +func (c *pathInfoServiceClient) CalculateNAR(ctx context.Context, in *protos.Node, opts ...grpc.CallOption) (*CalculateNARResponse, error) { out := new(CalculateNARResponse) err := c.cc.Invoke(ctx, PathInfoService_CalculateNAR_FullMethodName, in, out, opts...) if err != nil { @@ -165,7 +166,7 @@ type PathInfoServiceServer interface { // // It can also be used to calculate arbitrary NAR hashes of output paths, // in case a legacy Nix Binary Cache frontend is provided. - CalculateNAR(context.Context, *Node) (*CalculateNARResponse, error) + CalculateNAR(context.Context, *protos.Node) (*CalculateNARResponse, error) // Return a stream of PathInfo messages matching the criteria specified in // ListPathInfoRequest. List(*ListPathInfoRequest, PathInfoService_ListServer) error @@ -182,7 +183,7 @@ func (UnimplementedPathInfoServiceServer) Get(context.Context, *GetPathInfoReque func (UnimplementedPathInfoServiceServer) Put(context.Context, *PathInfo) (*PathInfo, error) { return nil, status.Errorf(codes.Unimplemented, "method Put not implemented") } -func (UnimplementedPathInfoServiceServer) CalculateNAR(context.Context, *Node) (*CalculateNARResponse, error) { +func (UnimplementedPathInfoServiceServer) CalculateNAR(context.Context, *protos.Node) (*CalculateNARResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method CalculateNAR not implemented") } func (UnimplementedPathInfoServiceServer) List(*ListPathInfoRequest, PathInfoService_ListServer) error { @@ -238,7 +239,7 @@ func _PathInfoService_Put_Handler(srv interface{}, ctx context.Context, dec func } func _PathInfoService_CalculateNAR_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(Node) + in := new(protos.Node) if err := dec(in); err != nil { return nil, err } @@ -250,7 +251,7 @@ func _PathInfoService_CalculateNAR_Handler(srv interface{}, ctx context.Context, FullMethod: PathInfoService_CalculateNAR_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(PathInfoServiceServer).CalculateNAR(ctx, req.(*Node)) + return srv.(PathInfoServiceServer).CalculateNAR(ctx, req.(*protos.Node)) } return interceptor(ctx, in, info, handler) } diff --git a/tvix/store/src/bin/tvix-store.rs b/tvix/store/src/bin/tvix-store.rs index 7761855cccb1..474a48c9fd1a 100644 --- a/tvix/store/src/bin/tvix-store.rs +++ b/tvix/store/src/bin/tvix-store.rs @@ -8,18 +8,18 @@ use std::path::Path; use std::path::PathBuf; use tokio::task::JoinHandle; use tracing_subscriber::prelude::*; -use tvix_store::blobservice; -use tvix_store::directoryservice; -use tvix_store::import; +use tvix_castore::blobservice; +use tvix_castore::directoryservice; +use tvix_castore::import; +use tvix_castore::proto::blob_service_server::BlobServiceServer; +use tvix_castore::proto::directory_service_server::DirectoryServiceServer; +use tvix_castore::proto::node::Node; +use tvix_castore::proto::GRPCBlobServiceWrapper; +use tvix_castore::proto::GRPCDirectoryServiceWrapper; +use tvix_castore::proto::NamedNode; use tvix_store::pathinfoservice; -use tvix_store::proto::blob_service_server::BlobServiceServer; -use tvix_store::proto::directory_service_server::DirectoryServiceServer; -use tvix_store::proto::node::Node; use tvix_store::proto::path_info_service_server::PathInfoServiceServer; -use tvix_store::proto::GRPCBlobServiceWrapper; -use tvix_store::proto::GRPCDirectoryServiceWrapper; use tvix_store::proto::GRPCPathInfoServiceWrapper; -use tvix_store::proto::NamedNode; use tvix_store::proto::NarInfo; use tvix_store::proto::PathInfo; @@ -30,6 +30,8 @@ use tvix_store::fs::TvixStoreFs; use tvix_store::fs::fuse::FuseDaemon; #[cfg(feature = "reflection")] +use tvix_castore::proto::FILE_DESCRIPTOR_SET as CASTORE_FILE_DESCRIPTOR_SET; +#[cfg(feature = "reflection")] use tvix_store::proto::FILE_DESCRIPTOR_SET; use clap::Parser; @@ -185,6 +187,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> { #[cfg(feature = "reflection")] { let reflection_svc = tonic_reflection::server::Builder::configure() + .register_encoded_file_descriptor_set(CASTORE_FILE_DESCRIPTOR_SET) .register_encoded_file_descriptor_set(FILE_DESCRIPTOR_SET) .build()?; router = router.add_service(reflection_svc); @@ -248,7 +251,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> { // assemble the [crate::proto::PathInfo] object. let path_info = PathInfo { - node: Some(tvix_store::proto::Node { + node: Some(tvix_castore::proto::Node { node: Some(root_node), }), // There's no reference scanning on path contents ingested like this. diff --git a/tvix/store/src/fs/inode_tracker.rs b/tvix/store/src/fs/inode_tracker.rs index ad1ef859a2f3..daf6b4ee79c2 100644 --- a/tvix/store/src/fs/inode_tracker.rs +++ b/tvix/store/src/fs/inode_tracker.rs @@ -1,8 +1,8 @@ use std::{collections::HashMap, sync::Arc}; -use crate::{proto, B3Digest}; - use super::inodes::{DirectoryInodeData, InodeData}; +use tvix_castore::proto as castorepb; +use tvix_castore::B3Digest; /// InodeTracker keeps track of inodes, stores data being these inodes and deals /// with inode allocation. @@ -139,21 +139,21 @@ impl InodeTracker { // Consume a list of children with zeroed inodes, and allocate (or fetch existing) inodes. fn allocate_inodes_for_children( &mut self, - children: Vec<(u64, proto::node::Node)>, - ) -> Vec<(u64, proto::node::Node)> { + children: Vec<(u64, castorepb::node::Node)>, + ) -> Vec<(u64, castorepb::node::Node)> { // allocate new inodes for all children - let mut children_new: Vec<(u64, proto::node::Node)> = Vec::new(); + let mut children_new: Vec<(u64, castorepb::node::Node)> = Vec::new(); for (child_ino, ref child_node) in children { debug_assert_eq!(0, child_ino, "expected child inode to be 0"); let child_ino = match child_node { - proto::node::Node::Directory(directory_node) => { + castorepb::node::Node::Directory(directory_node) => { // Try putting the sparse data in. If we already have a // populated version, it'll not update it. self.put(directory_node.into()) } - proto::node::Node::File(file_node) => self.put(file_node.into()), - proto::node::Node::Symlink(symlink_node) => self.put(symlink_node.into()), + castorepb::node::Node::File(file_node) => self.put(file_node.into()), + castorepb::node::Node::Symlink(symlink_node) => self.put(symlink_node.into()), }; children_new.push((child_ino, child_node.clone())) @@ -198,8 +198,8 @@ impl InodeTracker { #[cfg(test)] mod tests { use crate::fs::inodes::DirectoryInodeData; - use crate::proto; use crate::tests::fixtures; + use tvix_castore::proto as castorepb; use super::InodeData; use super::InodeTracker; @@ -304,7 +304,7 @@ mod tests { let (child_ino, child_node) = children.first().unwrap(); assert_ne!(dir_ino, *child_ino); assert_eq!( - &proto::node::Node::File( + &castorepb::node::Node::File( fixtures::DIRECTORY_WITH_KEEP.files.first().unwrap().clone() ), child_node @@ -362,7 +362,9 @@ mod tests { let (child_ino, child_node) = &children[0]; assert!(!seen_inodes.contains(child_ino)); assert_eq!( - &proto::node::Node::File(fixtures::DIRECTORY_COMPLICATED.files[0].clone()), + &castorepb::node::Node::File( + fixtures::DIRECTORY_COMPLICATED.files[0].clone() + ), child_node ); seen_inodes.push(*child_ino); @@ -373,7 +375,7 @@ mod tests { let (child_ino, child_node) = &children[1]; assert!(!seen_inodes.contains(child_ino)); assert_eq!( - &proto::node::Node::Symlink( + &castorepb::node::Node::Symlink( fixtures::DIRECTORY_COMPLICATED.symlinks[0].clone() ), child_node @@ -386,7 +388,7 @@ mod tests { let (child_ino, child_node) = &children[2]; assert!(!seen_inodes.contains(child_ino)); assert_eq!( - &proto::node::Node::Directory( + &castorepb::node::Node::Directory( fixtures::DIRECTORY_COMPLICATED.directories[0].clone() ), child_node @@ -439,7 +441,7 @@ mod tests { let (child_node_inode, child_node) = children.first().unwrap(); assert_ne!(dir_complicated_ino, *child_node_inode); assert_eq!( - &proto::node::Node::File( + &castorepb::node::Node::File( fixtures::DIRECTORY_WITH_KEEP.files.first().unwrap().clone() ), child_node diff --git a/tvix/store/src/fs/inodes.rs b/tvix/store/src/fs/inodes.rs index e8959ce3629b..928f51059002 100644 --- a/tvix/store/src/fs/inodes.rs +++ b/tvix/store/src/fs/inodes.rs @@ -1,6 +1,7 @@ //! This module contains all the data structures used to track information //! about inodes, which present tvix-store nodes in a filesystem. -use crate::{proto, B3Digest}; +use tvix_castore::proto as castorepb; +use tvix_castore::B3Digest; #[derive(Clone, Debug)] pub enum InodeData { @@ -10,33 +11,33 @@ pub enum InodeData { } /// This encodes the two different states of [InodeData::Directory]. -/// Either the data still is sparse (we only saw a [proto::DirectoryNode], but -/// didn't fetch the [proto::Directory] struct yet, -/// or we processed a lookup and did fetch the data. +/// Either the data still is sparse (we only saw a [castorepb::DirectoryNode], +/// but didn't fetch the [castorepb::Directory] struct yet, or we processed a +/// lookup and did fetch the data. #[derive(Clone, Debug)] pub enum DirectoryInodeData { - Sparse(B3Digest, u32), // digest, size - Populated(B3Digest, Vec<(u64, proto::node::Node)>), // [(child_inode, node)] + Sparse(B3Digest, u32), // digest, size + Populated(B3Digest, Vec<(u64, castorepb::node::Node)>), // [(child_inode, node)] } -impl From<&proto::node::Node> for InodeData { - fn from(value: &proto::node::Node) -> Self { +impl From<&castorepb::node::Node> for InodeData { + fn from(value: &castorepb::node::Node) -> Self { match value { - proto::node::Node::Directory(directory_node) => directory_node.into(), - proto::node::Node::File(file_node) => file_node.into(), - proto::node::Node::Symlink(symlink_node) => symlink_node.into(), + castorepb::node::Node::Directory(directory_node) => directory_node.into(), + castorepb::node::Node::File(file_node) => file_node.into(), + castorepb::node::Node::Symlink(symlink_node) => symlink_node.into(), } } } -impl From<&proto::SymlinkNode> for InodeData { - fn from(value: &proto::SymlinkNode) -> Self { +impl From<&castorepb::SymlinkNode> for InodeData { + fn from(value: &castorepb::SymlinkNode) -> Self { InodeData::Symlink(value.target.clone()) } } -impl From<&proto::FileNode> for InodeData { - fn from(value: &proto::FileNode) -> Self { +impl From<&castorepb::FileNode> for InodeData { + fn from(value: &castorepb::FileNode) -> Self { InodeData::Regular( value.digest.clone().try_into().unwrap(), value.size, @@ -46,8 +47,8 @@ impl From<&proto::FileNode> for InodeData { } /// Converts a DirectoryNode to a sparsely populated InodeData::Directory. -impl From<&proto::DirectoryNode> for InodeData { - fn from(value: &proto::DirectoryNode) -> Self { +impl From<&castorepb::DirectoryNode> for InodeData { + fn from(value: &castorepb::DirectoryNode) -> Self { InodeData::Directory(DirectoryInodeData::Sparse( value.digest.clone().try_into().unwrap(), value.size, @@ -57,11 +58,12 @@ impl From<&proto::DirectoryNode> for InodeData { /// converts a proto::Directory to a InodeData::Directory(DirectoryInodeData::Populated(..)). /// The inodes for each child are 0, because it's up to the InodeTracker to allocate them. -impl From<proto::Directory> for InodeData { - fn from(value: proto::Directory) -> Self { +impl From<castorepb::Directory> for InodeData { + fn from(value: castorepb::Directory) -> Self { let digest = value.digest(); - let children: Vec<(u64, proto::node::Node)> = value.nodes().map(|node| (0, node)).collect(); + let children: Vec<(u64, castorepb::node::Node)> = + value.nodes().map(|node| (0, node)).collect(); InodeData::Directory(DirectoryInodeData::Populated(digest, children)) } diff --git a/tvix/store/src/fs/mod.rs b/tvix/store/src/fs/mod.rs index 02d3bb3221ad..59b8f0d0854f 100644 --- a/tvix/store/src/fs/mod.rs +++ b/tvix/store/src/fs/mod.rs @@ -8,13 +8,8 @@ pub mod fuse; #[cfg(test)] mod tests; -use crate::{ - blobservice::{BlobReader, BlobService}, - directoryservice::DirectoryService, - pathinfoservice::PathInfoService, - proto::{node::Node, NamedNode}, - B3Digest, Error, -}; +use crate::pathinfoservice::PathInfoService; + use fuse_backend_rs::api::filesystem::{Context, FileSystem, FsOptions, ROOT_ID}; use futures::StreamExt; use nix_compat::store_path::StorePath; @@ -32,6 +27,12 @@ use tokio::{ sync::mpsc, }; use tracing::{debug, info_span, warn}; +use tvix_castore::{ + blobservice::{BlobReader, BlobService}, + directoryservice::DirectoryService, + proto::{node::Node, NamedNode}, + B3Digest, Error, +}; use self::{ file_attr::{gen_file_attr, ROOT_FILE_ATTR}, diff --git a/tvix/store/src/fs/tests.rs b/tvix/store/src/fs/tests.rs index 6837f8aa293a..2adea0ceb3a9 100644 --- a/tvix/store/src/fs/tests.rs +++ b/tvix/store/src/fs/tests.rs @@ -5,17 +5,17 @@ use std::path::Path; use std::sync::Arc; use tokio::{fs, io}; use tokio_stream::wrappers::ReadDirStream; +use tvix_castore::blobservice::BlobService; +use tvix_castore::directoryservice::DirectoryService; use tempfile::TempDir; -use crate::blobservice::BlobService; -use crate::directoryservice::DirectoryService; use crate::fs::{fuse::FuseDaemon, TvixStoreFs}; use crate::pathinfoservice::PathInfoService; -use crate::proto; -use crate::proto::{DirectoryNode, FileNode, PathInfo}; +use crate::proto::PathInfo; use crate::tests::fixtures; use crate::tests::utils::{gen_blob_service, gen_directory_service, gen_pathinfo_service}; +use tvix_castore::proto as castorepb; const BLOB_A_NAME: &str = "00000000000000000000000000000000-test"; const BLOB_B_NAME: &str = "55555555555555555555555555555555-test"; @@ -67,8 +67,8 @@ async fn populate_blob_a( // Create a PathInfo for it let path_info = PathInfo { - node: Some(proto::Node { - node: Some(proto::node::Node::File(FileNode { + node: Some(castorepb::Node { + node: Some(castorepb::node::Node::File(castorepb::FileNode { name: BLOB_A_NAME.into(), digest: fixtures::BLOB_A_DIGEST.clone().into(), size: fixtures::BLOB_A.len() as u32, @@ -97,8 +97,8 @@ async fn populate_blob_b( // Create a PathInfo for it let path_info = PathInfo { - node: Some(proto::Node { - node: Some(proto::node::Node::File(FileNode { + node: Some(castorepb::Node { + node: Some(castorepb::node::Node::File(castorepb::FileNode { name: BLOB_B_NAME.into(), digest: fixtures::BLOB_B_DIGEST.clone().into(), size: fixtures::BLOB_B.len() as u32, @@ -131,8 +131,8 @@ async fn populate_helloworld_blob( // Create a PathInfo for it let path_info = PathInfo { - node: Some(proto::Node { - node: Some(proto::node::Node::File(FileNode { + node: Some(castorepb::Node { + node: Some(castorepb::node::Node::File(castorepb::FileNode { name: HELLOWORLD_BLOB_NAME.into(), digest: fixtures::HELLOWORLD_BLOB_DIGEST.clone().into(), size: fixtures::HELLOWORLD_BLOB_CONTENTS.len() as u32, @@ -154,8 +154,8 @@ async fn populate_symlink( ) { // Create a PathInfo for it let path_info = PathInfo { - node: Some(proto::Node { - node: Some(proto::node::Node::Symlink(proto::SymlinkNode { + node: Some(castorepb::Node { + node: Some(castorepb::node::Node::Symlink(castorepb::SymlinkNode { name: SYMLINK_NAME.into(), target: BLOB_A_NAME.into(), })), @@ -177,8 +177,8 @@ async fn populate_symlink2( ) { // Create a PathInfo for it let path_info = PathInfo { - node: Some(proto::Node { - node: Some(proto::node::Node::Symlink(proto::SymlinkNode { + node: Some(castorepb::Node { + node: Some(castorepb::node::Node::Symlink(castorepb::SymlinkNode { name: SYMLINK_NAME2.into(), target: "/nix/store/somewhereelse".into(), })), @@ -211,8 +211,8 @@ async fn populate_directory_with_keep( // upload pathinfo let path_info = PathInfo { - node: Some(proto::Node { - node: Some(proto::node::Node::Directory(DirectoryNode { + node: Some(castorepb::Node { + node: Some(castorepb::node::Node::Directory(castorepb::DirectoryNode { name: DIRECTORY_WITH_KEEP_NAME.into(), digest: fixtures::DIRECTORY_WITH_KEEP.digest().into(), size: fixtures::DIRECTORY_WITH_KEEP.size(), @@ -235,8 +235,8 @@ async fn populate_pathinfo_without_directory( ) { // upload pathinfo let path_info = PathInfo { - node: Some(proto::Node { - node: Some(proto::node::Node::Directory(DirectoryNode { + node: Some(castorepb::Node { + node: Some(castorepb::node::Node::Directory(castorepb::DirectoryNode { name: DIRECTORY_WITH_KEEP_NAME.into(), digest: fixtures::DIRECTORY_WITH_KEEP.digest().into(), size: fixtures::DIRECTORY_WITH_KEEP.size(), @@ -258,8 +258,8 @@ async fn populate_blob_a_without_blob( ) { // Create a PathInfo for blob A let path_info = PathInfo { - node: Some(proto::Node { - node: Some(proto::node::Node::File(FileNode { + node: Some(castorepb::Node { + node: Some(castorepb::node::Node::File(castorepb::FileNode { name: BLOB_A_NAME.into(), digest: fixtures::BLOB_A_DIGEST.clone().into(), size: fixtures::BLOB_A.len() as u32, @@ -300,8 +300,8 @@ async fn populate_directory_complicated( // upload pathinfo let path_info = PathInfo { - node: Some(proto::Node { - node: Some(proto::node::Node::Directory(DirectoryNode { + node: Some(castorepb::Node { + node: Some(castorepb::node::Node::Directory(castorepb::DirectoryNode { name: DIRECTORY_COMPLICATED_NAME.into(), digest: fixtures::DIRECTORY_COMPLICATED.digest().into(), size: fixtures::DIRECTORY_COMPLICATED.size(), diff --git a/tvix/store/src/lib.rs b/tvix/store/src/lib.rs index 6270812d47fc..c59121453352 100644 --- a/tvix/store/src/lib.rs +++ b/tvix/store/src/lib.rs @@ -1,18 +1,9 @@ -mod digests; -mod errors; - #[cfg(feature = "fs")] pub mod fs; -pub mod blobservice; -pub mod directoryservice; -pub mod import; pub mod nar; pub mod pathinfoservice; pub mod proto; -pub use digests::B3Digest; -pub use errors::Error; - #[cfg(test)] mod tests; diff --git a/tvix/store/src/nar/mod.rs b/tvix/store/src/nar/mod.rs index 5a8bc21ae953..fc6805e9e758 100644 --- a/tvix/store/src/nar/mod.rs +++ b/tvix/store/src/nar/mod.rs @@ -1,16 +1,15 @@ -use crate::B3Digest; use data_encoding::BASE64; -use thiserror::Error; +use tvix_castore::{B3Digest, Error}; mod renderer; pub use renderer::calculate_size_and_sha256; pub use renderer::write_nar; /// Errors that can encounter while rendering NARs. -#[derive(Debug, Error)] +#[derive(Debug, thiserror::Error)] pub enum RenderError { #[error("failure talking to a backing store client: {0}")] - StoreError(crate::Error), + StoreError(Error), #[error("unable to find directory {}, referred from {:?}", .0, .1)] DirectoryNotFound(B3Digest, bytes::Bytes), diff --git a/tvix/store/src/nar/renderer.rs b/tvix/store/src/nar/renderer.rs index f1392472a50e..55dce911ee1a 100644 --- a/tvix/store/src/nar/renderer.rs +++ b/tvix/store/src/nar/renderer.rs @@ -1,20 +1,21 @@ use super::RenderError; -use crate::{ - blobservice::BlobService, - directoryservice::DirectoryService, - proto::{self, NamedNode}, -}; use count_write::CountWrite; use nix_compat::nar; use sha2::{Digest, Sha256}; use std::{io, sync::Arc}; use tokio::{io::BufReader, task::spawn_blocking}; use tracing::warn; +use tvix_castore::{ + blobservice::BlobService, + directoryservice::DirectoryService, + proto::{self as castorepb, NamedNode}, + Error, +}; /// Invoke [write_nar], and return the size and sha256 digest of the produced /// NAR output. pub async fn calculate_size_and_sha256( - root_node: &proto::node::Node, + root_node: &castorepb::node::Node, blob_service: Arc<dyn BlobService>, directory_service: Arc<dyn DirectoryService>, ) -> Result<(u64, [u8; 32]), RenderError> { @@ -26,9 +27,9 @@ pub async fn calculate_size_and_sha256( Ok((cw.count(), cw.into_inner().finalize().into())) } -/// Accepts a [proto::node::Node] pointing to the root of a (store) path, -/// and uses the passed blob_service and directory_service to -/// perform the necessary lookups as it traverses the structure. +/// Accepts a [castorepb::node::Node] pointing to the root of a (store) path, +/// and uses the passed blob_service and directory_service to perform the +/// necessary lookups as it traverses the structure. /// The contents in NAR serialization are writen to the passed [std::io::Write]. /// /// The writer is passed back in the return value. This is done because async Rust @@ -39,7 +40,7 @@ pub async fn calculate_size_and_sha256( /// This will panic if called outside the context of a Tokio runtime. pub async fn write_nar<W: std::io::Write + Send + 'static>( mut w: W, - proto_root_node: &proto::node::Node, + proto_root_node: &castorepb::node::Node, blob_service: Arc<dyn BlobService>, directory_service: Arc<dyn DirectoryService>, ) -> Result<W, RenderError> { @@ -69,24 +70,24 @@ pub async fn write_nar<W: std::io::Write + Send + 'static>( fn walk_node( tokio_handle: tokio::runtime::Handle, nar_node: nar::writer::Node, - proto_node: &proto::node::Node, + proto_node: &castorepb::node::Node, blob_service: Arc<dyn BlobService>, directory_service: Arc<dyn DirectoryService>, ) -> Result<(), RenderError> { match proto_node { - proto::node::Node::Symlink(proto_symlink_node) => { + castorepb::node::Node::Symlink(proto_symlink_node) => { nar_node .symlink(&proto_symlink_node.target) .map_err(RenderError::NARWriterError)?; } - proto::node::Node::File(proto_file_node) => { + castorepb::node::Node::File(proto_file_node) => { let digest = proto_file_node.digest.clone().try_into().map_err(|_e| { warn!( file_node = ?proto_file_node, "invalid digest length in file node", ); - RenderError::StoreError(crate::Error::StorageError( + RenderError::StoreError(Error::StorageError( "invalid digest len in file node".to_string(), )) })?; @@ -110,13 +111,13 @@ fn walk_node( ) .map_err(RenderError::NARWriterError)?; } - proto::node::Node::Directory(proto_directory_node) => { + castorepb::node::Node::Directory(proto_directory_node) => { let digest = proto_directory_node .digest .clone() .try_into() .map_err(|_e| { - RenderError::StoreError(crate::Error::StorageError( + RenderError::StoreError(Error::StorageError( "invalid digest len in directory node".to_string(), )) })?; diff --git a/tvix/store/src/pathinfoservice/from_addr.rs b/tvix/store/src/pathinfoservice/from_addr.rs index 36b30aecdcf5..93cb487f29b9 100644 --- a/tvix/store/src/pathinfoservice/from_addr.rs +++ b/tvix/store/src/pathinfoservice/from_addr.rs @@ -1,10 +1,9 @@ +use super::{GRPCPathInfoService, MemoryPathInfoService, PathInfoService, SledPathInfoService}; + use std::sync::Arc; +use tvix_castore::{blobservice::BlobService, directoryservice::DirectoryService, Error}; use url::Url; -use crate::{blobservice::BlobService, directoryservice::DirectoryService}; - -use super::{GRPCPathInfoService, MemoryPathInfoService, PathInfoService, SledPathInfoService}; - /// Constructs a new instance of a [PathInfoService] from an URI. /// /// The following URIs are supported: @@ -26,9 +25,9 @@ pub fn from_addr( uri: &str, blob_service: Arc<dyn BlobService>, directory_service: Arc<dyn DirectoryService>, -) -> Result<Arc<dyn PathInfoService>, crate::Error> { - let url = Url::parse(uri) - .map_err(|e| crate::Error::StorageError(format!("unable to parse url: {}", e)))?; +) -> Result<Arc<dyn PathInfoService>, Error> { + let url = + Url::parse(uri).map_err(|e| Error::StorageError(format!("unable to parse url: {}", e)))?; Ok(if url.scheme() == "memory" { Arc::new(MemoryPathInfoService::from_url( @@ -49,7 +48,7 @@ pub fn from_addr( directory_service, )?) } else { - Err(crate::Error::StorageError(format!( + Err(Error::StorageError(format!( "unknown scheme: {}", url.scheme() )))? diff --git a/tvix/store/src/pathinfoservice/grpc.rs b/tvix/store/src/pathinfoservice/grpc.rs index c116ddbc8905..6883c56104a6 100644 --- a/tvix/store/src/pathinfoservice/grpc.rs +++ b/tvix/store/src/pathinfoservice/grpc.rs @@ -1,14 +1,13 @@ use super::PathInfoService; -use crate::{ - blobservice::BlobService, - directoryservice::DirectoryService, - proto::{self, ListPathInfoRequest}, -}; +use crate::proto::{self, ListPathInfoRequest, PathInfo}; use async_stream::try_stream; use futures::Stream; use std::{pin::Pin, sync::Arc}; use tokio::net::UnixStream; use tonic::{async_trait, transport::Channel, Code}; +use tvix_castore::{ + blobservice::BlobService, directoryservice::DirectoryService, proto as castorepb, Error, +}; /// Connects to a (remote) tvix-store PathInfoService over gRPC. #[derive(Clone)] @@ -40,16 +39,14 @@ impl PathInfoService for GRPCPathInfoService { url: &url::Url, _blob_service: Arc<dyn BlobService>, _directory_service: Arc<dyn DirectoryService>, - ) -> Result<Self, crate::Error> { + ) -> Result<Self, tvix_castore::Error> { // Start checking for the scheme to start with grpc+. match url.scheme().strip_prefix("grpc+") { - None => Err(crate::Error::StorageError("invalid scheme".to_string())), + None => Err(Error::StorageError("invalid scheme".to_string())), Some(rest) => { if rest == "unix" { if url.host_str().is_some() { - return Err(crate::Error::StorageError( - "host may not be set".to_string(), - )); + return Err(Error::StorageError("host may not be set".to_string())); } let path = url.path().to_string(); let channel = tonic::transport::Endpoint::try_from("http://[::]:50051") // doesn't matter @@ -63,7 +60,7 @@ impl PathInfoService for GRPCPathInfoService { } else { // ensure path is empty, not supported with gRPC. if !url.path().is_empty() { - return Err(crate::Error::StorageError( + return Err(tvix_castore::Error::StorageError( "path may not be set".to_string(), )); } @@ -89,7 +86,7 @@ impl PathInfoService for GRPCPathInfoService { } } - async fn get(&self, digest: [u8; 20]) -> Result<Option<proto::PathInfo>, crate::Error> { + async fn get(&self, digest: [u8; 20]) -> Result<Option<PathInfo>, Error> { // Get a new handle to the gRPC client. let mut grpc_client = self.grpc_client.clone(); @@ -104,18 +101,18 @@ impl PathInfoService for GRPCPathInfoService { match path_info { Ok(path_info) => Ok(Some(path_info.into_inner())), Err(e) if e.code() == Code::NotFound => Ok(None), - Err(e) => Err(crate::Error::StorageError(e.to_string())), + Err(e) => Err(Error::StorageError(e.to_string())), } } - async fn put(&self, path_info: proto::PathInfo) -> Result<proto::PathInfo, crate::Error> { + async fn put(&self, path_info: PathInfo) -> Result<PathInfo, Error> { // Get a new handle to the gRPC client. let mut grpc_client = self.grpc_client.clone(); let path_info = grpc_client .put(path_info) .await - .map_err(|e| crate::Error::StorageError(e.to_string()))? + .map_err(|e| Error::StorageError(e.to_string()))? .into_inner(); Ok(path_info) @@ -123,36 +120,36 @@ impl PathInfoService for GRPCPathInfoService { async fn calculate_nar( &self, - root_node: &proto::node::Node, - ) -> Result<(u64, [u8; 32]), crate::Error> { + root_node: &castorepb::node::Node, + ) -> Result<(u64, [u8; 32]), Error> { // Get a new handle to the gRPC client. let mut grpc_client = self.grpc_client.clone(); let root_node = root_node.clone(); let path_info = grpc_client - .calculate_nar(proto::Node { + .calculate_nar(castorepb::Node { node: Some(root_node), }) .await - .map_err(|e| crate::Error::StorageError(e.to_string()))? + .map_err(|e| Error::StorageError(e.to_string()))? .into_inner(); let nar_sha256: [u8; 32] = path_info .nar_sha256 .to_vec() .try_into() - .map_err(|_e| crate::Error::StorageError("invalid digest length".to_string()))?; + .map_err(|_e| Error::StorageError("invalid digest length".to_string()))?; Ok((path_info.nar_size, nar_sha256)) } - fn list(&self) -> Pin<Box<dyn Stream<Item = Result<proto::PathInfo, crate::Error>> + Send>> { + fn list(&self) -> Pin<Box<dyn Stream<Item = Result<PathInfo, Error>> + Send>> { let mut grpc_client = self.grpc_client.clone(); let stream = try_stream! { let resp = grpc_client.list(ListPathInfoRequest::default()).await; - let mut stream = resp.map_err(|e| crate::Error::StorageError(e.to_string()))?.into_inner(); + let mut stream = resp.map_err(|e| Error::StorageError(e.to_string()))?.into_inner(); loop { match stream.message().await { @@ -160,7 +157,7 @@ impl PathInfoService for GRPCPathInfoService { Some(pathinfo) => { // validate the pathinfo if let Err(e) = pathinfo.validate() { - Err(crate::Error::StorageError(format!( + Err(Error::StorageError(format!( "pathinfo {:?} failed validation: {}", pathinfo, e )))?; @@ -171,7 +168,7 @@ impl PathInfoService for GRPCPathInfoService { return; }, }, - Err(e) => Err(crate::Error::StorageError(e.to_string()))?, + Err(e) => Err(Error::StorageError(e.to_string()))?, } } }; diff --git a/tvix/store/src/pathinfoservice/memory.rs b/tvix/store/src/pathinfoservice/memory.rs index 4cdc411ffb28..dbb4b02dd013 100644 --- a/tvix/store/src/pathinfoservice/memory.rs +++ b/tvix/store/src/pathinfoservice/memory.rs @@ -1,8 +1,5 @@ use super::PathInfoService; -use crate::{ - blobservice::BlobService, directoryservice::DirectoryService, nar::calculate_size_and_sha256, - proto, Error, -}; +use crate::{nar::calculate_size_and_sha256, proto::PathInfo}; use futures::{stream::iter, Stream}; use std::{ collections::HashMap, @@ -10,9 +7,12 @@ use std::{ sync::{Arc, RwLock}, }; use tonic::async_trait; +use tvix_castore::proto as castorepb; +use tvix_castore::Error; +use tvix_castore::{blobservice::BlobService, directoryservice::DirectoryService}; pub struct MemoryPathInfoService { - db: Arc<RwLock<HashMap<[u8; 20], proto::PathInfo>>>, + db: Arc<RwLock<HashMap<[u8; 20], PathInfo>>>, blob_service: Arc<dyn BlobService>, directory_service: Arc<dyn DirectoryService>, @@ -43,17 +43,17 @@ impl PathInfoService for MemoryPathInfoService { directory_service: Arc<dyn DirectoryService>, ) -> Result<Self, Error> { if url.scheme() != "memory" { - return Err(crate::Error::StorageError("invalid scheme".to_string())); + return Err(Error::StorageError("invalid scheme".to_string())); } if url.has_host() || !url.path().is_empty() { - return Err(crate::Error::StorageError("invalid url".to_string())); + return Err(Error::StorageError("invalid url".to_string())); } Ok(Self::new(blob_service, directory_service)) } - async fn get(&self, digest: [u8; 20]) -> Result<Option<proto::PathInfo>, Error> { + async fn get(&self, digest: [u8; 20]) -> Result<Option<PathInfo>, Error> { let db = self.db.read().unwrap(); match db.get(&digest) { @@ -62,7 +62,7 @@ impl PathInfoService for MemoryPathInfoService { } } - async fn put(&self, path_info: proto::PathInfo) -> Result<proto::PathInfo, Error> { + async fn put(&self, path_info: PathInfo) -> Result<PathInfo, Error> { // Call validate on the received PathInfo message. match path_info.validate() { Err(e) => Err(Error::InvalidRequest(format!( @@ -81,7 +81,10 @@ impl PathInfoService for MemoryPathInfoService { } } - async fn calculate_nar(&self, root_node: &proto::node::Node) -> Result<(u64, [u8; 32]), Error> { + async fn calculate_nar( + &self, + root_node: &castorepb::node::Node, + ) -> Result<(u64, [u8; 32]), Error> { calculate_size_and_sha256( root_node, self.blob_service.clone(), @@ -91,7 +94,7 @@ impl PathInfoService for MemoryPathInfoService { .map_err(|e| Error::StorageError(e.to_string())) } - fn list(&self) -> Pin<Box<dyn Stream<Item = Result<proto::PathInfo, Error>> + Send>> { + fn list(&self) -> Pin<Box<dyn Stream<Item = Result<PathInfo, Error>> + Send>> { let db = self.db.read().unwrap(); // Copy all elements into a list. diff --git a/tvix/store/src/pathinfoservice/mod.rs b/tvix/store/src/pathinfoservice/mod.rs index b436ad0b16dc..af7bbc9f88e4 100644 --- a/tvix/store/src/pathinfoservice/mod.rs +++ b/tvix/store/src/pathinfoservice/mod.rs @@ -8,10 +8,12 @@ use std::sync::Arc; use futures::Stream; use tonic::async_trait; +use tvix_castore::blobservice::BlobService; +use tvix_castore::directoryservice::DirectoryService; +use tvix_castore::proto as castorepb; +use tvix_castore::Error; -use crate::blobservice::BlobService; -use crate::directoryservice::DirectoryService; -use crate::{proto, Error}; +use crate::proto::PathInfo; pub use self::from_addr::from_addr; pub use self::grpc::GRPCPathInfoService; @@ -34,16 +36,19 @@ pub trait PathInfoService: Send + Sync { Self: Sized; /// Retrieve a PathInfo message by the output digest. - async fn get(&self, digest: [u8; 20]) -> Result<Option<proto::PathInfo>, Error>; + async fn get(&self, digest: [u8; 20]) -> Result<Option<PathInfo>, Error>; /// Store a PathInfo message. Implementations MUST call validate and reject /// invalid messages. - async fn put(&self, path_info: proto::PathInfo) -> Result<proto::PathInfo, Error>; + async fn put(&self, path_info: PathInfo) -> Result<PathInfo, Error>; /// Return the nar size and nar sha256 digest for a given root node. /// This can be used to calculate NAR-based output paths, /// and implementations are encouraged to cache it. - async fn calculate_nar(&self, root_node: &proto::node::Node) -> Result<(u64, [u8; 32]), Error>; + async fn calculate_nar( + &self, + root_node: &castorepb::node::Node, + ) -> Result<(u64, [u8; 32]), Error>; /// Iterate over all PathInfo objects in the store. /// Implementations can decide to disallow listing. @@ -52,5 +57,5 @@ pub trait PathInfoService: Send + Sync { /// and the box allows different underlying stream implementations to be returned since /// Rust doesn't support this as a generic in traits yet. This is the same thing that /// [async_trait] generates, but for streams instead of futures. - fn list(&self) -> Pin<Box<dyn Stream<Item = Result<proto::PathInfo, Error>> + Send>>; + fn list(&self) -> Pin<Box<dyn Stream<Item = Result<PathInfo, Error>> + Send>>; } diff --git a/tvix/store/src/pathinfoservice/sled.rs b/tvix/store/src/pathinfoservice/sled.rs index a9d0b029ee6b..bac384ea0912 100644 --- a/tvix/store/src/pathinfoservice/sled.rs +++ b/tvix/store/src/pathinfoservice/sled.rs @@ -1,13 +1,13 @@ use super::PathInfoService; -use crate::{ - blobservice::BlobService, directoryservice::DirectoryService, nar::calculate_size_and_sha256, - proto, Error, -}; +use crate::nar::calculate_size_and_sha256; +use crate::proto::PathInfo; use futures::{stream::iter, Stream}; use prost::Message; use std::{path::PathBuf, pin::Pin, sync::Arc}; use tonic::async_trait; use tracing::warn; +use tvix_castore::proto as castorepb; +use tvix_castore::{blobservice::BlobService, directoryservice::DirectoryService, Error}; /// SledPathInfoService stores PathInfo in a [sled](https://github.com/spacejam/sled). /// @@ -63,11 +63,11 @@ impl PathInfoService for SledPathInfoService { directory_service: Arc<dyn DirectoryService>, ) -> Result<Self, Error> { if url.scheme() != "sled" { - return Err(crate::Error::StorageError("invalid scheme".to_string())); + return Err(Error::StorageError("invalid scheme".to_string())); } if url.has_host() { - return Err(crate::Error::StorageError(format!( + return Err(Error::StorageError(format!( "invalid host: {}", url.host().unwrap() ))); @@ -78,7 +78,7 @@ impl PathInfoService for SledPathInfoService { Self::new_temporary(blob_service, directory_service) .map_err(|e| Error::StorageError(e.to_string())) } else if url.path() == "/" { - Err(crate::Error::StorageError( + Err(Error::StorageError( "cowardly refusing to open / with sled".to_string(), )) } else { @@ -87,10 +87,10 @@ impl PathInfoService for SledPathInfoService { } } - async fn get(&self, digest: [u8; 20]) -> Result<Option<proto::PathInfo>, Error> { + async fn get(&self, digest: [u8; 20]) -> Result<Option<PathInfo>, Error> { match self.db.get(digest) { Ok(None) => Ok(None), - Ok(Some(data)) => match proto::PathInfo::decode(&*data) { + Ok(Some(data)) => match PathInfo::decode(&*data) { Ok(path_info) => Ok(Some(path_info)), Err(e) => { warn!("failed to decode stored PathInfo: {}", e); @@ -110,7 +110,7 @@ impl PathInfoService for SledPathInfoService { } } - async fn put(&self, path_info: proto::PathInfo) -> Result<proto::PathInfo, Error> { + async fn put(&self, path_info: PathInfo) -> Result<PathInfo, Error> { // Call validate on the received PathInfo message. match path_info.validate() { Err(e) => Err(Error::InvalidRequest(format!( @@ -131,7 +131,10 @@ impl PathInfoService for SledPathInfoService { } } - async fn calculate_nar(&self, root_node: &proto::node::Node) -> Result<(u64, [u8; 32]), Error> { + async fn calculate_nar( + &self, + root_node: &castorepb::node::Node, + ) -> Result<(u64, [u8; 32]), Error> { calculate_size_and_sha256( root_node, self.blob_service.clone(), @@ -141,11 +144,11 @@ impl PathInfoService for SledPathInfoService { .map_err(|e| Error::StorageError(e.to_string())) } - fn list(&self) -> Pin<Box<dyn Stream<Item = Result<proto::PathInfo, Error>> + Send>> { + fn list(&self) -> Pin<Box<dyn Stream<Item = Result<PathInfo, Error>> + Send>> { Box::pin(iter(self.db.iter().values().map(|v| match v { Ok(data) => { // we retrieved some bytes - match proto::PathInfo::decode(&*data) { + match PathInfo::decode(&*data) { Ok(path_info) => Ok(path_info), Err(e) => { warn!("failed to decode stored PathInfo: {}", e); diff --git a/tvix/store/src/proto/grpc_pathinfoservice_wrapper.rs b/tvix/store/src/proto/grpc_pathinfoservice_wrapper.rs index 14ceb34c3af7..7632614291dc 100644 --- a/tvix/store/src/proto/grpc_pathinfoservice_wrapper.rs +++ b/tvix/store/src/proto/grpc_pathinfoservice_wrapper.rs @@ -7,6 +7,7 @@ use tokio::task; use tokio_stream::wrappers::ReceiverStream; use tonic::{async_trait, Request, Response, Result, Status}; use tracing::{debug, instrument, warn}; +use tvix_castore::proto as castorepb; pub struct GRPCPathInfoServiceWrapper { path_info_service: Arc<dyn PathInfoService>, @@ -67,7 +68,7 @@ impl proto::path_info_service_server::PathInfoService for GRPCPathInfoServiceWra #[instrument(skip(self))] async fn calculate_nar( &self, - request: Request<proto::Node>, + request: Request<castorepb::Node>, ) -> Result<Response<proto::CalculateNarResponse>> { match request.into_inner().node { None => Err(Status::invalid_argument("no root node sent")), diff --git a/tvix/store/src/proto/mod.rs b/tvix/store/src/proto/mod.rs index 97a2694ac3de..6924b023c942 100644 --- a/tvix/store/src/proto/mod.rs +++ b/tvix/store/src/proto/mod.rs @@ -1,23 +1,13 @@ #![allow(clippy::derive_partial_eq_without_eq, non_snake_case)] // https://github.com/hyperium/tonic/issues/1056 -use data_encoding::BASE64; -use std::{collections::HashSet, iter::Peekable}; -use thiserror::Error; - -use prost::Message; - use nix_compat::store_path::{self, StorePath}; +use thiserror::Error; +use tvix_castore::{proto as castorepb, B3Digest}; -mod grpc_blobservice_wrapper; -mod grpc_directoryservice_wrapper; mod grpc_pathinfoservice_wrapper; -pub use grpc_blobservice_wrapper::GRPCBlobServiceWrapper; -pub use grpc_directoryservice_wrapper::GRPCDirectoryServiceWrapper; pub use grpc_pathinfoservice_wrapper::GRPCPathInfoServiceWrapper; -use crate::B3Digest; - tonic::include_proto!("tvix.store.v1"); #[cfg(feature = "reflection")] @@ -29,23 +19,6 @@ pub const FILE_DESCRIPTOR_SET: &[u8] = tonic::include_file_descriptor_set!("tvix #[cfg(test)] mod tests; -/// Errors that can occur during the validation of Directory messages. -#[derive(Debug, PartialEq, Eq, Error)] -pub enum ValidateDirectoryError { - /// Elements are not in sorted order - #[error("{} is not sorted", std::str::from_utf8(.0).unwrap_or(&BASE64.encode(.0)))] - WrongSorting(Vec<u8>), - /// Multiple elements with the same name encountered - #[error("{0:?} is a duplicate name")] - DuplicateName(Vec<u8>), - /// Invalid name encountered - #[error("Invalid name in {0:?}")] - InvalidName(Vec<u8>), - /// Invalid digest length encountered - #[error("Invalid Digest length: {0}")] - InvalidDigestLen(usize), -} - /// Errors that can occur during the validation of PathInfo messages. #[derive(Debug, Error, PartialEq)] pub enum ValidatePathInfoError { @@ -67,31 +40,6 @@ pub enum ValidatePathInfoError { InconsistentNumberOfReferences(usize, usize), } -/// Checks a Node name for validity as an intermediate node, and returns an -/// error that's generated from the supplied constructor. -/// -/// We disallow slashes, null bytes, '.', '..' and the empty string. -fn validate_node_name<E>(name: &[u8], err: fn(Vec<u8>) -> E) -> Result<(), E> { - if name.is_empty() - || name == b".." - || name == b"." - || name.contains(&0x00) - || name.contains(&b'/') - { - return Err(err(name.to_vec())); - } - Ok(()) -} - -/// Checks a digest for validity. -/// Digests are 32 bytes long, as we store blake3 digests. -fn validate_digest<E>(digest: &bytes::Bytes, err: fn(usize) -> E) -> Result<(), E> { - if digest.len() != 32 { - return Err(err(digest.len())); - } - Ok(()) -} - /// Parses a root node name. /// /// On success, this returns the parsed [StorePath]. @@ -129,16 +77,17 @@ impl PathInfo { None => { return Err(ValidatePathInfoError::NoNodePresent()); } - Some(Node { node }) => match node { + Some(castorepb::Node { node }) => match node { None => { return Err(ValidatePathInfoError::NoNodePresent()); } - Some(node::Node::Directory(directory_node)) => { + Some(castorepb::node::Node::Directory(directory_node)) => { // ensure the digest has the appropriate size. - validate_digest( - &directory_node.digest, - ValidatePathInfoError::InvalidDigestLen, - )?; + if TryInto::<B3Digest>::try_into(directory_node.digest.clone()).is_err() { + return Err(ValidatePathInfoError::InvalidDigestLen( + directory_node.digest.len(), + )); + } // parse the name parse_node_name_root( @@ -146,14 +95,18 @@ impl PathInfo { ValidatePathInfoError::InvalidNodeName, )? } - Some(node::Node::File(file_node)) => { + Some(castorepb::node::Node::File(file_node)) => { // ensure the digest has the appropriate size. - validate_digest(&file_node.digest, ValidatePathInfoError::InvalidDigestLen)?; + if TryInto::<B3Digest>::try_into(file_node.digest.clone()).is_err() { + return Err(ValidatePathInfoError::InvalidDigestLen( + file_node.digest.len(), + )); + } // parse the name parse_node_name_root(&file_node.name, ValidatePathInfoError::InvalidNodeName)? } - Some(node::Node::Symlink(symlink_node)) => { + Some(castorepb::node::Node::Symlink(symlink_node)) => { // parse the name parse_node_name_root( &symlink_node.name, @@ -167,217 +120,3 @@ impl PathInfo { Ok(root_nix_path) } } - -/// NamedNode is implemented for [FileNode], [DirectoryNode] and [SymlinkNode] -/// and [node::Node], so we can ask all of them for the name easily. -pub trait NamedNode { - fn get_name(&self) -> &[u8]; -} - -impl NamedNode for &FileNode { - fn get_name(&self) -> &[u8] { - &self.name - } -} - -impl NamedNode for &DirectoryNode { - fn get_name(&self) -> &[u8] { - &self.name - } -} - -impl NamedNode for &SymlinkNode { - fn get_name(&self) -> &[u8] { - &self.name - } -} - -impl NamedNode for node::Node { - fn get_name(&self) -> &[u8] { - match self { - node::Node::File(node_file) => &node_file.name, - node::Node::Directory(node_directory) => &node_directory.name, - node::Node::Symlink(node_symlink) => &node_symlink.name, - } - } -} - -impl node::Node { - /// Returns the node with a new name. - pub fn rename(self, name: bytes::Bytes) -> Self { - match self { - node::Node::Directory(n) => node::Node::Directory(DirectoryNode { name, ..n }), - node::Node::File(n) => node::Node::File(FileNode { name, ..n }), - node::Node::Symlink(n) => node::Node::Symlink(SymlinkNode { name, ..n }), - } - } -} - -/// Accepts a name, and a mutable reference to the previous name. -/// If the passed name is larger than the previous one, the reference is updated. -/// If it's not, an error is returned. -fn update_if_lt_prev<'n>( - prev_name: &mut &'n [u8], - name: &'n [u8], -) -> Result<(), ValidateDirectoryError> { - if *name < **prev_name { - return Err(ValidateDirectoryError::WrongSorting(name.to_vec())); - } - *prev_name = name; - Ok(()) -} - -/// Inserts the given name into a HashSet if it's not already in there. -/// If it is, an error is returned. -fn insert_once<'n>( - seen_names: &mut HashSet<&'n [u8]>, - name: &'n [u8], -) -> Result<(), ValidateDirectoryError> { - if seen_names.get(name).is_some() { - return Err(ValidateDirectoryError::DuplicateName(name.to_vec())); - } - seen_names.insert(name); - Ok(()) -} - -impl Directory { - /// The size of a directory is the number of all regular and symlink elements, - /// the number of directory elements, and their size fields. - pub fn size(&self) -> u32 { - self.files.len() as u32 - + self.symlinks.len() as u32 - + self - .directories - .iter() - .fold(0, |acc: u32, e| (acc + 1 + e.size)) - } - - /// Calculates the digest of a Directory, which is the blake3 hash of a - /// Directory protobuf message, serialized in protobuf canonical form. - pub fn digest(&self) -> B3Digest { - let mut hasher = blake3::Hasher::new(); - - hasher - .update(&self.encode_to_vec()) - .finalize() - .as_bytes() - .into() - } - - /// validate checks the directory for invalid data, such as: - /// - violations of name restrictions - /// - invalid digest lengths - /// - not properly sorted lists - /// - duplicate names in the three lists - pub fn validate(&self) -> Result<(), ValidateDirectoryError> { - let mut seen_names: HashSet<&[u8]> = HashSet::new(); - - let mut last_directory_name: &[u8] = b""; - let mut last_file_name: &[u8] = b""; - let mut last_symlink_name: &[u8] = b""; - - // check directories - for directory_node in &self.directories { - validate_node_name(&directory_node.name, ValidateDirectoryError::InvalidName)?; - validate_digest( - &directory_node.digest, - ValidateDirectoryError::InvalidDigestLen, - )?; - - update_if_lt_prev(&mut last_directory_name, &directory_node.name)?; - insert_once(&mut seen_names, &directory_node.name)?; - } - - // check files - for file_node in &self.files { - validate_node_name(&file_node.name, ValidateDirectoryError::InvalidName)?; - validate_digest(&file_node.digest, ValidateDirectoryError::InvalidDigestLen)?; - - update_if_lt_prev(&mut last_file_name, &file_node.name)?; - insert_once(&mut seen_names, &file_node.name)?; - } - - // check symlinks - for symlink_node in &self.symlinks { - validate_node_name(&symlink_node.name, ValidateDirectoryError::InvalidName)?; - - update_if_lt_prev(&mut last_symlink_name, &symlink_node.name)?; - insert_once(&mut seen_names, &symlink_node.name)?; - } - - Ok(()) - } - - /// Allows iterating over all three nodes ([DirectoryNode], [FileNode], - /// [SymlinkNode]) in an ordered fashion, as long as the individual lists - /// are sorted (which can be checked by the [Directory::validate]). - pub fn nodes(&self) -> DirectoryNodesIterator { - return DirectoryNodesIterator { - i_directories: self.directories.iter().peekable(), - i_files: self.files.iter().peekable(), - i_symlinks: self.symlinks.iter().peekable(), - }; - } -} - -/// Struct to hold the state of an iterator over all nodes of a Directory. -/// -/// Internally, this keeps peekable Iterators over all three lists of a -/// Directory message. -pub struct DirectoryNodesIterator<'a> { - // directory: &Directory, - i_directories: Peekable<std::slice::Iter<'a, DirectoryNode>>, - i_files: Peekable<std::slice::Iter<'a, FileNode>>, - i_symlinks: Peekable<std::slice::Iter<'a, SymlinkNode>>, -} - -/// looks at two elements implementing NamedNode, and returns true if "left -/// is smaller / comes first". -/// -/// Some(_) is preferred over None. -fn left_name_lt_right<A: NamedNode, B: NamedNode>(left: Option<&A>, right: Option<&B>) -> bool { - match left { - // if left is None, right always wins - None => false, - Some(left_inner) => { - // left is Some. - match right { - // left is Some, right is None - left wins. - None => true, - Some(right_inner) => { - // both are Some - compare the name. - return left_inner.get_name() < right_inner.get_name(); - } - } - } - } -} - -impl Iterator for DirectoryNodesIterator<'_> { - type Item = node::Node; - - // next returns the next node in the Directory. - // we peek at all three internal iterators, and pick the one with the - // smallest name, to ensure lexicographical ordering. - // The individual lists are already known to be sorted. - fn next(&mut self) -> Option<Self::Item> { - if left_name_lt_right(self.i_directories.peek(), self.i_files.peek()) { - // i_directories is still in the game, compare with symlinks - if left_name_lt_right(self.i_directories.peek(), self.i_symlinks.peek()) { - self.i_directories - .next() - .cloned() - .map(node::Node::Directory) - } else { - self.i_symlinks.next().cloned().map(node::Node::Symlink) - } - } else { - // i_files is still in the game, compare with symlinks - if left_name_lt_right(self.i_files.peek(), self.i_symlinks.peek()) { - self.i_files.next().cloned().map(node::Node::File) - } else { - self.i_symlinks.next().cloned().map(node::Node::Symlink) - } - } - } -} diff --git a/tvix/store/src/proto/tests/grpc_pathinfoservice.rs b/tvix/store/src/proto/tests/grpc_pathinfoservice.rs index 114e89cacc10..c0b953d0f2e9 100644 --- a/tvix/store/src/proto/tests/grpc_pathinfoservice.rs +++ b/tvix/store/src/proto/tests/grpc_pathinfoservice.rs @@ -1,9 +1,8 @@ use crate::proto::get_path_info_request::ByWhat::ByOutputHash; -use crate::proto::node::Node::Symlink; use crate::proto::path_info_service_server::PathInfoService as GRPCPathInfoService; use crate::proto::GRPCPathInfoServiceWrapper; +use crate::proto::GetPathInfoRequest; use crate::proto::PathInfo; -use crate::proto::{GetPathInfoRequest, Node, SymlinkNode}; use crate::tests::fixtures::DUMMY_OUTPUT_HASH; use crate::tests::utils::gen_blob_service; use crate::tests::utils::gen_directory_service; @@ -11,6 +10,7 @@ use crate::tests::utils::gen_pathinfo_service; use std::sync::Arc; use tokio_stream::wrappers::ReceiverStream; use tonic::Request; +use tvix_castore::proto as castorepb; /// generates a GRPCPathInfoService out of blob, directory and pathinfo services. /// @@ -48,8 +48,8 @@ async fn put_get() { let service = gen_grpc_service(); let path_info = PathInfo { - node: Some(Node { - node: Some(Symlink(SymlinkNode { + node: Some(castorepb::Node { + node: Some(castorepb::node::Node::Symlink(castorepb::SymlinkNode { name: "00000000000000000000000000000000-foo".into(), target: "doesntmatter".into(), })), diff --git a/tvix/store/src/proto/tests/mod.rs b/tvix/store/src/proto/tests/mod.rs index 0a96ea3a0d59..bff885624380 100644 --- a/tvix/store/src/proto/tests/mod.rs +++ b/tvix/store/src/proto/tests/mod.rs @@ -1,6 +1,2 @@ -mod directory; -mod directory_nodes_iterator; -mod grpc_blobservice; -mod grpc_directoryservice; mod grpc_pathinfoservice; mod pathinfo; diff --git a/tvix/store/src/proto/tests/pathinfo.rs b/tvix/store/src/proto/tests/pathinfo.rs index 779b46ed168e..dfbeb831d7d2 100644 --- a/tvix/store/src/proto/tests/pathinfo.rs +++ b/tvix/store/src/proto/tests/pathinfo.rs @@ -1,31 +1,10 @@ -use crate::proto::{self, Node, PathInfo, ValidatePathInfoError}; -use crate::B3Digest; +use crate::proto::{NarInfo, PathInfo, ValidatePathInfoError}; +use crate::tests::fixtures::*; use bytes::Bytes; -use lazy_static::lazy_static; use nix_compat::store_path::{self, StorePath}; use std::str::FromStr; use test_case::test_case; - -lazy_static! { - static ref DUMMY_DIGEST: B3Digest = { - let u: &[u8; 32] = &[ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - ]; - u.into() - }; - static ref DUMMY_DIGEST_2: B3Digest = { - let u: &[u8; 32] = &[ - 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - ]; - u.into() - }; -} - -const DUMMY_NAME: &str = "00000000000000000000000000000000-dummy"; +use tvix_castore::proto as castorepb; #[test_case( None, @@ -33,12 +12,12 @@ const DUMMY_NAME: &str = "00000000000000000000000000000000-dummy"; "No node" )] #[test_case( - Some(Node { node: None }), + Some(castorepb::Node { node: None }), Err(ValidatePathInfoError::NoNodePresent()); "No node 2" )] fn validate_no_node( - t_node: Option<proto::Node>, + t_node: Option<castorepb::Node>, t_result: Result<StorePath, ValidatePathInfoError>, ) { // construct the PathInfo object @@ -50,7 +29,7 @@ fn validate_no_node( } #[test_case( - proto::DirectoryNode { + castorepb::DirectoryNode { name: DUMMY_NAME.into(), digest: DUMMY_DIGEST.clone().into(), size: 0, @@ -59,7 +38,7 @@ fn validate_no_node( "ok" )] #[test_case( - proto::DirectoryNode { + castorepb::DirectoryNode { name: DUMMY_NAME.into(), digest: Bytes::new(), size: 0, @@ -68,7 +47,7 @@ fn validate_no_node( "invalid digest length" )] #[test_case( - proto::DirectoryNode { + castorepb::DirectoryNode { name: "invalid".into(), digest: DUMMY_DIGEST.clone().into(), size: 0, @@ -80,13 +59,13 @@ fn validate_no_node( "invalid node name" )] fn validate_directory( - t_directory_node: proto::DirectoryNode, + t_directory_node: castorepb::DirectoryNode, t_result: Result<StorePath, ValidatePathInfoError>, ) { // construct the PathInfo object let p = PathInfo { - node: Some(Node { - node: Some(proto::node::Node::Directory(t_directory_node)), + node: Some(castorepb::Node { + node: Some(castorepb::node::Node::Directory(t_directory_node)), }), ..Default::default() }; @@ -94,7 +73,7 @@ fn validate_directory( } #[test_case( - proto::FileNode { + castorepb::FileNode { name: DUMMY_NAME.into(), digest: DUMMY_DIGEST.clone().into(), size: 0, @@ -104,7 +83,7 @@ fn validate_directory( "ok" )] #[test_case( - proto::FileNode { + castorepb::FileNode { name: DUMMY_NAME.into(), digest: Bytes::new(), ..Default::default() @@ -113,7 +92,7 @@ fn validate_directory( "invalid digest length" )] #[test_case( - proto::FileNode { + castorepb::FileNode { name: "invalid".into(), digest: DUMMY_DIGEST.clone().into(), ..Default::default() @@ -124,11 +103,14 @@ fn validate_directory( )); "invalid node name" )] -fn validate_file(t_file_node: proto::FileNode, t_result: Result<StorePath, ValidatePathInfoError>) { +fn validate_file( + t_file_node: castorepb::FileNode, + t_result: Result<StorePath, ValidatePathInfoError>, +) { // construct the PathInfo object let p = PathInfo { - node: Some(Node { - node: Some(proto::node::Node::File(t_file_node)), + node: Some(castorepb::Node { + node: Some(castorepb::node::Node::File(t_file_node)), }), ..Default::default() }; @@ -136,7 +118,7 @@ fn validate_file(t_file_node: proto::FileNode, t_result: Result<StorePath, Valid } #[test_case( - proto::SymlinkNode { + castorepb::SymlinkNode { name: DUMMY_NAME.into(), ..Default::default() }, @@ -144,7 +126,7 @@ fn validate_file(t_file_node: proto::FileNode, t_result: Result<StorePath, Valid "ok" )] #[test_case( - proto::SymlinkNode { + castorepb::SymlinkNode { name: "invalid".into(), ..Default::default() }, @@ -155,13 +137,13 @@ fn validate_file(t_file_node: proto::FileNode, t_result: Result<StorePath, Valid "invalid node name" )] fn validate_symlink( - t_symlink_node: proto::SymlinkNode, + t_symlink_node: castorepb::SymlinkNode, t_result: Result<StorePath, ValidatePathInfoError>, ) { // construct the PathInfo object let p = PathInfo { - node: Some(Node { - node: Some(proto::node::Node::Symlink(t_symlink_node)), + node: Some(castorepb::Node { + node: Some(castorepb::node::Node::Symlink(t_symlink_node)), }), ..Default::default() }; @@ -172,8 +154,8 @@ fn validate_symlink( fn validate_references() { // create a PathInfo without narinfo field. let path_info = PathInfo { - node: Some(Node { - node: Some(proto::node::Node::Directory(proto::DirectoryNode { + node: Some(castorepb::Node { + node: Some(castorepb::node::Node::Directory(castorepb::DirectoryNode { name: DUMMY_NAME.into(), digest: DUMMY_DIGEST.clone().into(), size: 0, @@ -186,7 +168,7 @@ fn validate_references() { // create a PathInfo with a narinfo field, but an inconsistent set of references let path_info_with_narinfo_missing_refs = PathInfo { - narinfo: Some(proto::NarInfo { + narinfo: Some(NarInfo { nar_size: 0, nar_sha256: DUMMY_DIGEST.clone().into(), signatures: vec![], @@ -204,7 +186,7 @@ fn validate_references() { // create a pathinfo with the correct number of references, should suceed let path_info_with_narinfo = PathInfo { - narinfo: Some(proto::NarInfo { + narinfo: Some(NarInfo { nar_size: 0, nar_sha256: DUMMY_DIGEST.clone().into(), signatures: vec![], diff --git a/tvix/store/src/tests/fixtures.rs b/tvix/store/src/tests/fixtures.rs index c362744a34a7..4d820af1578e 100644 --- a/tvix/store/src/tests/fixtures.rs +++ b/tvix/store/src/tests/fixtures.rs @@ -1,90 +1,9 @@ -use crate::{ - proto::{self, Directory, DirectoryNode, FileNode, SymlinkNode}, - B3Digest, -}; use lazy_static::lazy_static; +pub use tvix_castore::fixtures::*; -pub const HELLOWORLD_BLOB_CONTENTS: &[u8] = b"Hello World!"; -pub const EMPTY_BLOB_CONTENTS: &[u8] = b""; +pub const DUMMY_NAME: &str = "00000000000000000000000000000000-dummy"; lazy_static! { - pub static ref DUMMY_DIGEST: B3Digest = { - let u: &[u8; 32] = &[ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, - ]; - u.into() - }; - pub static ref DUMMY_DATA_1: bytes::Bytes = vec![0x01, 0x02, 0x03].into(); - pub static ref DUMMY_DATA_2: bytes::Bytes = vec![0x04, 0x05].into(); - - pub static ref HELLOWORLD_BLOB_DIGEST: B3Digest = - blake3::hash(HELLOWORLD_BLOB_CONTENTS).as_bytes().into(); - pub static ref EMPTY_BLOB_DIGEST: B3Digest = - blake3::hash(EMPTY_BLOB_CONTENTS).as_bytes().into(); - - // 2 bytes - pub static ref BLOB_A: bytes::Bytes = vec![0x00, 0x01].into(); - pub static ref BLOB_A_DIGEST: B3Digest = blake3::hash(&BLOB_A).as_bytes().into(); - - // 1MB - pub static ref BLOB_B: bytes::Bytes = (0..255).collect::<Vec<u8>>().repeat(4 * 1024).into(); - pub static ref BLOB_B_DIGEST: B3Digest = blake3::hash(&BLOB_B).as_bytes().into(); - - // Directories - pub static ref DIRECTORY_WITH_KEEP: proto::Directory = proto::Directory { - directories: vec![], - files: vec![FileNode { - name: b".keep".to_vec().into(), - digest: EMPTY_BLOB_DIGEST.clone().into(), - size: 0, - executable: false, - }], - symlinks: vec![], - }; - pub static ref DIRECTORY_COMPLICATED: proto::Directory = proto::Directory { - directories: vec![DirectoryNode { - name: b"keep".to_vec().into(), - digest: DIRECTORY_WITH_KEEP.digest().into(), - size: DIRECTORY_WITH_KEEP.size(), - }], - files: vec![FileNode { - name: b".keep".to_vec().into(), - digest: EMPTY_BLOB_DIGEST.clone().into(), - size: 0, - executable: false, - }], - symlinks: vec![SymlinkNode { - name: b"aa".to_vec().into(), - target: b"/nix/store/somewhereelse".to_vec().into(), - }], - }; - pub static ref DIRECTORY_A: Directory = Directory::default(); - pub static ref DIRECTORY_B: Directory = Directory { - directories: vec![DirectoryNode { - name: b"a".to_vec().into(), - digest: DIRECTORY_A.digest().into(), - size: DIRECTORY_A.size(), - }], - ..Default::default() - }; - pub static ref DIRECTORY_C: Directory = Directory { - directories: vec![ - DirectoryNode { - name: b"a".to_vec().into(), - digest: DIRECTORY_A.digest().into(), - size: DIRECTORY_A.size(), - }, - DirectoryNode { - name: b"a'".to_vec().into(), - digest: DIRECTORY_A.digest().into(), - size: DIRECTORY_A.size(), - } - ], - ..Default::default() - }; - // output hash pub static ref DUMMY_OUTPUT_HASH: bytes::Bytes = vec![ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, diff --git a/tvix/store/src/tests/mod.rs b/tvix/store/src/tests/mod.rs index 8ceea01e3190..daea048deddf 100644 --- a/tvix/store/src/tests/mod.rs +++ b/tvix/store/src/tests/mod.rs @@ -1,4 +1,3 @@ pub mod fixtures; -mod import; mod nar_renderer; pub mod utils; diff --git a/tvix/store/src/tests/nar_renderer.rs b/tvix/store/src/tests/nar_renderer.rs index e0163dc7bd93..485d7d115ff5 100644 --- a/tvix/store/src/tests/nar_renderer.rs +++ b/tvix/store/src/tests/nar_renderer.rs @@ -1,12 +1,12 @@ use crate::nar::calculate_size_and_sha256; use crate::nar::write_nar; -use crate::proto::DirectoryNode; -use crate::proto::FileNode; -use crate::proto::SymlinkNode; use crate::tests::fixtures::*; use crate::tests::utils::*; use sha2::{Digest, Sha256}; use std::io; +use tvix_castore::proto::DirectoryNode; +use tvix_castore::proto::FileNode; +use tvix_castore::proto::{self as castorepb, SymlinkNode}; #[tokio::test] async fn single_symlink() { @@ -14,7 +14,7 @@ async fn single_symlink() { let buf = write_nar( buf, - &crate::proto::node::Node::Symlink(SymlinkNode { + &castorepb::node::Node::Symlink(SymlinkNode { name: "doesntmatter".into(), target: "/nix/store/somewhereelse".into(), }), @@ -35,7 +35,7 @@ async fn single_file_missing_blob() { let e = write_nar( buf, - &crate::proto::node::Node::File(FileNode { + &castorepb::node::Node::File(FileNode { name: "doesntmatter".into(), digest: HELLOWORLD_BLOB_DIGEST.clone().into(), size: HELLOWORLD_BLOB_CONTENTS.len() as u32, @@ -82,7 +82,7 @@ async fn single_file_wrong_blob_size() { let e = write_nar( buf, - &crate::proto::node::Node::File(FileNode { + &castorepb::node::Node::File(FileNode { name: "doesntmatter".into(), digest: HELLOWORLD_BLOB_DIGEST.clone().into(), size: 42, // <- note the wrong size here! @@ -109,7 +109,7 @@ async fn single_file_wrong_blob_size() { let e = write_nar( buf, - &crate::proto::node::Node::File(FileNode { + &castorepb::node::Node::File(FileNode { name: "doesntmatter".into(), digest: HELLOWORLD_BLOB_DIGEST.clone().into(), size: 2, // <- note the wrong size here! @@ -152,7 +152,7 @@ async fn single_file() { let buf = write_nar( buf, - &crate::proto::node::Node::File(FileNode { + &castorepb::node::Node::File(FileNode { name: "doesntmatter".into(), digest: HELLOWORLD_BLOB_DIGEST.clone().into(), size: HELLOWORLD_BLOB_CONTENTS.len() as u32, @@ -199,7 +199,7 @@ async fn test_complicated() { let buf = write_nar( buf, - &crate::proto::node::Node::Directory(DirectoryNode { + &castorepb::node::Node::Directory(DirectoryNode { name: "doesntmatter".into(), digest: DIRECTORY_COMPLICATED.digest().into(), size: DIRECTORY_COMPLICATED.size(), @@ -216,7 +216,7 @@ async fn test_complicated() { let bs = blob_service.clone(); let ds = directory_service.clone(); let (nar_size, nar_digest) = calculate_size_and_sha256( - &crate::proto::node::Node::Directory(DirectoryNode { + &castorepb::node::Node::Directory(DirectoryNode { name: "doesntmatter".into(), digest: DIRECTORY_COMPLICATED.digest().into(), size: DIRECTORY_COMPLICATED.size(), diff --git a/tvix/store/src/tests/utils.rs b/tvix/store/src/tests/utils.rs index 9ccd3dcc65b7..961be6e7ac07 100644 --- a/tvix/store/src/tests/utils.rs +++ b/tvix/store/src/tests/utils.rs @@ -1,18 +1,8 @@ +use crate::pathinfoservice::{MemoryPathInfoService, PathInfoService}; use std::sync::Arc; +use tvix_castore::{blobservice::BlobService, directoryservice::DirectoryService}; -use crate::{ - blobservice::{BlobService, MemoryBlobService}, - directoryservice::{DirectoryService, MemoryDirectoryService}, - pathinfoservice::{MemoryPathInfoService, PathInfoService}, -}; - -pub fn gen_blob_service() -> Arc<dyn BlobService> { - Arc::new(MemoryBlobService::default()) -} - -pub fn gen_directory_service() -> Arc<dyn DirectoryService> { - Arc::new(MemoryDirectoryService::default()) -} +pub use tvix_castore::utils::*; pub fn gen_pathinfo_service( blob_service: Arc<dyn BlobService>, |