From b38badf2063b4eba31abffbeba01c1c8c3212be8 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Tue, 20 Feb 2024 15:35:55 +0700 Subject: docs(tvix/castore/directorysvc): K/V is not necessarily flat Some implementations of DirectoryService might not allow retrieval of intermediate Directory nodes, that are not at the "root". Think about an object store implementation. The client is doing a get_recursive anyways to reduce the number of roundtrips. By documenting the fact we don't need to support looking up intermediate Directory messages, we can just batch all directories into the same object, keyed by the root. Change-Id: I019d720186d03c4125cec9191e93d20586a20963 Reviewed-on: https://cl.tvl.fyi/c/depot/+/10988 Autosubmit: flokli Reviewed-by: tazjin Tested-by: BuildkiteCI --- tvix/castore-go/rpc_directory_grpc.pb.go | 10 ++++++++++ tvix/castore/protos/rpc_directory.proto | 5 +++++ tvix/castore/src/directoryservice/mod.rs | 5 +++++ 3 files changed, 20 insertions(+) (limited to 'tvix') diff --git a/tvix/castore-go/rpc_directory_grpc.pb.go b/tvix/castore-go/rpc_directory_grpc.pb.go index f19e457d867b..98789fef833e 100644 --- a/tvix/castore-go/rpc_directory_grpc.pb.go +++ b/tvix/castore-go/rpc_directory_grpc.pb.go @@ -35,6 +35,11 @@ type DirectoryServiceClient interface { // Keep in mind multiple DirectoryNodes in different parts of the graph might // have the same digest if they have the same underlying contents, // so sending subsequent ones can be omitted. + // + // It is okay for certain implementations to only allow retrieval of + // Directory digests that are at the "root", aka the last element that's + // sent in a Put. This makes sense for implementations bundling closures of + // directories together in batches. Get(ctx context.Context, in *GetDirectoryRequest, opts ...grpc.CallOption) (DirectoryService_GetClient, error) // Put uploads a graph of Directory messages. // Individual Directory messages need to be send in an order walking up @@ -131,6 +136,11 @@ type DirectoryServiceServer interface { // Keep in mind multiple DirectoryNodes in different parts of the graph might // have the same digest if they have the same underlying contents, // so sending subsequent ones can be omitted. + // + // It is okay for certain implementations to only allow retrieval of + // Directory digests that are at the "root", aka the last element that's + // sent in a Put. This makes sense for implementations bundling closures of + // directories together in batches. Get(*GetDirectoryRequest, DirectoryService_GetServer) error // Put uploads a graph of Directory messages. // Individual Directory messages need to be send in an order walking up diff --git a/tvix/castore/protos/rpc_directory.proto b/tvix/castore/protos/rpc_directory.proto index 7fccb6c75e34..f4f41c433a76 100644 --- a/tvix/castore/protos/rpc_directory.proto +++ b/tvix/castore/protos/rpc_directory.proto @@ -14,6 +14,11 @@ service DirectoryService { // Keep in mind multiple DirectoryNodes in different parts of the graph might // have the same digest if they have the same underlying contents, // so sending subsequent ones can be omitted. + // + // It is okay for certain implementations to only allow retrieval of + // Directory digests that are at the "root", aka the last element that's + // sent in a Put. This makes sense for implementations bundling closures of + // directories together in batches. rpc Get(GetDirectoryRequest) returns (stream Directory); // Put uploads a graph of Directory messages. diff --git a/tvix/castore/src/directoryservice/mod.rs b/tvix/castore/src/directoryservice/mod.rs index db3d5767eadd..8a10121d6d88 100644 --- a/tvix/castore/src/directoryservice/mod.rs +++ b/tvix/castore/src/directoryservice/mod.rs @@ -23,6 +23,11 @@ pub trait DirectoryService: Send + Sync { /// Looks up a single Directory message by its digest. /// The returned Directory message *must* be valid. /// In case the directory is not found, Ok(None) is returned. + /// + /// It is okay for certain implementations to only allow retrieval of + /// Directory digests that are at the "root", aka the last element that's + /// sent to a DirectoryPutter. This makes sense for implementations bundling + /// closures of directories together in batches. async fn get(&self, digest: &B3Digest) -> Result, Error>; /// Uploads a single Directory message, and returns the calculated /// digest, or an error. An error *must* also be returned if the message is -- cgit 1.4.1