From 1c15154b834fd529b95b3026c867e34ad98787f0 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Tue, 27 Dec 2022 18:32:07 +0100 Subject: feat(tvix/store): make blobstore stream chunks This changes the RPC methods to return/consume a stream of chunks, instead of a very big message containing the whole blob, to keep message sizes in manageable sizes (less than 4MiB). Change-Id: I2a3a50f07b059d8a2f5196860254adff98c8a352 Reviewed-on: https://cl.tvl.fyi/c/depot/+/7651 Reviewed-by: tazjin Tested-by: BuildkiteCI --- tvix/store/protos/rpc_blobstore.proto | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) (limited to 'tvix/store/protos/rpc_blobstore.proto') diff --git a/tvix/store/protos/rpc_blobstore.proto b/tvix/store/protos/rpc_blobstore.proto index 1d4349bc6bb6..cca195c3d975 100644 --- a/tvix/store/protos/rpc_blobstore.proto +++ b/tvix/store/protos/rpc_blobstore.proto @@ -7,8 +7,9 @@ package tvix.store.v1; option go_package = "code.tvl.fyi/tvix/store/protos;storev1"; service BlobService { - rpc Get(GetBlobRequest) returns (GetBlobResponse); - rpc Put(PutBlobRequest) returns (PutBlobResponse); + rpc Get(GetBlobRequest) returns (stream BlobChunk); + + rpc Put(stream BlobChunk) returns (PutBlobResponse); // TODO(flokli): We can get fancy here, and add methods to retrieve // [Bao](https://github.com/oconnor663/bao/blob/master/docs/spec.md), and @@ -20,15 +21,13 @@ message GetBlobRequest { bytes digest = 1; } -message GetBlobResponse { - bytes data = 1; -} - -message PutBlobRequest { - bytes data = 1; -} - message PutBlobResponse { // The blake3 digest of the data that was sent. bytes digest = 1; } + +// This represents a part of a chunk. +// Blobs are sent in smaller chunks to keep message sizes manageable. +message BlobChunk { + bytes data = 1; +} -- cgit 1.4.1