diff options
| -rw-r--r-- | tvix/store/protos/rpc_blobstore.pb.go | 371 | ||||
| -rw-r--r-- | tvix/store/protos/rpc_blobstore.proto | 58 | ||||
| -rw-r--r-- | tvix/store/protos/rpc_blobstore_grpc.pb.go | 93 |
3 files changed, 434 insertions, 88 deletions
diff --git a/tvix/store/protos/rpc_blobstore.pb.go b/tvix/store/protos/rpc_blobstore.pb.go index 6ea9fb03a5bb..426a2836dfae 100644 --- a/tvix/store/protos/rpc_blobstore.pb.go +++ b/tvix/store/protos/rpc_blobstore.pb.go @@ -23,17 +23,28 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -type GetBlobRequest struct { +type StatBlobRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields // The blake3 digest of the blob requested Digest []byte `protobuf:"bytes,1,opt,name=digest,proto3" json:"digest,omitempty"` + // Whether to include the chunks field + IncludeChunks bool `protobuf:"varint,2,opt,name=include_chunks,json=includeChunks,proto3" json:"include_chunks,omitempty"` + // Whether to include the inline_bao field, containing an (outboard) bao. + // The [bao](https://github.com/oconnor663/bao/blob/master/docs/spec.md) + // can be used to validate chunks end up hashing to the same root digest. + // These only really matter when only downloading parts of a blob. Some + // caution needs to be applied when validating chunks - the bao works with + // 1K leaf nodes, which might not align with the chunk sizes - this might + // imply a neighboring chunk might need to be (partially) fetched to + // validate the hash. + IncludeBao bool `protobuf:"varint,3,opt,name=include_bao,json=includeBao,proto3" json:"include_bao,omitempty"` } -func (x *GetBlobRequest) Reset() { - *x = GetBlobRequest{} +func (x *StatBlobRequest) Reset() { + *x = StatBlobRequest{} if protoimpl.UnsafeEnabled { mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -41,13 +52,13 @@ func (x *GetBlobRequest) Reset() { } } -func (x *GetBlobRequest) String() string { +func (x *StatBlobRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetBlobRequest) ProtoMessage() {} +func (*StatBlobRequest) ProtoMessage() {} -func (x *GetBlobRequest) ProtoReflect() protoreflect.Message { +func (x *StatBlobRequest) ProtoReflect() protoreflect.Message { mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -59,29 +70,48 @@ func (x *GetBlobRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetBlobRequest.ProtoReflect.Descriptor instead. -func (*GetBlobRequest) Descriptor() ([]byte, []int) { +// Deprecated: Use StatBlobRequest.ProtoReflect.Descriptor instead. +func (*StatBlobRequest) Descriptor() ([]byte, []int) { return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{0} } -func (x *GetBlobRequest) GetDigest() []byte { +func (x *StatBlobRequest) GetDigest() []byte { if x != nil { return x.Digest } return nil } -type PutBlobResponse struct { +func (x *StatBlobRequest) GetIncludeChunks() bool { + if x != nil { + return x.IncludeChunks + } + return false +} + +func (x *StatBlobRequest) GetIncludeBao() bool { + if x != nil { + return x.IncludeBao + } + return false +} + +// BlobMeta provides more granular chunking information for the requested blob, +// and baos. +type BlobMeta struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // The blake3 digest of the data that was sent. - Digest []byte `protobuf:"bytes,1,opt,name=digest,proto3" json:"digest,omitempty"` + // This provides a list of chunks. + // Concatenating their contents would produce a blob with the digest that + // was specified in the request. + Chunks []*BlobMeta_ChunkMeta `protobuf:"bytes,1,rep,name=chunks,proto3" json:"chunks,omitempty"` + InlineBao []byte `protobuf:"bytes,2,opt,name=inline_bao,json=inlineBao,proto3" json:"inline_bao,omitempty"` } -func (x *PutBlobResponse) Reset() { - *x = PutBlobResponse{} +func (x *BlobMeta) Reset() { + *x = BlobMeta{} if protoimpl.UnsafeEnabled { mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -89,13 +119,13 @@ func (x *PutBlobResponse) Reset() { } } -func (x *PutBlobResponse) String() string { +func (x *BlobMeta) String() string { return protoimpl.X.MessageStringOf(x) } -func (*PutBlobResponse) ProtoMessage() {} +func (*BlobMeta) ProtoMessage() {} -func (x *PutBlobResponse) ProtoReflect() protoreflect.Message { +func (x *BlobMeta) ProtoReflect() protoreflect.Message { mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -107,19 +137,74 @@ func (x *PutBlobResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use PutBlobResponse.ProtoReflect.Descriptor instead. -func (*PutBlobResponse) Descriptor() ([]byte, []int) { +// Deprecated: Use BlobMeta.ProtoReflect.Descriptor instead. +func (*BlobMeta) Descriptor() ([]byte, []int) { return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{1} } -func (x *PutBlobResponse) GetDigest() []byte { +func (x *BlobMeta) GetChunks() []*BlobMeta_ChunkMeta { + if x != nil { + return x.Chunks + } + return nil +} + +func (x *BlobMeta) GetInlineBao() []byte { + if x != nil { + return x.InlineBao + } + return nil +} + +type ReadBlobRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The blake3 digest of the blob requested + Digest []byte `protobuf:"bytes,1,opt,name=digest,proto3" json:"digest,omitempty"` +} + +func (x *ReadBlobRequest) Reset() { + *x = ReadBlobRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ReadBlobRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReadBlobRequest) ProtoMessage() {} + +func (x *ReadBlobRequest) ProtoReflect() protoreflect.Message { + mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReadBlobRequest.ProtoReflect.Descriptor instead. +func (*ReadBlobRequest) Descriptor() ([]byte, []int) { + return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{2} +} + +func (x *ReadBlobRequest) GetDigest() []byte { if x != nil { return x.Digest } return nil } -// This represents a part of a chunk. +// This represents some bytes of a blob. // Blobs are sent in smaller chunks to keep message sizes manageable. type BlobChunk struct { state protoimpl.MessageState @@ -132,7 +217,7 @@ type BlobChunk struct { func (x *BlobChunk) Reset() { *x = BlobChunk{} if protoimpl.UnsafeEnabled { - mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2] + mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -145,7 +230,7 @@ func (x *BlobChunk) String() string { func (*BlobChunk) ProtoMessage() {} func (x *BlobChunk) ProtoReflect() protoreflect.Message { - mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2] + mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -158,7 +243,7 @@ func (x *BlobChunk) ProtoReflect() protoreflect.Message { // Deprecated: Use BlobChunk.ProtoReflect.Descriptor instead. func (*BlobChunk) Descriptor() ([]byte, []int) { - return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{2} + return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{3} } func (x *BlobChunk) GetData() []byte { @@ -168,33 +253,157 @@ func (x *BlobChunk) GetData() []byte { return nil } +type PutBlobResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The blake3 digest of the data that was sent. + Digest []byte `protobuf:"bytes,1,opt,name=digest,proto3" json:"digest,omitempty"` +} + +func (x *PutBlobResponse) Reset() { + *x = PutBlobResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PutBlobResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PutBlobResponse) ProtoMessage() {} + +func (x *PutBlobResponse) ProtoReflect() protoreflect.Message { + mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PutBlobResponse.ProtoReflect.Descriptor instead. +func (*PutBlobResponse) Descriptor() ([]byte, []int) { + return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{4} +} + +func (x *PutBlobResponse) GetDigest() []byte { + if x != nil { + return x.Digest + } + return nil +} + +type BlobMeta_ChunkMeta struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Digest []byte `protobuf:"bytes,1,opt,name=digest,proto3" json:"digest,omitempty"` + Size uint32 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` +} + +func (x *BlobMeta_ChunkMeta) Reset() { + *x = BlobMeta_ChunkMeta{} + if protoimpl.UnsafeEnabled { + mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BlobMeta_ChunkMeta) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BlobMeta_ChunkMeta) ProtoMessage() {} + +func (x *BlobMeta_ChunkMeta) ProtoReflect() protoreflect.Message { + mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BlobMeta_ChunkMeta.ProtoReflect.Descriptor instead. +func (*BlobMeta_ChunkMeta) Descriptor() ([]byte, []int) { + return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{1, 0} +} + +func (x *BlobMeta_ChunkMeta) GetDigest() []byte { + if x != nil { + return x.Digest + } + return nil +} + +func (x *BlobMeta_ChunkMeta) GetSize() uint32 { + if x != nil { + return x.Size + } + return 0 +} + var File_tvix_store_protos_rpc_blobstore_proto protoreflect.FileDescriptor var file_tvix_store_protos_rpc_blobstore_proto_rawDesc = []byte{ 0x0a, 0x25, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x5f, 0x62, 0x6c, 0x6f, 0x62, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x22, 0x28, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, - 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, + 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x22, 0x71, 0x0a, 0x0f, 0x53, 0x74, 0x61, 0x74, 0x42, 0x6c, + 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, + 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, + 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x63, 0x68, 0x75, + 0x6e, 0x6b, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x69, 0x6e, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x63, 0x6c, + 0x75, 0x64, 0x65, 0x5f, 0x62, 0x61, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x69, + 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x42, 0x61, 0x6f, 0x22, 0x9d, 0x01, 0x0a, 0x08, 0x42, 0x6c, + 0x6f, 0x62, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x39, 0x0a, 0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, + 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x4d, 0x65, 0x74, 0x61, 0x2e, + 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b, + 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x62, 0x61, 0x6f, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x69, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x42, 0x61, 0x6f, + 0x1a, 0x37, 0x0a, 0x09, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x16, 0x0a, + 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, + 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x22, 0x29, 0x0a, 0x0f, 0x52, 0x65, 0x61, + 0x64, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, + 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, + 0x67, 0x65, 0x73, 0x74, 0x22, 0x1f, 0x0a, 0x09, 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e, + 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x29, 0x0a, 0x0f, 0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, - 0x22, 0x29, 0x0a, 0x0f, 0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0x1f, 0x0a, 0x09, 0x42, - 0x6c, 0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x32, 0x92, 0x01, 0x0a, - 0x0b, 0x42, 0x6c, 0x6f, 0x62, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x40, 0x0a, 0x03, - 0x47, 0x65, 0x74, 0x12, 0x1d, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x30, 0x01, 0x12, 0x41, - 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x18, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, - 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x1a, - 0x1e, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, - 0x01, 0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, - 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x32, 0xd5, 0x01, 0x0a, 0x0b, 0x42, 0x6c, 0x6f, 0x62, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x3f, 0x0a, 0x04, 0x53, 0x74, 0x61, 0x74, 0x12, 0x1e, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, + 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x42, 0x6c, 0x6f, + 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, + 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x4d, 0x65, 0x74, + 0x61, 0x12, 0x42, 0x0a, 0x04, 0x52, 0x65, 0x61, 0x64, 0x12, 0x1e, 0x2e, 0x74, 0x76, 0x69, 0x78, + 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x42, 0x6c, + 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x74, 0x76, 0x69, 0x78, + 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x68, + 0x75, 0x6e, 0x6b, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x18, 0x2e, 0x74, + 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, + 0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x1a, 0x1e, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, + 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, + 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, + 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, + 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -209,22 +418,28 @@ func file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP() []byte { return file_tvix_store_protos_rpc_blobstore_proto_rawDescData } -var file_tvix_store_protos_rpc_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_tvix_store_protos_rpc_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_tvix_store_protos_rpc_blobstore_proto_goTypes = []interface{}{ - (*GetBlobRequest)(nil), // 0: tvix.store.v1.GetBlobRequest - (*PutBlobResponse)(nil), // 1: tvix.store.v1.PutBlobResponse - (*BlobChunk)(nil), // 2: tvix.store.v1.BlobChunk + (*StatBlobRequest)(nil), // 0: tvix.store.v1.StatBlobRequest + (*BlobMeta)(nil), // 1: tvix.store.v1.BlobMeta + (*ReadBlobRequest)(nil), // 2: tvix.store.v1.ReadBlobRequest + (*BlobChunk)(nil), // 3: tvix.store.v1.BlobChunk + (*PutBlobResponse)(nil), // 4: tvix.store.v1.PutBlobResponse + (*BlobMeta_ChunkMeta)(nil), // 5: tvix.store.v1.BlobMeta.ChunkMeta } var file_tvix_store_protos_rpc_blobstore_proto_depIdxs = []int32{ - 0, // 0: tvix.store.v1.BlobService.Get:input_type -> tvix.store.v1.GetBlobRequest - 2, // 1: tvix.store.v1.BlobService.Put:input_type -> tvix.store.v1.BlobChunk - 2, // 2: tvix.store.v1.BlobService.Get:output_type -> tvix.store.v1.BlobChunk - 1, // 3: tvix.store.v1.BlobService.Put:output_type -> tvix.store.v1.PutBlobResponse - 2, // [2:4] is the sub-list for method output_type - 0, // [0:2] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name + 5, // 0: tvix.store.v1.BlobMeta.chunks:type_name -> tvix.store.v1.BlobMeta.ChunkMeta + 0, // 1: tvix.store.v1.BlobService.Stat:input_type -> tvix.store.v1.StatBlobRequest + 2, // 2: tvix.store.v1.BlobService.Read:input_type -> tvix.store.v1.ReadBlobRequest + 3, // 3: tvix.store.v1.BlobService.Put:input_type -> tvix.store.v1.BlobChunk + 1, // 4: tvix.store.v1.BlobService.Stat:output_type -> tvix.store.v1.BlobMeta + 3, // 5: tvix.store.v1.BlobService.Read:output_type -> tvix.store.v1.BlobChunk + 4, // 6: tvix.store.v1.BlobService.Put:output_type -> tvix.store.v1.PutBlobResponse + 4, // [4:7] is the sub-list for method output_type + 1, // [1:4] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name } func init() { file_tvix_store_protos_rpc_blobstore_proto_init() } @@ -234,7 +449,7 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() { } if !protoimpl.UnsafeEnabled { file_tvix_store_protos_rpc_blobstore_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetBlobRequest); i { + switch v := v.(*StatBlobRequest); i { case 0: return &v.state case 1: @@ -246,7 +461,7 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() { } } file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutBlobResponse); i { + switch v := v.(*BlobMeta); i { case 0: return &v.state case 1: @@ -258,6 +473,18 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() { } } file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ReadBlobRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_tvix_store_protos_rpc_blobstore_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*BlobChunk); i { case 0: return &v.state @@ -269,6 +496,30 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() { return nil } } + file_tvix_store_protos_rpc_blobstore_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutBlobResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_tvix_store_protos_rpc_blobstore_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BlobMeta_ChunkMeta); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -276,7 +527,7 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_tvix_store_protos_rpc_blobstore_proto_rawDesc, NumEnums: 0, - NumMessages: 3, + NumMessages: 6, NumExtensions: 0, NumServices: 1, }, diff --git a/tvix/store/protos/rpc_blobstore.proto b/tvix/store/protos/rpc_blobstore.proto index cca195c3d975..e0b700f2d0da 100644 --- a/tvix/store/protos/rpc_blobstore.proto +++ b/tvix/store/protos/rpc_blobstore.proto @@ -7,27 +7,67 @@ package tvix.store.v1; option go_package = "code.tvl.fyi/tvix/store/protos;storev1"; service BlobService { - rpc Get(GetBlobRequest) returns (stream BlobChunk); + // Stat exposes metadata about a given blob, + // such as more granular chunking, baos. + // It implicitly allows checking for existence too, as asking this for a + // non-existing Blob will return a Status::not_found grpc error. + // If there's no more granular chunking available, the response will simply + // contain a single chunk. + rpc Stat(StatBlobRequest) returns (BlobMeta); - rpc Put(stream BlobChunk) returns (PutBlobResponse); + // Read returns a stream of BlobChunk, which is just a stream of bytes - not necessarily + // using the chunking that's returned in the reply of a Stat() call. + rpc Read(ReadBlobRequest) returns (stream BlobChunk); - // TODO(flokli): We can get fancy here, and add methods to retrieve - // [Bao](https://github.com/oconnor663/bao/blob/master/docs/spec.md), and - // then support range requests, but that's left for later. + // Put uploads a Blob, by reading a stream of bytes. + rpc Put(stream BlobChunk) returns (PutBlobResponse); } -message GetBlobRequest { +message StatBlobRequest { // The blake3 digest of the blob requested bytes digest = 1; + + // Whether to include the chunks field + bool include_chunks = 2; + // Whether to include the inline_bao field, containing an (outboard) bao. + // The [bao](https://github.com/oconnor663/bao/blob/master/docs/spec.md) + // can be used to validate chunks end up hashing to the same root digest. + // These only really matter when only downloading parts of a blob. Some + // caution needs to be applied when validating chunks - the bao works with + // 1K leaf nodes, which might not align with the chunk sizes - this might + // imply a neighboring chunk might need to be (partially) fetched to + // validate the hash. + bool include_bao = 3; } -message PutBlobResponse { - // The blake3 digest of the data that was sent. +// BlobMeta provides more granular chunking information for the requested blob, +// and baos. +message BlobMeta { + // This provides a list of chunks. + // Concatenating their contents would produce a blob with the digest that + // was specified in the request. + repeated ChunkMeta chunks = 1; + + message ChunkMeta { + bytes digest = 1; + uint32 size = 2; + } + + bytes inline_bao = 2; +} + +message ReadBlobRequest { + // The blake3 digest of the blob requested bytes digest = 1; } -// This represents a part of a chunk. +// This represents some bytes of a blob. // Blobs are sent in smaller chunks to keep message sizes manageable. message BlobChunk { bytes data = 1; } + +message PutBlobResponse { + // The blake3 digest of the data that was sent. + bytes digest = 1; +} diff --git a/tvix/store/protos/rpc_blobstore_grpc.pb.go b/tvix/store/protos/rpc_blobstore_grpc.pb.go index f99181cb4d06..f952e78d5a6e 100644 --- a/tvix/store/protos/rpc_blobstore_grpc.pb.go +++ b/tvix/store/protos/rpc_blobstore_grpc.pb.go @@ -22,7 +22,17 @@ const _ = grpc.SupportPackageIsVersion7 // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type BlobServiceClient interface { - Get(ctx context.Context, in *GetBlobRequest, opts ...grpc.CallOption) (BlobService_GetClient, error) + // Stat exposes metadata about a given blob, + // such as more granular chunking, baos. + // It implicitly allows checking for existence too, as asking this for a + // non-existing Blob will return a Status::not_found grpc error. + // If there's no more granular chunking available, the response will simply + // contain a single chunk. + Stat(ctx context.Context, in *StatBlobRequest, opts ...grpc.CallOption) (*BlobMeta, error) + // Read returns a stream of BlobChunk, which is just a stream of bytes - not necessarily + // using the chunking that's returned in the reply of a Stat() call. + Read(ctx context.Context, in *ReadBlobRequest, opts ...grpc.CallOption) (BlobService_ReadClient, error) + // Put uploads a Blob, by reading a stream of bytes. Put(ctx context.Context, opts ...grpc.CallOption) (BlobService_PutClient, error) } @@ -34,12 +44,21 @@ func NewBlobServiceClient(cc grpc.ClientConnInterface) BlobServiceClient { return &blobServiceClient{cc} } -func (c *blobServiceClient) Get(ctx context.Context, in *GetBlobRequest, opts ...grpc.CallOption) (BlobService_GetClient, error) { - stream, err := c.cc.NewStream(ctx, &BlobService_ServiceDesc.Streams[0], "/tvix.store.v1.BlobService/Get", opts...) +func (c *blobServiceClient) Stat(ctx context.Context, in *StatBlobRequest, opts ...grpc.CallOption) (*BlobMeta, error) { + out := new(BlobMeta) + err := c.cc.Invoke(ctx, "/tvix.store.v1.BlobService/Stat", in, out, opts...) if err != nil { return nil, err } - x := &blobServiceGetClient{stream} + return out, nil +} + +func (c *blobServiceClient) Read(ctx context.Context, in *ReadBlobRequest, opts ...grpc.CallOption) (BlobService_ReadClient, error) { + stream, err := c.cc.NewStream(ctx, &BlobService_ServiceDesc.Streams[0], "/tvix.store.v1.BlobService/Read", opts...) + if err != nil { + return nil, err + } + x := &blobServiceReadClient{stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -49,16 +68,16 @@ func (c *blobServiceClient) Get(ctx context.Context, in *GetBlobRequest, opts .. return x, nil } -type BlobService_GetClient interface { +type BlobService_ReadClient interface { Recv() (*BlobChunk, error) grpc.ClientStream } -type blobServiceGetClient struct { +type blobServiceReadClient struct { grpc.ClientStream } -func (x *blobServiceGetClient) Recv() (*BlobChunk, error) { +func (x *blobServiceReadClient) Recv() (*BlobChunk, error) { m := new(BlobChunk) if err := x.ClientStream.RecvMsg(m); err != nil { return nil, err @@ -104,7 +123,17 @@ func (x *blobServicePutClient) CloseAndRecv() (*PutBlobResponse, error) { // All implementations must embed UnimplementedBlobServiceServer // for forward compatibility type BlobServiceServer interface { - Get(*GetBlobRequest, BlobService_GetServer) error + // Stat exposes metadata about a given blob, + // such as more granular chunking, baos. + // It implicitly allows checking for existence too, as asking this for a + // non-existing Blob will return a Status::not_found grpc error. + // If there's no more granular chunking available, the response will simply + // contain a single chunk. + Stat(context.Context, *StatBlobRequest) (*BlobMeta, error) + // Read returns a stream of BlobChunk, which is just a stream of bytes - not necessarily + // using the chunking that's returned in the reply of a Stat() call. + Read(*ReadBlobRequest, BlobService_ReadServer) error + // Put uploads a Blob, by reading a stream of bytes. Put(BlobService_PutServer) error mustEmbedUnimplementedBlobServiceServer() } @@ -113,8 +142,11 @@ type BlobServiceServer interface { type UnimplementedBlobServiceServer struct { } -func (UnimplementedBlobServiceServer) Get(*GetBlobRequest, BlobService_GetServer) error { - return status.Errorf(codes.Unimplemented, "method Get not implemented") +func (UnimplementedBlobServiceServer) Stat(context.Context, *StatBlobRequest) (*BlobMeta, error) { + return nil, status.Errorf(codes.Unimplemented, "method Stat not implemented") +} +func (UnimplementedBlobServiceServer) Read(*ReadBlobRequest, BlobService_ReadServer) error { + return status.Errorf(codes.Unimplemented, "method Read not implemented") } func (UnimplementedBlobServiceServer) Put(BlobService_PutServer) error { return status.Errorf(codes.Unimplemented, "method Put not implemented") @@ -132,24 +164,42 @@ func RegisterBlobServiceServer(s grpc.ServiceRegistrar, srv BlobServiceServer) { s.RegisterService(&BlobService_ServiceDesc, srv) } -func _BlobService_Get_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(GetBlobRequest) +func _BlobService_Stat_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(StatBlobRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlobServiceServer).Stat(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/tvix.store.v1.BlobService/Stat", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlobServiceServer).Stat(ctx, req.(*StatBlobRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _BlobService_Read_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(ReadBlobRequest) if err := stream.RecvMsg(m); err != nil { return err } - return srv.(BlobServiceServer).Get(m, &blobServiceGetServer{stream}) + return srv.(BlobServiceServer).Read(m, &blobServiceReadServer{stream}) } -type BlobService_GetServer interface { +type BlobService_ReadServer interface { Send(*BlobChunk) error grpc.ServerStream } -type blobServiceGetServer struct { +type blobServiceReadServer struct { grpc.ServerStream } -func (x *blobServiceGetServer) Send(m *BlobChunk) error { +func (x *blobServiceReadServer) Send(m *BlobChunk) error { return x.ServerStream.SendMsg(m) } @@ -185,11 +235,16 @@ func (x *blobServicePutServer) Recv() (*BlobChunk, error) { var BlobService_ServiceDesc = grpc.ServiceDesc{ ServiceName: "tvix.store.v1.BlobService", HandlerType: (*BlobServiceServer)(nil), - Methods: []grpc.MethodDesc{}, + Methods: []grpc.MethodDesc{ + { + MethodName: "Stat", + Handler: _BlobService_Stat_Handler, + }, + }, Streams: []grpc.StreamDesc{ { - StreamName: "Get", - Handler: _BlobService_Get_Handler, + StreamName: "Read", + Handler: _BlobService_Read_Handler, ServerStreams: true, }, { |