From 72e82ffcb11b1aaf1f1cc8db4189ced5ec0aa42e Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Tue, 18 Jul 2023 19:37:25 +0300 Subject: refactor(tvix/store): use bytes for node names and symlink targets Some paths might use names that are not valid UTF-8. We should be able to represent them. We don't actually need to touch the PathInfo structures, as they need to represent StorePaths, which come with their own harder restrictions, which can't encode non-UTF8 data. While this doesn't change any of the wire format of the gRPC messages, it does however change the interface of tvix_eval::EvalIO - its read_dir() method does now return a list of Vec, rather than SmolStr. Maybe this should be OsString instead? Change-Id: I821016d9a58ec441ee081b0b9f01c9240723af0b Reviewed-on: https://cl.tvl.fyi/c/depot/+/8974 Autosubmit: flokli Reviewed-by: raitobezarius Tested-by: BuildkiteCI --- tvix/store/protos/castore.pb.go | 32 ++++++++++++++++---------------- tvix/store/protos/castore.proto | 8 ++++---- 2 files changed, 20 insertions(+), 20 deletions(-) (limited to 'tvix/store/protos') diff --git a/tvix/store/protos/castore.pb.go b/tvix/store/protos/castore.pb.go index e96f115d5cf9..074b39d548e3 100644 --- a/tvix/store/protos/castore.pb.go +++ b/tvix/store/protos/castore.pb.go @@ -103,7 +103,7 @@ type DirectoryNode struct { unknownFields protoimpl.UnknownFields // The (base)name of the directory - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // The blake3 hash of a Directory message, serialized in protobuf canonical form. Digest []byte `protobuf:"bytes,2,opt,name=digest,proto3" json:"digest,omitempty"` // Number of child elements in the Directory referred to by `digest`. @@ -151,11 +151,11 @@ func (*DirectoryNode) Descriptor() ([]byte, []int) { return file_tvix_store_protos_castore_proto_rawDescGZIP(), []int{1} } -func (x *DirectoryNode) GetName() string { +func (x *DirectoryNode) GetName() []byte { if x != nil { return x.Name } - return "" + return nil } func (x *DirectoryNode) GetDigest() []byte { @@ -179,7 +179,7 @@ type FileNode struct { unknownFields protoimpl.UnknownFields // The (base)name of the file - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // The blake3 digest of the file contents Digest []byte `protobuf:"bytes,2,opt,name=digest,proto3" json:"digest,omitempty"` // The file content size @@ -220,11 +220,11 @@ func (*FileNode) Descriptor() ([]byte, []int) { return file_tvix_store_protos_castore_proto_rawDescGZIP(), []int{2} } -func (x *FileNode) GetName() string { +func (x *FileNode) GetName() []byte { if x != nil { return x.Name } - return "" + return nil } func (x *FileNode) GetDigest() []byte { @@ -255,9 +255,9 @@ type SymlinkNode struct { unknownFields protoimpl.UnknownFields // The (base)name of the symlink - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // The target of the symlink. - Target string `protobuf:"bytes,2,opt,name=target,proto3" json:"target,omitempty"` + Target []byte `protobuf:"bytes,2,opt,name=target,proto3" json:"target,omitempty"` } func (x *SymlinkNode) Reset() { @@ -292,18 +292,18 @@ func (*SymlinkNode) Descriptor() ([]byte, []int) { return file_tvix_store_protos_castore_proto_rawDescGZIP(), []int{3} } -func (x *SymlinkNode) GetName() string { +func (x *SymlinkNode) GetName() []byte { if x != nil { return x.Name } - return "" + return nil } -func (x *SymlinkNode) GetTarget() string { +func (x *SymlinkNode) GetTarget() []byte { if x != nil { return x.Target } - return "" + return nil } var File_tvix_store_protos_castore_proto protoreflect.FileDescriptor @@ -325,20 +325,20 @@ var file_tvix_store_protos_castore_proto_rawDesc = []byte{ 0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x08, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x22, 0x4f, 0x0a, 0x0d, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x22, 0x6a, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x6f, - 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x22, 0x39, 0x0a, 0x0b, 0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x4e, 0x6f, 0x64, - 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x42, 0x28, 0x5a, + 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, diff --git a/tvix/store/protos/castore.proto b/tvix/store/protos/castore.proto index 747aab08bdf4..347815107198 100644 --- a/tvix/store/protos/castore.proto +++ b/tvix/store/protos/castore.proto @@ -25,7 +25,7 @@ message Directory { // A DirectoryNode represents a directory in a Directory. message DirectoryNode { // The (base)name of the directory - string name = 1; + bytes name = 1; // The blake3 hash of a Directory message, serialized in protobuf canonical form. bytes digest = 2; // Number of child elements in the Directory referred to by `digest`. @@ -44,7 +44,7 @@ message DirectoryNode { // A FileNode represents a regular or executable file in a Directory. message FileNode { // The (base)name of the file - string name = 1; + bytes name = 1; // The blake3 digest of the file contents bytes digest = 2; // The file content size @@ -56,7 +56,7 @@ message FileNode { // A SymlinkNode represents a symbolic link in a Directory. message SymlinkNode { // The (base)name of the symlink - string name = 1; + bytes name = 1; // The target of the symlink. - string target = 2; + bytes target = 2; } -- cgit 1.4.1