diff options
author | Florian Klink <flokli@flokli.de> | 2023-07-19T15·52+0300 |
---|---|---|
committer | clbot <clbot@tvl.fyi> | 2023-07-22T09·03+0000 |
commit | 432222f098bfceb033e63e9a63687e35574457f9 (patch) | |
tree | 43a5e38cf79827dd63aa2adbd841da6aa62cf1de /tvix/store | |
parent | 7971d7d9ff42ed00f6f70121f372dd744f45915b (diff) |
feat(tvix/store/proto): use Bytes instead of Vec<u8> r/6439
Makes use of https://github.com/tokio-rs/prost/pull/341, which makes our bytes field cheaper to clone. It's a bit annoying to configure due to https://github.com/hyperium/tonic/issues/908, but the workaround does get the job done. Change-Id: I25714600b041bb5432d3adf5859b151e72b12778 Reviewed-on: https://cl.tvl.fyi/c/depot/+/8975 Reviewed-by: raitobezarius <tvl@lahfa.xyz> Tested-by: BuildkiteCI Reviewed-by: tazjin <tazjin@tvl.su> Autosubmit: flokli <flokli@flokli.de>
Diffstat (limited to 'tvix/store')
24 files changed, 185 insertions, 154 deletions
diff --git a/tvix/store/build.rs b/tvix/store/build.rs index e75f021a2bcf..a021dc328af6 100644 --- a/tvix/store/build.rs +++ b/tvix/store/build.rs @@ -12,21 +12,29 @@ fn main() -> Result<()> { builder = builder.file_descriptor_set_path(descriptor_path); }; - builder.build_server(true).build_client(true).compile( - &[ - "tvix/store/protos/castore.proto", - "tvix/store/protos/pathinfo.proto", - "tvix/store/protos/rpc_blobstore.proto", - "tvix/store/protos/rpc_directory.proto", - "tvix/store/protos/rpc_pathinfo.proto", - ], - // If we are in running `cargo build` manually, using `../..` works fine, - // but in case we run inside a nix build, we need to instead point PROTO_ROOT - // to a sparseTree containing that structure. - &[match std::env::var_os("PROTO_ROOT") { - Some(proto_root) => proto_root.to_str().unwrap().to_owned(), - None => "../..".to_string(), - }], - )?; + // https://github.com/hyperium/tonic/issues/908 + let mut config = prost_build::Config::new(); + config.bytes(["."]); + + builder + .build_server(true) + .build_client(true) + .compile_with_config( + config, + &[ + "tvix/store/protos/castore.proto", + "tvix/store/protos/pathinfo.proto", + "tvix/store/protos/rpc_blobstore.proto", + "tvix/store/protos/rpc_directory.proto", + "tvix/store/protos/rpc_pathinfo.proto", + ], + // If we are in running `cargo build` manually, using `../..` works fine, + // but in case we run inside a nix build, we need to instead point PROTO_ROOT + // to a sparseTree containing that structure. + &[match std::env::var_os("PROTO_ROOT") { + Some(proto_root) => proto_root.to_str().unwrap().to_owned(), + None => "../..".to_string(), + }], + )?; Ok(()) } diff --git a/tvix/store/src/blobservice/grpc.rs b/tvix/store/src/blobservice/grpc.rs index a7f0e7c6e873..a2ca16e6b3a6 100644 --- a/tvix/store/src/blobservice/grpc.rs +++ b/tvix/store/src/blobservice/grpc.rs @@ -98,7 +98,7 @@ impl BlobService for GRPCBlobService { self.tokio_handle.spawn(async move { Ok(grpc_client .stat(proto::StatBlobRequest { - digest: digest.to_vec(), + digest: digest.into(), ..Default::default() }) .await? @@ -126,7 +126,7 @@ impl BlobService for GRPCBlobService { self.tokio_handle.spawn(async move { let stream = grpc_client .read(proto::ReadBlobRequest { - digest: digest.to_vec(), + digest: digest.into(), }) .await? .into_inner(); diff --git a/tvix/store/src/directoryservice/grpc.rs b/tvix/store/src/directoryservice/grpc.rs index e6f34b2bd80c..22805523845d 100644 --- a/tvix/store/src/directoryservice/grpc.rs +++ b/tvix/store/src/directoryservice/grpc.rs @@ -91,13 +91,12 @@ impl DirectoryService for GRPCDirectoryService { fn get(&self, digest: &B3Digest) -> Result<Option<crate::proto::Directory>, crate::Error> { // Get a new handle to the gRPC client, and copy the digest. let mut grpc_client = self.grpc_client.clone(); - - let digest_as_vec = digest.to_vec(); + let digest_cpy = digest.clone(); let task = self.tokio_handle.spawn(async move { let mut s = grpc_client .get(proto::GetDirectoryRequest { recursive: false, - by_what: Some(ByWhat::Digest(digest_as_vec)), + by_what: Some(ByWhat::Digest(digest_cpy.into())), }) .await? .into_inner(); @@ -160,13 +159,15 @@ impl DirectoryService for GRPCDirectoryService { ) -> Box<dyn Iterator<Item = Result<proto::Directory, Error>> + Send> { let mut grpc_client = self.grpc_client.clone(); - let root_directory_digest_as_vec = root_directory_digest.to_vec(); + // clone so we can move it + let root_directory_digest_cpy = root_directory_digest.clone(); + let task: tokio::task::JoinHandle<Result<Streaming<proto::Directory>, Status>> = self.tokio_handle.spawn(async move { let s = grpc_client .get(proto::GetDirectoryRequest { recursive: true, - by_what: Some(ByWhat::Digest(root_directory_digest_as_vec)), + by_what: Some(ByWhat::Digest(root_directory_digest_cpy.into())), }) .await? .into_inner(); diff --git a/tvix/store/src/directoryservice/traverse.rs b/tvix/store/src/directoryservice/traverse.rs index a6e61a813b16..a385da3c63ef 100644 --- a/tvix/store/src/directoryservice/traverse.rs +++ b/tvix/store/src/directoryservice/traverse.rs @@ -108,7 +108,7 @@ mod tests { let node_directory_complicated = crate::proto::node::Node::Directory(crate::proto::DirectoryNode { name: "doesntmatter".into(), - digest: DIRECTORY_COMPLICATED.digest().to_vec(), + digest: DIRECTORY_COMPLICATED.digest().into(), size: DIRECTORY_COMPLICATED.size(), }); diff --git a/tvix/store/src/fuse/inode_tracker.rs b/tvix/store/src/fuse/inode_tracker.rs index 8d91564712d0..67d9d9b9bf46 100644 --- a/tvix/store/src/fuse/inode_tracker.rs +++ b/tvix/store/src/fuse/inode_tracker.rs @@ -13,7 +13,7 @@ pub struct InodeTracker { blob_digest_to_inode: HashMap<B3Digest, u64>, // lookup table for symlinks by their target - symlink_target_to_inode: HashMap<Vec<u8>, u64>, + symlink_target_to_inode: HashMap<bytes::Bytes, u64>, // lookup table for directories by their B3Digest. // Note the corresponding directory may not be present in data yet. @@ -171,7 +171,7 @@ impl InodeTracker { self.blob_digest_to_inode.insert(digest.clone(), ino); } InodeData::Symlink(ref target) => { - self.symlink_target_to_inode.insert(target.to_vec(), ino); + self.symlink_target_to_inode.insert(target.clone(), ino); } InodeData::Directory(DirectoryInodeData::Sparse(ref digest, _size)) => { self.directory_digest_to_inode.insert(digest.clone(), ino); diff --git a/tvix/store/src/fuse/inodes.rs b/tvix/store/src/fuse/inodes.rs index a52ba7989e80..f44dde7b804f 100644 --- a/tvix/store/src/fuse/inodes.rs +++ b/tvix/store/src/fuse/inodes.rs @@ -5,7 +5,7 @@ use crate::{proto, B3Digest}; #[derive(Clone, Debug)] pub enum InodeData { Regular(B3Digest, u32, bool), // digest, size, executable - Symlink(Vec<u8>), // target + Symlink(bytes::Bytes), // target Directory(DirectoryInodeData), // either [DirectoryInodeData:Sparse] or [DirectoryInodeData:Populated] } diff --git a/tvix/store/src/fuse/tests.rs b/tvix/store/src/fuse/tests.rs index 8577e062e928..7d01902c6848 100644 --- a/tvix/store/src/fuse/tests.rs +++ b/tvix/store/src/fuse/tests.rs @@ -58,7 +58,7 @@ fn populate_blob_a( node: Some(proto::Node { node: Some(proto::node::Node::File(FileNode { name: BLOB_A_NAME.into(), - digest: fixtures::BLOB_A_DIGEST.to_vec(), + digest: fixtures::BLOB_A_DIGEST.clone().into(), size: fixtures::BLOB_A.len() as u32, executable: false, })), @@ -84,7 +84,7 @@ fn populate_blob_b( node: Some(proto::Node { node: Some(proto::node::Node::File(FileNode { name: BLOB_B_NAME.into(), - digest: fixtures::BLOB_B_DIGEST.to_vec(), + digest: fixtures::BLOB_B_DIGEST.clone().into(), size: fixtures::BLOB_B.len() as u32, executable: false, })), @@ -154,7 +154,7 @@ fn populate_directory_with_keep( node: Some(proto::Node { node: Some(proto::node::Node::Directory(DirectoryNode { name: DIRECTORY_WITH_KEEP_NAME.into(), - digest: fixtures::DIRECTORY_WITH_KEEP.digest().to_vec(), + digest: fixtures::DIRECTORY_WITH_KEEP.digest().into(), size: fixtures::DIRECTORY_WITH_KEEP.size(), })), }), @@ -175,7 +175,7 @@ fn populate_pathinfo_without_directory( node: Some(proto::Node { node: Some(proto::node::Node::Directory(DirectoryNode { name: DIRECTORY_WITH_KEEP_NAME.into(), - digest: fixtures::DIRECTORY_WITH_KEEP.digest().to_vec(), + digest: fixtures::DIRECTORY_WITH_KEEP.digest().into(), size: fixtures::DIRECTORY_WITH_KEEP.size(), })), }), @@ -195,7 +195,7 @@ fn populate_blob_a_without_blob( node: Some(proto::Node { node: Some(proto::node::Node::File(FileNode { name: BLOB_A_NAME.into(), - digest: fixtures::BLOB_A_DIGEST.to_vec(), + digest: fixtures::BLOB_A_DIGEST.clone().into(), size: fixtures::BLOB_A.len() as u32, executable: false, })), @@ -232,7 +232,7 @@ fn populate_directory_complicated( node: Some(proto::Node { node: Some(proto::node::Node::Directory(DirectoryNode { name: DIRECTORY_COMPLICATED_NAME.into(), - digest: fixtures::DIRECTORY_COMPLICATED.digest().to_vec(), + digest: fixtures::DIRECTORY_COMPLICATED.digest().into(), size: fixtures::DIRECTORY_COMPLICATED.size(), })), }), diff --git a/tvix/store/src/import.rs b/tvix/store/src/import.rs index 74c45c7a7da8..ec45d9013edc 100644 --- a/tvix/store/src/import.rs +++ b/tvix/store/src/import.rs @@ -66,8 +66,6 @@ fn process_entry( ) -> Result<proto::node::Node, Error> { let file_type = entry.file_type(); - let entry_path: PathBuf = entry.path().to_path_buf(); - if file_type.is_dir() { let directory = maybe_directory .expect("tvix bug: must be called with some directory in the case of directory"); @@ -80,41 +78,45 @@ fn process_entry( .map_err(|e| Error::UploadDirectoryError(entry.path().to_path_buf(), e))?; return Ok(proto::node::Node::Directory(proto::DirectoryNode { - name: entry.file_name().as_bytes().to_vec(), - digest: directory_digest.to_vec(), + name: entry.file_name().as_bytes().to_owned().into(), + digest: directory_digest.into(), size: directory_size, })); } if file_type.is_symlink() { - let target = std::fs::read_link(&entry_path) - .map_err(|e| Error::UnableToStat(entry_path.clone(), e))?; + let target: bytes::Bytes = std::fs::read_link(entry.path()) + .map_err(|e| Error::UnableToStat(entry.path().to_path_buf(), e))? + .as_os_str() + .as_bytes() + .to_owned() + .into(); return Ok(proto::node::Node::Symlink(proto::SymlinkNode { - name: entry.file_name().as_bytes().to_vec(), - target: target.as_os_str().as_bytes().to_vec(), + name: entry.file_name().as_bytes().to_owned().into(), + target, })); } if file_type.is_file() { let metadata = entry .metadata() - .map_err(|e| Error::UnableToStat(entry_path.clone(), e.into()))?; + .map_err(|e| Error::UnableToStat(entry.path().to_path_buf(), e.into()))?; - let mut file = File::open(entry_path.clone()) - .map_err(|e| Error::UnableToOpen(entry_path.clone(), e))?; + let mut file = File::open(entry.path()) + .map_err(|e| Error::UnableToOpen(entry.path().to_path_buf(), e))?; let mut writer = blob_service.open_write(); if let Err(e) = io::copy(&mut file, &mut writer) { - return Err(Error::UnableToRead(entry_path, e)); + return Err(Error::UnableToRead(entry.path().to_path_buf(), e)); }; let digest = writer.close()?; return Ok(proto::node::Node::File(proto::FileNode { - name: entry.file_name().as_bytes().to_vec(), - digest: digest.to_vec(), + name: entry.file_name().as_bytes().to_vec().into(), + digest: digest.into(), size: metadata.len() as u32, // If it's executable by the user, it'll become executable. // This matches nix's dump() function behaviour. @@ -150,8 +152,9 @@ pub fn ingest_path<P: AsRef<Path> + Debug>( .file_name() .unwrap_or_default() .as_bytes() - .to_vec(), - target: target.as_os_str().as_bytes().to_vec(), + .to_owned() + .into(), + target: target.as_os_str().as_bytes().to_vec().into(), })); } diff --git a/tvix/store/src/nar/mod.rs b/tvix/store/src/nar/mod.rs index 84a48ba5f6b5..5a8bc21ae953 100644 --- a/tvix/store/src/nar/mod.rs +++ b/tvix/store/src/nar/mod.rs @@ -13,13 +13,13 @@ pub enum RenderError { StoreError(crate::Error), #[error("unable to find directory {}, referred from {:?}", .0, .1)] - DirectoryNotFound(B3Digest, Vec<u8>), + DirectoryNotFound(B3Digest, bytes::Bytes), #[error("unable to find blob {}, referred from {:?}", BASE64.encode(.0), .1)] - BlobNotFound([u8; 32], Vec<u8>), + BlobNotFound([u8; 32], bytes::Bytes), #[error("unexpected size in metadata for blob {}, referred from {:?} returned, expected {}, got {}", BASE64.encode(.0), .1, .2, .3)] - UnexpectedBlobMeta([u8; 32], Vec<u8>, u32, u32), + UnexpectedBlobMeta([u8; 32], bytes::Bytes, u32, u32), #[error("failure using the NAR writer: {0}")] NARWriterError(std::io::Error), diff --git a/tvix/store/src/nar/renderer.rs b/tvix/store/src/nar/renderer.rs index e2119ae0796d..cc5af488abf8 100644 --- a/tvix/store/src/nar/renderer.rs +++ b/tvix/store/src/nar/renderer.rs @@ -115,7 +115,7 @@ fn walk_node( None => { return Err(RenderError::DirectoryNotFound( digest, - proto_directory_node.name.to_owned(), + proto_directory_node.name.clone(), )) } Some(proto_directory) => { diff --git a/tvix/store/src/pathinfoservice/grpc.rs b/tvix/store/src/pathinfoservice/grpc.rs index db9fd63366ea..c98a89c4b8e7 100644 --- a/tvix/store/src/pathinfoservice/grpc.rs +++ b/tvix/store/src/pathinfoservice/grpc.rs @@ -97,7 +97,7 @@ impl PathInfoService for GRPCPathInfoService { let path_info = grpc_client .get(proto::GetPathInfoRequest { by_what: Some(proto::get_path_info_request::ByWhat::ByOutputHash( - digest.to_vec(), + digest.to_vec().into(), )), }) .await? @@ -154,6 +154,7 @@ impl PathInfoService for GRPCPathInfoService { let nar_sha256: [u8; 32] = resp .nar_sha256 + .to_vec() .try_into() .map_err(|_e| crate::Error::StorageError("invalid digest length".to_string()))?; diff --git a/tvix/store/src/proto/grpc_blobservice_wrapper.rs b/tvix/store/src/proto/grpc_blobservice_wrapper.rs index e60ff2ef1d26..2d8c396539d8 100644 --- a/tvix/store/src/proto/grpc_blobservice_wrapper.rs +++ b/tvix/store/src/proto/grpc_blobservice_wrapper.rs @@ -133,9 +133,7 @@ impl super::blob_service_server::BlobService for GRPCBlobServiceWrapper { x: Result<bytes::Bytes, io::Error>, ) -> Result<super::BlobChunk, Status> { match x { - Ok(bytes) => Ok(super::BlobChunk { - data: bytes.to_vec(), - }), + Ok(bytes) => Ok(super::BlobChunk { data: bytes }), Err(e) => Err(Status::from(e)), } } @@ -156,7 +154,7 @@ impl super::blob_service_server::BlobService for GRPCBlobServiceWrapper { let req_inner = request.into_inner(); let data_stream = req_inner.map(|x| { - x.map(|x| VecDeque::from(x.data)) + x.map(|x| VecDeque::from(x.data.to_vec())) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e)) }); @@ -182,7 +180,9 @@ impl super::blob_service_server::BlobService for GRPCBlobServiceWrapper { })? .to_vec(); - Ok(super::PutBlobResponse { digest }) + Ok(super::PutBlobResponse { + digest: digest.into(), + }) }) .await .map_err(|_| Status::internal("failed to wait for task"))??; diff --git a/tvix/store/src/proto/grpc_directoryservice_wrapper.rs b/tvix/store/src/proto/grpc_directoryservice_wrapper.rs index 22fcd2fa6a64..ec53d7d76ce3 100644 --- a/tvix/store/src/proto/grpc_directoryservice_wrapper.rs +++ b/tvix/store/src/proto/grpc_directoryservice_wrapper.rs @@ -176,7 +176,7 @@ impl proto::directory_service_server::DirectoryService for GRPCDirectoryServiceW match last_directory_dgst { None => Err(Status::invalid_argument("no directories received")), Some(last_directory_dgst) => Ok(Response::new(proto::PutDirectoryResponse { - root_digest: last_directory_dgst.to_vec(), + root_digest: last_directory_dgst.into(), })), } } diff --git a/tvix/store/src/proto/grpc_pathinfoservice_wrapper.rs b/tvix/store/src/proto/grpc_pathinfoservice_wrapper.rs index 9f26da213bd5..bbf235f8ace9 100644 --- a/tvix/store/src/proto/grpc_pathinfoservice_wrapper.rs +++ b/tvix/store/src/proto/grpc_pathinfoservice_wrapper.rs @@ -26,10 +26,11 @@ impl proto::path_info_service_server::PathInfoService for GRPCPathInfoServiceWra ) -> Result<Response<proto::PathInfo>> { match request.into_inner().by_what { None => Err(Status::unimplemented("by_what needs to be specified")), - Some(proto::get_path_info_request::ByWhat::ByOutputHash(digest)) => { - let digest: [u8; 20] = digest + Some(proto::get_path_info_request::ByWhat::ByOutputHash(output_digest)) => { + let digest: [u8; 20] = output_digest + .to_vec() .try_into() - .map_err(|_e| Status::invalid_argument("invalid digest length"))?; + .map_err(|_e| Status::invalid_argument("invalid output digest length"))?; match self.path_info_service.get(digest) { Ok(None) => Err(Status::not_found("PathInfo not found")), Ok(Some(path_info)) => Ok(Response::new(path_info)), @@ -72,7 +73,7 @@ impl proto::path_info_service_server::PathInfoService for GRPCPathInfoServiceWra Ok(Response::new(proto::CalculateNarResponse { nar_size, - nar_sha256: nar_sha256.to_vec(), + nar_sha256: nar_sha256.to_vec().into(), })) } } diff --git a/tvix/store/src/proto/mod.rs b/tvix/store/src/proto/mod.rs index 126a8b0edce3..9ee98d8c1c51 100644 --- a/tvix/store/src/proto/mod.rs +++ b/tvix/store/src/proto/mod.rs @@ -86,7 +86,7 @@ fn validate_node_name<E>(name: &[u8], err: fn(Vec<u8>) -> E) -> Result<(), E> { /// Checks a digest for validity. /// Digests are 32 bytes long, as we store blake3 digests. -fn validate_digest<E>(digest: &Vec<u8>, err: fn(usize) -> E) -> Result<(), E> { +fn validate_digest<E>(digest: &bytes::Bytes, err: fn(usize) -> E) -> Result<(), E> { if digest.len() != 32 { return Err(err(digest.len())); } diff --git a/tvix/store/src/proto/tests/directory.rs b/tvix/store/src/proto/tests/directory.rs index 22b10ca746a9..eed49b2b593c 100644 --- a/tvix/store/src/proto/tests/directory.rs +++ b/tvix/store/src/proto/tests/directory.rs @@ -18,7 +18,7 @@ fn size() { let d = Directory { directories: vec![DirectoryNode { name: "foo".into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.to_vec().into(), size: 0, }], ..Default::default() @@ -29,7 +29,7 @@ fn size() { let d = Directory { directories: vec![DirectoryNode { name: "foo".into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.to_vec().into(), size: 4, }], ..Default::default() @@ -40,7 +40,7 @@ fn size() { let d = Directory { files: vec![FileNode { name: "foo".into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.to_vec().into(), size: 42, executable: false, }], @@ -88,7 +88,7 @@ fn validate_invalid_names() { let d = Directory { directories: vec![DirectoryNode { name: "".into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.to_vec().into(), size: 42, }], ..Default::default() @@ -105,7 +105,7 @@ fn validate_invalid_names() { let d = Directory { directories: vec![DirectoryNode { name: ".".into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.to_vec().into(), size: 42, }], ..Default::default() @@ -122,7 +122,7 @@ fn validate_invalid_names() { let d = Directory { files: vec![FileNode { name: "..".into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.to_vec().into(), size: 42, executable: false, }], @@ -174,7 +174,7 @@ fn validate_invalid_digest() { let d = Directory { directories: vec![DirectoryNode { name: "foo".into(), - digest: vec![0x00, 0x42], // invalid length + digest: vec![0x00, 0x42].into(), // invalid length size: 42, }], ..Default::default() @@ -195,12 +195,12 @@ fn validate_sorting() { directories: vec![ DirectoryNode { name: "b".into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.to_vec().into(), size: 42, }, DirectoryNode { name: "a".into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.to_vec().into(), size: 42, }, ], @@ -220,12 +220,12 @@ fn validate_sorting() { directories: vec![ DirectoryNode { name: "a".into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.to_vec().into(), size: 42, }, DirectoryNode { name: "a".into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.to_vec().into(), size: 42, }, ], @@ -245,12 +245,12 @@ fn validate_sorting() { directories: vec![ DirectoryNode { name: "a".into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.to_vec().into(), size: 42, }, DirectoryNode { name: "b".into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.to_vec().into(), size: 42, }, ], @@ -266,12 +266,12 @@ fn validate_sorting() { directories: vec![ DirectoryNode { name: "b".into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.to_vec().into(), size: 42, }, DirectoryNode { name: "c".into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.to_vec().into(), size: 42, }, ], diff --git a/tvix/store/src/proto/tests/grpc_blobservice.rs b/tvix/store/src/proto/tests/grpc_blobservice.rs index 2f18ea4abb99..8ad4e33ca469 100644 --- a/tvix/store/src/proto/tests/grpc_blobservice.rs +++ b/tvix/store/src/proto/tests/grpc_blobservice.rs @@ -16,7 +16,7 @@ async fn not_found_read() { let resp = service .read(tonic::Request::new(ReadBlobRequest { - digest: BLOB_A_DIGEST.to_vec(), + digest: BLOB_A_DIGEST.clone().into(), })) .await; @@ -36,7 +36,7 @@ async fn not_found_stat() { let resp = service .stat(tonic::Request::new(StatBlobRequest { - digest: BLOB_A_DIGEST.to_vec(), + digest: BLOB_A_DIGEST.clone().into(), ..Default::default() })) .await @@ -54,7 +54,7 @@ async fn put_read_stat() { // Send blob A. let put_resp = service .put(tonic_mock::streaming_request(vec![BlobChunk { - data: BLOB_A.clone(), + data: BLOB_A.clone().into(), }])) .await .expect("must succeed") @@ -67,7 +67,7 @@ async fn put_read_stat() { // expose it yet. let _resp = service .stat(tonic::Request::new(StatBlobRequest { - digest: BLOB_A_DIGEST.to_vec(), + digest: BLOB_A_DIGEST.clone().into(), ..Default::default() })) .await @@ -77,7 +77,7 @@ async fn put_read_stat() { // Read the blob. It should return the same data. let resp = service .read(tonic::Request::new(ReadBlobRequest { - digest: BLOB_A_DIGEST.to_vec(), + digest: BLOB_A_DIGEST.clone().into(), })) .await; @@ -90,7 +90,7 @@ async fn put_read_stat() { .expect("must be some") .expect("must succeed"); - assert_eq!(BLOB_A.to_vec(), item.data); + assert_eq!(BLOB_A.clone(), item.data); // … and no more elements assert!(rx.next().await.is_none()); diff --git a/tvix/store/src/proto/tests/grpc_directoryservice.rs b/tvix/store/src/proto/tests/grpc_directoryservice.rs index 73ce0082d3ca..a5300039fb9f 100644 --- a/tvix/store/src/proto/tests/grpc_directoryservice.rs +++ b/tvix/store/src/proto/tests/grpc_directoryservice.rs @@ -42,7 +42,7 @@ async fn not_found() { let resp = service .get(tonic::Request::new(GetDirectoryRequest { - by_what: Some(ByWhat::Digest(DIRECTORY_A.digest().to_vec())), + by_what: Some(ByWhat::Digest(DIRECTORY_A.digest().into())), ..Default::default() })) .await; @@ -80,7 +80,7 @@ async fn put_get() { let items = get_directories( &service, GetDirectoryRequest { - by_what: Some(ByWhat::Digest(DIRECTORY_A.digest().to_vec())), + by_what: Some(ByWhat::Digest(DIRECTORY_A.digest().into())), ..Default::default() }, ) @@ -122,7 +122,7 @@ async fn put_get_multiple() { &service, GetDirectoryRequest { recursive: false, - by_what: Some(ByWhat::Digest(DIRECTORY_B.digest().to_vec())), + by_what: Some(ByWhat::Digest(DIRECTORY_B.digest().into())), }, ) .await @@ -136,7 +136,7 @@ async fn put_get_multiple() { &service, GetDirectoryRequest { recursive: true, - by_what: Some(ByWhat::Digest(DIRECTORY_B.digest().to_vec())), + by_what: Some(ByWhat::Digest(DIRECTORY_B.digest().into())), }, ) .await @@ -172,7 +172,7 @@ async fn put_get_dedup() { &service, GetDirectoryRequest { recursive: true, - by_what: Some(ByWhat::Digest(DIRECTORY_C.digest().to_vec())), + by_what: Some(ByWhat::Digest(DIRECTORY_C.digest().into())), }, ) .await @@ -215,7 +215,7 @@ async fn put_reject_wrong_size() { let broken_parent_directory = Directory { directories: vec![DirectoryNode { name: "foo".into(), - digest: DIRECTORY_A.digest().to_vec(), + digest: DIRECTORY_A.digest().into(), size: 42, }], ..Default::default() diff --git a/tvix/store/src/proto/tests/grpc_pathinfoservice.rs b/tvix/store/src/proto/tests/grpc_pathinfoservice.rs index 57c88c2863b1..95d97bb12890 100644 --- a/tvix/store/src/proto/tests/grpc_pathinfoservice.rs +++ b/tvix/store/src/proto/tests/grpc_pathinfoservice.rs @@ -32,7 +32,7 @@ async fn not_found() { let resp = service .get(Request::new(GetPathInfoRequest { - by_what: Some(ByOutputHash(DUMMY_OUTPUT_HASH.to_vec())), + by_what: Some(ByOutputHash(DUMMY_OUTPUT_HASH.clone())), })) .await; @@ -62,7 +62,7 @@ async fn put_get() { let resp = service .get(Request::new(GetPathInfoRequest { - by_what: Some(ByOutputHash(DUMMY_OUTPUT_HASH.to_vec())), + by_what: Some(ByOutputHash(DUMMY_OUTPUT_HASH.clone())), })) .await; diff --git a/tvix/store/src/proto/tests/pathinfo.rs b/tvix/store/src/proto/tests/pathinfo.rs index a14554ee4fd3..779b46ed168e 100644 --- a/tvix/store/src/proto/tests/pathinfo.rs +++ b/tvix/store/src/proto/tests/pathinfo.rs @@ -1,20 +1,28 @@ use crate::proto::{self, Node, PathInfo, ValidatePathInfoError}; +use crate::B3Digest; +use bytes::Bytes; use lazy_static::lazy_static; use nix_compat::store_path::{self, StorePath}; use std::str::FromStr; use test_case::test_case; lazy_static! { - static ref DUMMY_DIGEST: Vec<u8> = vec![ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, - ]; - static ref DUMMY_DIGEST_2: Vec<u8> = vec![ - 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, - ]; + static ref DUMMY_DIGEST: B3Digest = { + let u: &[u8; 32] = &[ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ]; + u.into() + }; + static ref DUMMY_DIGEST_2: B3Digest = { + let u: &[u8; 32] = &[ + 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ]; + u.into() + }; } const DUMMY_NAME: &str = "00000000000000000000000000000000-dummy"; @@ -44,7 +52,7 @@ fn validate_no_node( #[test_case( proto::DirectoryNode { name: DUMMY_NAME.into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.clone().into(), size: 0, }, Ok(StorePath::from_str(DUMMY_NAME).expect("must succeed")); @@ -53,7 +61,7 @@ fn validate_no_node( #[test_case( proto::DirectoryNode { name: DUMMY_NAME.into(), - digest: vec![], + digest: Bytes::new(), size: 0, }, Err(ValidatePathInfoError::InvalidDigestLen(0)); @@ -62,7 +70,7 @@ fn validate_no_node( #[test_case( proto::DirectoryNode { name: "invalid".into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.clone().into(), size: 0, }, Err(ValidatePathInfoError::InvalidNodeName( @@ -88,7 +96,7 @@ fn validate_directory( #[test_case( proto::FileNode { name: DUMMY_NAME.into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.clone().into(), size: 0, executable: false, }, @@ -98,7 +106,7 @@ fn validate_directory( #[test_case( proto::FileNode { name: DUMMY_NAME.into(), - digest: vec![], + digest: Bytes::new(), ..Default::default() }, Err(ValidatePathInfoError::InvalidDigestLen(0)); @@ -107,7 +115,7 @@ fn validate_directory( #[test_case( proto::FileNode { name: "invalid".into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.clone().into(), ..Default::default() }, Err(ValidatePathInfoError::InvalidNodeName( @@ -167,11 +175,11 @@ fn validate_references() { node: Some(Node { node: Some(proto::node::Node::Directory(proto::DirectoryNode { name: DUMMY_NAME.into(), - digest: DUMMY_DIGEST.to_vec(), + digest: DUMMY_DIGEST.clone().into(), size: 0, })), }), - references: vec![DUMMY_DIGEST_2.to_vec()], + references: vec![DUMMY_DIGEST_2.clone().into()], narinfo: None, }; assert!(path_info.validate().is_ok()); @@ -180,7 +188,7 @@ fn validate_references() { let path_info_with_narinfo_missing_refs = PathInfo { narinfo: Some(proto::NarInfo { nar_size: 0, - nar_sha256: DUMMY_DIGEST.to_vec(), + nar_sha256: DUMMY_DIGEST.clone().into(), signatures: vec![], reference_names: vec![], }), @@ -198,7 +206,7 @@ fn validate_references() { let path_info_with_narinfo = PathInfo { narinfo: Some(proto::NarInfo { nar_size: 0, - nar_sha256: DUMMY_DIGEST.to_vec(), + nar_sha256: DUMMY_DIGEST.clone().into(), signatures: vec![], reference_names: vec![format!("/nix/store/{}", DUMMY_NAME)], }), diff --git a/tvix/store/src/store_io.rs b/tvix/store/src/store_io.rs index 19a809b6a1a7..1030bbdd337f 100644 --- a/tvix/store/src/store_io.rs +++ b/tvix/store/src/store_io.rs @@ -129,7 +129,7 @@ impl TvixStoreIO { // assemble a new root_node with a name that is derived from the nar hash. let renamed_root_node = { - let name = output_path.to_string().into_bytes(); + let name = output_path.to_string().into_bytes().into(); match root_node { crate::proto::node::Node::Directory(n) => { @@ -153,7 +153,7 @@ impl TvixStoreIO { references: vec![], narinfo: Some(crate::proto::NarInfo { nar_size, - nar_sha256: nar_sha256.to_vec(), + nar_sha256: nar_sha256.to_vec().into(), signatures: vec![], reference_names: vec![], // TODO: narinfo for talosctl.src contains `CA: fixed:r:sha256:1x13j5hy75221bf6kz7cpgld9vgic6bqx07w5xjs4pxnksj6lxb6` @@ -264,7 +264,7 @@ impl EvalIO for TvixStoreIO { } #[instrument(skip(self), ret, err)] - fn read_dir(&self, path: &Path) -> Result<Vec<(Vec<u8>, FileType)>, io::Error> { + fn read_dir(&self, path: &Path) -> Result<Vec<(bytes::Bytes, FileType)>, io::Error> { if let Ok((store_path, sub_path)) = StorePath::from_absolute_path_full(&path.to_string_lossy()) { @@ -283,7 +283,7 @@ impl EvalIO for TvixStoreIO { })?; if let Some(directory) = self.directory_service.get(&digest)? { - let mut children: Vec<(Vec<u8>, FileType)> = Vec::new(); + let mut children: Vec<(bytes::Bytes, FileType)> = Vec::new(); for node in directory.nodes() { children.push(match node { crate::proto::node::Node::Directory(e) => { diff --git a/tvix/store/src/tests/fixtures.rs b/tvix/store/src/tests/fixtures.rs index a1df729f1c5c..c362744a34a7 100644 --- a/tvix/store/src/tests/fixtures.rs +++ b/tvix/store/src/tests/fixtures.rs @@ -8,13 +8,16 @@ pub const HELLOWORLD_BLOB_CONTENTS: &[u8] = b"Hello World!"; pub const EMPTY_BLOB_CONTENTS: &[u8] = b""; lazy_static! { - pub static ref DUMMY_DIGEST: Vec<u8> = vec![ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, - ]; - pub static ref DUMMY_DATA_1: Vec<u8> = vec![0x01, 0x02, 0x03]; - pub static ref DUMMY_DATA_2: Vec<u8> = vec![0x04, 0x05]; + pub static ref DUMMY_DIGEST: B3Digest = { + let u: &[u8; 32] = &[ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, + ]; + u.into() + }; + pub static ref DUMMY_DATA_1: bytes::Bytes = vec![0x01, 0x02, 0x03].into(); + pub static ref DUMMY_DATA_2: bytes::Bytes = vec![0x04, 0x05].into(); pub static ref HELLOWORLD_BLOB_DIGEST: B3Digest = blake3::hash(HELLOWORLD_BLOB_CONTENTS).as_bytes().into(); @@ -22,19 +25,19 @@ lazy_static! { blake3::hash(EMPTY_BLOB_CONTENTS).as_bytes().into(); // 2 bytes - pub static ref BLOB_A: Vec<u8> = vec![0x00, 0x01]; + pub static ref BLOB_A: bytes::Bytes = vec![0x00, 0x01].into(); pub static ref BLOB_A_DIGEST: B3Digest = blake3::hash(&BLOB_A).as_bytes().into(); // 1MB - pub static ref BLOB_B: Vec<u8> = (0..255).collect::<Vec<u8>>().repeat(4 * 1024); + pub static ref BLOB_B: bytes::Bytes = (0..255).collect::<Vec<u8>>().repeat(4 * 1024).into(); pub static ref BLOB_B_DIGEST: B3Digest = blake3::hash(&BLOB_B).as_bytes().into(); // Directories pub static ref DIRECTORY_WITH_KEEP: proto::Directory = proto::Directory { directories: vec![], files: vec![FileNode { - name: b".keep".to_vec(), - digest: EMPTY_BLOB_DIGEST.to_vec(), + name: b".keep".to_vec().into(), + digest: EMPTY_BLOB_DIGEST.clone().into(), size: 0, executable: false, }], @@ -42,26 +45,26 @@ lazy_static! { }; pub static ref DIRECTORY_COMPLICATED: proto::Directory = proto::Directory { directories: vec![DirectoryNode { - name: b"keep".to_vec(), - digest: DIRECTORY_WITH_KEEP.digest().to_vec(), + name: b"keep".to_vec().into(), + digest: DIRECTORY_WITH_KEEP.digest().into(), size: DIRECTORY_WITH_KEEP.size(), }], files: vec![FileNode { - name: b".keep".to_vec(), - digest: EMPTY_BLOB_DIGEST.to_vec(), + name: b".keep".to_vec().into(), + digest: EMPTY_BLOB_DIGEST.clone().into(), size: 0, executable: false, }], symlinks: vec![SymlinkNode { - name: b"aa".to_vec(), - target: b"/nix/store/somewhereelse".to_vec(), + name: b"aa".to_vec().into(), + target: b"/nix/store/somewhereelse".to_vec().into(), }], }; pub static ref DIRECTORY_A: Directory = Directory::default(); pub static ref DIRECTORY_B: Directory = Directory { directories: vec![DirectoryNode { - name: b"a".to_vec(), - digest: DIRECTORY_A.digest().to_vec(), + name: b"a".to_vec().into(), + digest: DIRECTORY_A.digest().into(), size: DIRECTORY_A.size(), }], ..Default::default() @@ -69,13 +72,13 @@ lazy_static! { pub static ref DIRECTORY_C: Directory = Directory { directories: vec![ DirectoryNode { - name: b"a".to_vec(), - digest: DIRECTORY_A.digest().to_vec(), + name: b"a".to_vec().into(), + digest: DIRECTORY_A.digest().into(), size: DIRECTORY_A.size(), }, DirectoryNode { - name: b"a'".to_vec(), - digest: DIRECTORY_A.digest().to_vec(), + name: b"a'".to_vec().into(), + digest: DIRECTORY_A.digest().into(), size: DIRECTORY_A.size(), } ], @@ -83,10 +86,10 @@ lazy_static! { }; // output hash - pub static ref DUMMY_OUTPUT_HASH: Vec<u8> = vec![ + pub static ref DUMMY_OUTPUT_HASH: bytes::Bytes = vec![ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 - ]; + ].into(); /// The NAR representation of a symlink pointing to `/nix/store/somewhereelse` pub static ref NAR_CONTENTS_SYMLINK: Vec<u8> = vec![ diff --git a/tvix/store/src/tests/import.rs b/tvix/store/src/tests/import.rs index 291501f72768..ccaa4f4e4244 100644 --- a/tvix/store/src/tests/import.rs +++ b/tvix/store/src/tests/import.rs @@ -54,7 +54,7 @@ fn single_file() { assert_eq!( crate::proto::node::Node::File(proto::FileNode { name: "root".into(), - digest: HELLOWORLD_BLOB_DIGEST.to_vec(), + digest: HELLOWORLD_BLOB_DIGEST.clone().into(), size: HELLOWORLD_BLOB_CONTENTS.len() as u32, executable: false, }), @@ -92,8 +92,14 @@ fn complicated() { // ensure root_node matched expectations assert_eq!( crate::proto::node::Node::Directory(proto::DirectoryNode { - name: tmpdir.path().file_name().unwrap().as_bytes().to_vec(), - digest: DIRECTORY_COMPLICATED.digest().to_vec(), + name: tmpdir + .path() + .file_name() + .unwrap() + .as_bytes() + .to_owned() + .into(), + digest: DIRECTORY_COMPLICATED.digest().into(), size: DIRECTORY_COMPLICATED.size(), }), root_node, diff --git a/tvix/store/src/tests/nar_renderer.rs b/tvix/store/src/tests/nar_renderer.rs index 055538376b72..75dab76a95d9 100644 --- a/tvix/store/src/tests/nar_renderer.rs +++ b/tvix/store/src/tests/nar_renderer.rs @@ -36,7 +36,7 @@ fn single_file_missing_blob() { &mut buf, &crate::proto::node::Node::File(FileNode { name: "doesntmatter".into(), - digest: HELLOWORLD_BLOB_DIGEST.to_vec(), + digest: HELLOWORLD_BLOB_DIGEST.clone().into(), size: HELLOWORLD_BLOB_CONTENTS.len() as u32, executable: false, }), @@ -77,7 +77,7 @@ fn single_file_wrong_blob_size() { &mut buf, &crate::proto::node::Node::File(FileNode { name: "doesntmatter".into(), - digest: HELLOWORLD_BLOB_DIGEST.to_vec(), + digest: HELLOWORLD_BLOB_DIGEST.clone().into(), size: 42, // <- note the wrong size here! executable: false, }), @@ -102,7 +102,7 @@ fn single_file_wrong_blob_size() { &mut buf, &crate::proto::node::Node::File(FileNode { name: "doesntmatter".into(), - digest: HELLOWORLD_BLOB_DIGEST.to_vec(), + digest: HELLOWORLD_BLOB_DIGEST.clone().into(), size: 2, // <- note the wrong size here! executable: false, }), @@ -127,7 +127,7 @@ fn single_file() { // insert blob into the store let mut writer = blob_service.open_write(); io::copy( - &mut io::Cursor::new(HELLOWORLD_BLOB_CONTENTS.to_vec()), + &mut io::Cursor::new(HELLOWORLD_BLOB_CONTENTS.clone()), &mut writer, ) .unwrap(); @@ -139,7 +139,7 @@ fn single_file() { &mut buf, &crate::proto::node::Node::File(FileNode { name: "doesntmatter".into(), - digest: HELLOWORLD_BLOB_DIGEST.to_vec(), + digest: HELLOWORLD_BLOB_DIGEST.clone().into(), size: HELLOWORLD_BLOB_CONTENTS.len() as u32, executable: false, }), @@ -160,7 +160,7 @@ fn test_complicated() { // insert blob into the store let mut writer = blob_service.open_write(); io::copy( - &mut io::Cursor::new(EMPTY_BLOB_CONTENTS.to_vec()), + &mut io::Cursor::new(EMPTY_BLOB_CONTENTS.clone()), &mut writer, ) .unwrap(); @@ -177,7 +177,7 @@ fn test_complicated() { &mut buf, &crate::proto::node::Node::Directory(DirectoryNode { name: "doesntmatter".into(), - digest: DIRECTORY_COMPLICATED.digest().to_vec(), + digest: DIRECTORY_COMPLICATED.digest().clone().into(), size: DIRECTORY_COMPLICATED.size(), }), blob_service.clone(), @@ -191,7 +191,7 @@ fn test_complicated() { let (nar_size, nar_digest) = calculate_size_and_sha256( &crate::proto::node::Node::Directory(DirectoryNode { name: "doesntmatter".into(), - digest: DIRECTORY_COMPLICATED.digest().to_vec(), + digest: DIRECTORY_COMPLICATED.digest().clone().into(), size: DIRECTORY_COMPLICATED.size(), }), blob_service, |