diff options
Diffstat (limited to '')
-rw-r--r-- | tvix/store/src/chunkservice/util.rs | 30 |
1 files changed, 30 insertions, 0 deletions
diff --git a/tvix/store/src/chunkservice/util.rs b/tvix/store/src/chunkservice/util.rs new file mode 100644 index 000000000000..cf644fa51ad4 --- /dev/null +++ b/tvix/store/src/chunkservice/util.rs @@ -0,0 +1,30 @@ +use tracing::{debug, instrument}; + +use crate::Error; + +use super::ChunkService; + +// upload a chunk to the chunk service, and return its digest (or an error) when done. +#[instrument(skip_all, err)] +pub fn upload_chunk<CS: ChunkService>( + chunk_service: &CS, + chunk_data: Vec<u8>, +) -> Result<Vec<u8>, Error> { + let mut hasher = blake3::Hasher::new(); + // TODO: benchmark this number and factor it out + if chunk_data.len() >= 128 * 1024 { + hasher.update_rayon(&chunk_data); + } else { + hasher.update(&chunk_data); + } + let digest = hasher.finalize(); + + if chunk_service.has(digest.as_bytes())? { + debug!("already has chunk, skipping"); + } + let digest_resp = chunk_service.put(chunk_data)?; + + assert_eq!(digest_resp, digest.as_bytes()); + + Ok(digest.as_bytes().to_vec()) +} |