about summary refs log tree commit diff
path: root/tvix/store/src/chunkservice/util.rs
diff options
context:
space:
mode:
authorFlorian Klink <flokli@flokli.de>2023-03-10T13·21+0100
committerflokli <flokli@flokli.de>2023-03-10T15·45+0000
commit3506d3bd0ebbbcd907d3cd627d8dd828171fd6d1 (patch)
treedd326e72908f80cb02fa5f2e3d38028b594be1d6 /tvix/store/src/chunkservice/util.rs
parent05b2f1ccb40f5cf4c020ebca481a97b8567f868e (diff)
refactor(tvix/store): move upload_chunk out of blobwriter r/5948
This is useful not only in blobwriter contexts.

Change-Id: I4c584b5264ff7b4bb3b1a9671affc39e18bf4ccf
Reviewed-on: https://cl.tvl.fyi/c/depot/+/8245
Tested-by: BuildkiteCI
Autosubmit: flokli <flokli@flokli.de>
Reviewed-by: raitobezarius <tvl@lahfa.xyz>
Diffstat (limited to 'tvix/store/src/chunkservice/util.rs')
-rw-r--r--tvix/store/src/chunkservice/util.rs30
1 files changed, 30 insertions, 0 deletions
diff --git a/tvix/store/src/chunkservice/util.rs b/tvix/store/src/chunkservice/util.rs
new file mode 100644
index 000000000000..cf644fa51ad4
--- /dev/null
+++ b/tvix/store/src/chunkservice/util.rs
@@ -0,0 +1,30 @@
+use tracing::{debug, instrument};
+
+use crate::Error;
+
+use super::ChunkService;
+
+// upload a chunk to the chunk service, and return its digest (or an error) when done.
+#[instrument(skip_all, err)]
+pub fn upload_chunk<CS: ChunkService>(
+    chunk_service: &CS,
+    chunk_data: Vec<u8>,
+) -> Result<Vec<u8>, Error> {
+    let mut hasher = blake3::Hasher::new();
+    // TODO: benchmark this number and factor it out
+    if chunk_data.len() >= 128 * 1024 {
+        hasher.update_rayon(&chunk_data);
+    } else {
+        hasher.update(&chunk_data);
+    }
+    let digest = hasher.finalize();
+
+    if chunk_service.has(digest.as_bytes())? {
+        debug!("already has chunk, skipping");
+    }
+    let digest_resp = chunk_service.put(chunk_data)?;
+
+    assert_eq!(digest_resp, digest.as_bytes());
+
+    Ok(digest.as_bytes().to_vec())
+}