From 1c2db676a08847c9b73256d976f2b1eccf17310b Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Fri, 1 Mar 2024 18:00:53 +0200 Subject: feat(tvix/castore/blobsvc): add object storage implementation This uses the `object_store` crate to expose a tvix-castore BlobService backed by object storage. It's using FastCDC to chunk blobs into smaller chunks when writing to it. These are exposed at the .chunks() method. Change-Id: I2858c403d4d6490cdca73ebef03c26290b2b3c8e Reviewed-on: https://cl.tvl.fyi/c/depot/+/11076 Reviewed-by: Connor Brewster Tested-by: BuildkiteCI Reviewed-by: Brian Olsen --- tvix/castore/Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) (limited to 'tvix/castore/Cargo.toml') diff --git a/tvix/castore/Cargo.toml b/tvix/castore/Cargo.toml index a4b74cba1118..36b2d7ad50ce 100644 --- a/tvix/castore/Cargo.toml +++ b/tvix/castore/Cargo.toml @@ -11,8 +11,10 @@ bstr = "1.6.0" bytes = "1.4.0" data-encoding = "2.3.3" digest = "0.10.7" +fastcdc = { version = "3.1.0", features = ["tokio"] } futures = "0.3.30" lazy_static = "1.4.0" +object_store = { version = "0.9.1", features = ["aws", "azure", "gcp", "http"] } parking_lot = "0.12.1" pin-project-lite = "0.2.13" prost = "0.12.1" @@ -26,6 +28,7 @@ tower = "0.4.13" tracing = "0.1.37" url = "2.4.0" walkdir = "2.4.0" +zstd = "0.13.0" [dependencies.fuse-backend-rs] optional = true -- cgit 1.4.1