about summary refs log tree commit diff
path: root/tvix/tools/crunch-v2/Cargo.toml
diff options
context:
space:
mode:
authoredef <edef@edef.eu>2024-01-17T16·04+0000
committeredef <edef@edef.eu>2024-01-27T18·23+0000
commit4f22203a3aecd070881ae9b4eabc47532d948f01 (patch)
tree39e493d8d4fcf6362dd4e25be8bcb64fe0b0e25e /tvix/tools/crunch-v2/Cargo.toml
parente0a1c03b2471271fc96690b3dc5dd5423f93fa42 (diff)
feat(tvix/tools/crunch-v2): init r/7452
This is a tool for ingesting subsets of cache.nixos.org into its own flattened castore format.
Currently, produced chunks are not preserved, and this purely serves as a way of measuring
compression/deduplication ratios for various chunking and compression parameters.

Change-Id: I3983af02a66f7837d76874ee0fc8b2fab62ac17e
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10486
Tested-by: BuildkiteCI
Reviewed-by: flokli <flokli@flokli.de>
Diffstat (limited to 'tvix/tools/crunch-v2/Cargo.toml')
-rw-r--r--tvix/tools/crunch-v2/Cargo.toml38
1 files changed, 38 insertions, 0 deletions
diff --git a/tvix/tools/crunch-v2/Cargo.toml b/tvix/tools/crunch-v2/Cargo.toml
new file mode 100644
index 000000000000..eb44805d7c2c
--- /dev/null
+++ b/tvix/tools/crunch-v2/Cargo.toml
@@ -0,0 +1,38 @@
+[package]
+name = "crunch-v2"
+version = "0.1.0"
+edition = "2021"
+
+[workspace]
+members = ["."]
+
+[dependencies]
+anyhow = { version = "1.0.75", features = ["backtrace"] }
+lazy_static = "1.4.0"
+
+bstr = "1.8.0"
+bytes = "1.5.0"
+
+futures = "0.3.29"
+tokio = { version = "1.34.0", features = ["full"] }
+
+rusoto_core = { version = "0.48.0", default-features = false, features = ["hyper-rustls"] }
+rusoto_s3 = { version = "0.48.0", default-features = false, features = ["rustls"] }
+
+nix-compat = { version = "0.1.0", path = "../../nix-compat" }
+sled = "0.34.7"
+
+fastcdc = "3.1.0"
+blake3 = "1.5.0"
+sha2 = { version = "0.10.8", features = ["asm"] }
+digest = "0.10.7"
+
+bzip2 = "0.4.4"
+xz2 = "0.1.7"
+zstd = "0.13.0"
+prost = "0.12.2"
+polars = { version = "0.35.4", default-features = false, features = ["parquet", "lazy", "sql", "dtype-struct"] }
+indicatif = "0.17.7"
+
+[build-dependencies]
+prost-build = "0.12.2"