diff options
| author | edef <edef@edef.eu> | 2024-10-17T13·26+0000 |
|---|---|---|
| committer | edef <edef@edef.eu> | 2024-10-19T17·01+0000 |
| commit | 313899c291f0295506c275418e570b39b4a5f079 (patch) | |
| tree | 0a4c737f7c54082eaf3957072693115e29fabcd1 /users/edef/weave/Cargo.toml | |
| parent | bdc2891053c28a5739c4be55f4816d362a5f08e2 (diff) | |
refactor(users/edef/weave/swizzle): use polars streaming r/8844
This vastly reduces the memory requirements, so we can run in ~40G RAM. Change-Id: I4952a780df294bd852a8b4682ba2fd59b9bae675 Reviewed-on: https://cl.tvl.fyi/c/depot/+/12667 Reviewed-by: flokli <flokli@flokli.de> Tested-by: BuildkiteCI
Diffstat (limited to 'users/edef/weave/Cargo.toml')
| -rw-r--r-- | users/edef/weave/Cargo.toml | 3 |
1 files changed, 1 insertions, 2 deletions
diff --git a/users/edef/weave/Cargo.toml b/users/edef/weave/Cargo.toml index 69cf3cf7effd..72a205f66914 100644 --- a/users/edef/weave/Cargo.toml +++ b/users/edef/weave/Cargo.toml @@ -13,8 +13,7 @@ hashbrown = "0.14.3" nix-compat = { version = "0.1.0", path = "../../../tvix/nix-compat" } safer_owning_ref = "0.5.0" rayon = "1.8.1" -tokio = { version = "1.36.0", features = ["full"] } [dependencies.polars] version = "0.36.2" -features = ["parquet"] +features = ["parquet", "lazy", "streaming"] |