diff options
| -rw-r--r-- | tvix/store/Cargo.lock | 199 | ||||
| -rw-r--r-- | tvix/store/Cargo.toml | 1 | ||||
| -rw-r--r-- | tvix/store/src/lib.rs | 3 | ||||
| -rw-r--r-- | tvix/store/src/main.rs | 2 | ||||
| -rw-r--r-- | tvix/store/src/proto.rs | 105 |
5 files changed, 307 insertions, 3 deletions
diff --git a/tvix/store/Cargo.lock b/tvix/store/Cargo.lock index b5b2fb9e3e62..215579aac53e 100644 --- a/tvix/store/Cargo.lock +++ b/tvix/store/Cargo.lock @@ -9,6 +9,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" [[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" + +[[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -21,18 +33,118 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] +name = "blake3" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a08e53fc5a564bb15bfe6fae56bd71522205f1f91893f9c0116edad6496c183f" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "digest", + "rayon", +] + +[[package]] +name = "block-buffer" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +dependencies = [ + "generic-array", +] + +[[package]] name = "bytes" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" [[package]] +name = "cc" +version = "1.0.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a284da2e6fe2092f2353e51713435363112dfd60030e22add80be333fb928f" + +[[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "crossbeam-channel" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f916dfc5d356b0ed9dae65f1db9fc9770aa2851d2662b988ccf4fe3516e86348" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] name = "either" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -54,6 +166,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] +name = "generic-array" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -66,6 +188,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" [[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] name = "indexmap" version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -115,12 +246,31 @@ dependencies = [ ] [[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] name = "multimap" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] +name = "num_cpus" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] name = "once_cell" version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -220,6 +370,30 @@ dependencies = [ ] [[package]] +name = "rayon" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + +[[package]] name = "redox_syscall" version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -253,6 +427,18 @@ dependencies = [ ] [[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + +[[package]] name = "syn" version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -281,17 +467,30 @@ dependencies = [ name = "tvix-store" version = "0.1.0" dependencies = [ + "blake3", "prost", "prost-build", ] [[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + +[[package]] name = "unicode-ident" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" [[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] name = "which" version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" diff --git a/tvix/store/Cargo.toml b/tvix/store/Cargo.toml index 1b758be64b72..c807da760ab5 100644 --- a/tvix/store/Cargo.toml +++ b/tvix/store/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] +blake3 = { version = "1.3.1", features = ["rayon", "std"] } prost = "0.11.2" [build-dependencies] diff --git a/tvix/store/src/lib.rs b/tvix/store/src/lib.rs deleted file mode 100644 index 0d8c560e22a8..000000000000 --- a/tvix/store/src/lib.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod protos { - include!(concat!(env!("OUT_DIR"), "/tvix.store.v1.rs")); -} diff --git a/tvix/store/src/main.rs b/tvix/store/src/main.rs index e7a11a969c03..fd11f3c1c9c1 100644 --- a/tvix/store/src/main.rs +++ b/tvix/store/src/main.rs @@ -1,3 +1,5 @@ +mod proto; + fn main() { println!("Hello, world!"); } diff --git a/tvix/store/src/proto.rs b/tvix/store/src/proto.rs new file mode 100644 index 000000000000..f819d7b4fc8f --- /dev/null +++ b/tvix/store/src/proto.rs @@ -0,0 +1,105 @@ +use prost::Message; + +include!(concat!(env!("OUT_DIR"), "/tvix.store.v1.rs")); + +impl Directory { + // The size of a directory is the number of all regular and symlink elements, + // the number of directory elements, and their size fields. + pub fn size(&self) -> u32 { + self.files.len() as u32 + + self.symlinks.len() as u32 + + self + .directories + .iter() + .fold(0, |acc: u32, e| (acc + 1 + e.size) as u32) + } + + pub fn digest(&self) -> Vec<u8> { + let mut hasher = blake3::Hasher::new(); + + hasher.update(&self.encode_to_vec()).finalize().as_bytes()[..].to_vec() + } +} + +#[cfg(test)] +mod tests { + use super::{Directory, DirectoryNode, FileNode, SymlinkNode}; + + #[test] + fn test_directory_size() { + { + let d = Directory { + directories: vec![], + files: vec![], + symlinks: vec![], + }; + assert_eq!(d.size(), 0); + } + { + let d = Directory { + directories: vec![DirectoryNode { + name: String::from("foo"), + digest: vec![], + size: 0, + }], + files: vec![], + symlinks: vec![], + }; + assert_eq!(d.size(), 1); + } + { + let d = Directory { + directories: vec![DirectoryNode { + name: String::from("foo"), + digest: vec![], + size: 4, + }], + files: vec![], + symlinks: vec![], + }; + assert_eq!(d.size(), 5); + } + { + let d = Directory { + directories: vec![], + files: vec![FileNode { + name: String::from("foo"), + digest: vec![], + size: 42, + executable: false, + }], + symlinks: vec![], + }; + assert_eq!(d.size(), 1); + } + { + let d = Directory { + directories: vec![], + files: vec![], + symlinks: vec![SymlinkNode { + name: String::from("foo"), + target: String::from("bar"), + }], + }; + assert_eq!(d.size(), 1); + } + } + + #[test] + fn test_digest() { + let d = Directory { + directories: vec![], + files: vec![], + symlinks: vec![], + }; + + assert_eq!( + d.digest(), + vec![ + 0xaf, 0x13, 0x49, 0xb9, 0xf5, 0xf9, 0xa1, 0xa6, 0xa0, 0x40, 0x4d, 0xea, 0x36, 0xdc, + 0xc9, 0x49, 0x9b, 0xcb, 0x25, 0xc9, 0xad, 0xc1, 0x12, 0xb7, 0xcc, 0x9a, 0x93, 0xca, + 0xe4, 0x1f, 0x32, 0x62 + ] + ) + } +} |