From 907ecff999e9f5e8cffbc1b6ab0edc97e672c833 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Thu, 14 Mar 2024 15:08:05 +0200 Subject: feat(nix-compat/wire): add low-level wire format primitives code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This brings some initial Nix wire format parsing code, used in the nix daemon protocol, remote store/builder protocol, as well as the NAR format itself (note we already have more specialized code for the last one). Thanks to embr, this code already exists, in https://codeberg.org/gorgon/gorgon/src/branch/main/nix-daemon/src/wire.rs, and we can vendor it into here, as EUPL is compatible with GPL (in that direction). The code uses the tokio::io Reader and Writer traits, not the ones from the `futures` crate, as they provide some more convenient `read_u64_le` functions. More application-specific parsing code, as well as code to read strings, or bytes are left out for now, as we want to be be more restrictive w.r.t allowed max sizes, and need to parse bytes, not strings. The code slightly diverges, as we have clippy looped into CI. `Ok(…?)` can be turned into just the inner expression, and some .and_then can be expressed in a simpler fashion. Change-Id: Ie3adcb485e9d66786673b1962a08d4e5df3781d9 Reviewed-on: https://cl.tvl.fyi/c/depot/+/11148 Autosubmit: flokli Tested-by: BuildkiteCI Reviewed-by: picnoir picnoir --- tvix/nix-compat/Cargo.toml | 8 +++- tvix/nix-compat/src/lib.rs | 1 + tvix/nix-compat/src/wire/mod.rs | 5 +++ tvix/nix-compat/src/wire/primitive.rs | 75 +++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 tvix/nix-compat/src/wire/mod.rs create mode 100644 tvix/nix-compat/src/wire/primitive.rs (limited to 'tvix/nix-compat') diff --git a/tvix/nix-compat/Cargo.toml b/tvix/nix-compat/Cargo.toml index c4672ff9fb..181eb94289 100644 --- a/tvix/nix-compat/Cargo.toml +++ b/tvix/nix-compat/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -async = ["futures-util"] +async = ["futures-util", "tokio"] [dependencies] bitflags = "2.4.1" @@ -22,6 +22,11 @@ serde_json = "1.0" sha2 = "0.10.6" thiserror = "1.0.38" +[dependencies.tokio] +optional = true +version = "1.32.0" +features = ["io-util", "macros"] + [dev-dependencies] futures = { version = "0.3.30", default-features = false, features = ["executor"] } lazy_static = "1.4.0" @@ -30,6 +35,7 @@ test-case = "3.3.1" criterion = { version = "0.5", features = ["html_reports"] } hex-literal = "0.4.1" pretty_assertions = "1.4.0" +tokio-test = "0.4.3" zstd = "^0.13.0" [dev-dependencies.test-generator] diff --git a/tvix/nix-compat/src/lib.rs b/tvix/nix-compat/src/lib.rs index dd161cc1f9..60dcbdf25c 100644 --- a/tvix/nix-compat/src/lib.rs +++ b/tvix/nix-compat/src/lib.rs @@ -5,3 +5,4 @@ pub mod narinfo; pub mod nixbase32; pub mod nixhash; pub mod store_path; +mod wire; diff --git a/tvix/nix-compat/src/wire/mod.rs b/tvix/nix-compat/src/wire/mod.rs new file mode 100644 index 0000000000..e0b184c78a --- /dev/null +++ b/tvix/nix-compat/src/wire/mod.rs @@ -0,0 +1,5 @@ +//! Module parsing and emitting the wire format used by Nix, both in the +//! nix-daemon protocol as well as in the NAR format. + +#[cfg(feature = "async")] +pub mod primitive; diff --git a/tvix/nix-compat/src/wire/primitive.rs b/tvix/nix-compat/src/wire/primitive.rs new file mode 100644 index 0000000000..54f00b15a0 --- /dev/null +++ b/tvix/nix-compat/src/wire/primitive.rs @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: 2023 embr +// +// SPDX-License-Identifier: EUPL-1.2 + +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; + +#[allow(dead_code)] +/// Read a u64 from the stream (little endian). +pub async fn read_u64(r: &mut R) -> std::io::Result { + r.read_u64_le().await +} + +#[allow(dead_code)] +/// Write a u64 from the stream (little endian). +pub async fn write_u64(w: &mut W, v: u64) -> std::io::Result<()> { + w.write_u64_le(v).await +} + +#[allow(dead_code)] +/// Read a boolean from the stream, encoded as u64 (>0 is true). +pub async fn read_bool(r: &mut R) -> std::io::Result { + Ok(read_u64(r).await? > 0) +} + +#[allow(dead_code)] +/// Write a boolean to the stream, encoded as u64 (>0 is true). +pub async fn write_bool(w: &mut W, v: bool) -> std::io::Result<()> { + write_u64(w, if v { 1u64 } else { 0u64 }).await +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio_test::io::Builder; + + // Integers. + #[tokio::test] + async fn test_read_u64() { + let mut mock = Builder::new().read(&1234567890u64.to_le_bytes()).build(); + assert_eq!(1234567890u64, read_u64(&mut mock).await.unwrap()); + } + #[tokio::test] + async fn test_write_u64() { + let mut mock = Builder::new().write(&1234567890u64.to_le_bytes()).build(); + write_u64(&mut mock, 1234567890).await.unwrap(); + } + + // Booleans. + #[tokio::test] + async fn test_read_bool_0() { + let mut mock = Builder::new().read(&0u64.to_le_bytes()).build(); + assert!(!read_bool(&mut mock).await.unwrap()); + } + #[tokio::test] + async fn test_read_bool_1() { + let mut mock = Builder::new().read(&1u64.to_le_bytes()).build(); + assert!(read_bool(&mut mock).await.unwrap()); + } + #[tokio::test] + async fn test_read_bool_2() { + let mut mock = Builder::new().read(&2u64.to_le_bytes()).build(); + assert!(read_bool(&mut mock).await.unwrap()); + } + + #[tokio::test] + async fn test_write_bool_false() { + let mut mock = Builder::new().write(&0u64.to_le_bytes()).build(); + write_bool(&mut mock, false).await.unwrap(); + } + #[tokio::test] + async fn test_write_bool_true() { + let mut mock = Builder::new().write(&1u64.to_le_bytes()).build(); + write_bool(&mut mock, true).await.unwrap(); + } +} -- cgit 1.4.1