diff options
Diffstat (limited to 'tvix/store')
| -rw-r--r-- | tvix/store/Cargo.toml | 3 | ||||
| -rw-r--r-- | tvix/store/src/lib.rs | 2 | ||||
| -rw-r--r-- | tvix/store/src/nixbase32.rs | 167 | ||||
| -rw-r--r-- | tvix/store/src/proto.rs | 2 | ||||
| -rw-r--r-- | tvix/store/src/sled_path_info_service.rs | 2 | ||||
| -rw-r--r-- | tvix/store/src/store_path.rs | 198 | ||||
| -rw-r--r-- | tvix/store/src/tests/pathinfo.rs | 6 |
7 files changed, 6 insertions, 374 deletions
diff --git a/tvix/store/Cargo.toml b/tvix/store/Cargo.toml index 151d985c82eb..769b496235d6 100644 --- a/tvix/store/Cargo.toml +++ b/tvix/store/Cargo.toml @@ -14,9 +14,10 @@ lazy_static = "1.4.0" clap = { version = "4.0", features = ["derive", "env"] } prost = "0.11.2" sled = { version = "0.34.7", features = ["compression"] } +nix-compat = { path = "../nix-compat" } thiserror = "1.0.38" -tokio = { version = "1.23.0", features = ["rt-multi-thread"] } tokio-stream = "0.1.11" +tokio = { version = "1.23.0", features = ["rt-multi-thread"] } tonic = "0.8.2" tracing-subscriber = "0.3.16" tracing = "0.1.37" diff --git a/tvix/store/src/lib.rs b/tvix/store/src/lib.rs index d0e505733a74..2c7f4887ce9f 100644 --- a/tvix/store/src/lib.rs +++ b/tvix/store/src/lib.rs @@ -1,6 +1,4 @@ -pub mod nixbase32; pub mod proto; -pub mod store_path; pub mod dummy_blob_service; pub mod sled_directory_service; diff --git a/tvix/store/src/nixbase32.rs b/tvix/store/src/nixbase32.rs deleted file mode 100644 index 39aa4f1d5461..000000000000 --- a/tvix/store/src/nixbase32.rs +++ /dev/null @@ -1,167 +0,0 @@ -//! Implements the slightly odd "base32" encoding that's used in Nix. -//! -//! Nix uses a custom alphabet. Contrary to other implementations (RFC4648), -//! encoding to "nix base32" doesn't use any padding, and reads in characters -//! in reverse order. -//! -//! This is also the main reason why we can't use `data_encoding::Encoding` - -//! it gets things wrong if there normally would be a need for padding. - -use std::fmt::Write; - -use thiserror::Error; - -const ALPHABET: &'static [u8; 32] = b"0123456789abcdfghijklmnpqrsvwxyz"; - -/// Errors that can occur while decoding nixbase32-encoded data. -#[derive(Debug, Eq, PartialEq, Error)] -pub enum Nixbase32DecodeError { - #[error("character {0:x} not in alphabet")] - CharacterNotInAlphabet(u8), - #[error("nonzero carry")] - NonzeroCarry(), -} - -/// Returns encoded input -pub fn encode(input: &[u8]) -> String { - let output_len = encode_len(input.len()); - let mut output = String::with_capacity(output_len); - - if output_len > 0 { - for n in (0..=output_len - 1).rev() { - let b = n * 5; // bit offset within the entire input - let i = b / 8; // input byte index - let j = b % 8; // bit offset within that input byte - - let mut c = input[i] >> j; - if i + 1 < input.len() { - // we want to right shift, and discard shifted out bits (unchecked) - // To do this without panicing, we need to do the shifting in u16 - // and convert back to u8 afterwards. - c |= ((input[i + 1] as u16) << 8 - j as u16) as u8 - } - - output - .write_char(ALPHABET[(c & 0x1f) as usize] as char) - .unwrap(); - } - } - - output -} - -/// This maps a nixbase32-encoded character to its binary representation, which -/// is also the index of the character in the alphabet. -fn decode_char(encoded_char: &u8) -> Option<u8> { - Some(match encoded_char { - b'0'..=b'9' => encoded_char - b'0', - b'a'..=b'd' => encoded_char - b'a' + 10_u8, - b'f'..=b'n' => encoded_char - b'f' + 14_u8, - b'p'..=b's' => encoded_char - b'p' + 23_u8, - b'v'..=b'z' => encoded_char - b'v' + 27_u8, - _ => return None, - }) -} - -/// Returns decoded input -pub fn decode(input: &[u8]) -> Result<Vec<u8>, Nixbase32DecodeError> { - let output_len = decode_len(input.len()); - let mut output: Vec<u8> = vec![0x00; output_len]; - - // loop over all characters in reverse, and keep the iteration count in n. - for (n, c) in input.iter().rev().enumerate() { - match decode_char(c) { - None => return Err(Nixbase32DecodeError::CharacterNotInAlphabet(*c)), - Some(c_decoded) => { - let b = n * 5; - let i = b / 8; - let j = b % 8; - - let val = (c_decoded as u16).rotate_left(j as u32); - output[i] |= (val & 0x00ff) as u8; - let carry = ((val & 0xff00) >> 8) as u8; - - // if we're at the end of dst… - if i == output_len - 1 { - // but have a nonzero carry, the encoding is invalid. - if carry != 0 { - return Err(Nixbase32DecodeError::NonzeroCarry()); - } - } else { - output[i + 1] |= carry; - } - } - } - } - - Ok(output) -} - -/// Returns the decoded length of an input of length len. -pub fn decode_len(len: usize) -> usize { - return (len * 5) / 8; -} - -/// Returns the encoded length of an input of length len -pub fn encode_len(len: usize) -> usize { - if len == 0 { - return 0; - } - return (len * 8 - 1) / 5 + 1; -} - -#[cfg(test)] -mod tests { - use test_case::test_case; - - #[test_case("", vec![] ; "empty bytes")] - #[test_case("0z", vec![0x1f]; "one byte")] - #[test_case("00bgd045z0d4icpbc2yyz4gx48ak44la", vec![ - 0x8a, 0x12, 0x32, 0x15, 0x22, 0xfd, 0x91, 0xef, 0xbd, 0x60, 0xeb, 0xb2, 0x48, 0x1a, - 0xf8, 0x85, 0x80, 0xf6, 0x16, 0x00]; "store path")] - #[test_case("0c5b8vw40dy178xlpddw65q9gf1h2186jcc3p4swinwggbllv8mk", vec![ - 0xb3, 0xa2, 0x4d, 0xe9, 0x7a, 0x8f, 0xdb, 0xc8, 0x35, 0xb9, 0x83, 0x31, 0x69, 0x50, 0x10, 0x30, - 0xb8, 0x97, 0x70, 0x31, 0xbc, 0xb5, 0x4b, 0x3b, 0x3a, 0xc1, 0x37, 0x40, 0xf8, 0x46, 0xab, 0x30, - ]; "sha256")] - fn encode(enc: &str, dec: Vec<u8>) { - assert_eq!(enc, super::encode(&dec)); - } - - #[test_case("", Some(vec![]) ; "empty bytes")] - #[test_case("0z", Some(vec![0x1f]); "one byte")] - #[test_case("00bgd045z0d4icpbc2yyz4gx48ak44la", Some(vec![ - 0x8a, 0x12, 0x32, 0x15, 0x22, 0xfd, 0x91, 0xef, 0xbd, 0x60, 0xeb, 0xb2, 0x48, 0x1a, - 0xf8, 0x85, 0x80, 0xf6, 0x16, 0x00]); "store path")] - #[test_case("0c5b8vw40dy178xlpddw65q9gf1h2186jcc3p4swinwggbllv8mk", Some(vec![ - 0xb3, 0xa2, 0x4d, 0xe9, 0x7a, 0x8f, 0xdb, 0xc8, 0x35, 0xb9, 0x83, 0x31, 0x69, 0x50, 0x10, 0x30, - 0xb8, 0x97, 0x70, 0x31, 0xbc, 0xb5, 0x4b, 0x3b, 0x3a, 0xc1, 0x37, 0x40, 0xf8, 0x46, 0xab, 0x30, - ]); "sha256")] - // this is invalid encoding, because it encodes 10 1-bytes, so the carry - // would be 2 1-bytes - #[test_case("zz", None; "invalid encoding-1")] - // this is an even more specific example - it'd decode as 00000000 11 - #[test_case("c0", None; "invalid encoding-2")] - - fn decode(enc: &str, dec: Option<Vec<u8>>) { - match dec { - Some(dec) => { - // The decode needs to match what's passed in dec - assert_eq!(dec, super::decode(enc.as_bytes()).unwrap()); - } - None => { - // the decode needs to be an error - assert_eq!(true, super::decode(enc.as_bytes()).is_err()); - } - } - } - - #[test] - fn encode_len() { - assert_eq!(super::encode_len(20), 32) - } - - #[test] - fn decode_len() { - assert_eq!(super::decode_len(32), 20) - } -} diff --git a/tvix/store/src/proto.rs b/tvix/store/src/proto.rs index 0c4a894c23b5..fc5db4446d5e 100644 --- a/tvix/store/src/proto.rs +++ b/tvix/store/src/proto.rs @@ -5,7 +5,7 @@ use thiserror::Error; use prost::Message; -use crate::store_path::{ParseStorePathError, StorePath}; +use nix_compat::store_path::{ParseStorePathError, StorePath}; tonic::include_proto!("tvix.store.v1"); diff --git a/tvix/store/src/sled_path_info_service.rs b/tvix/store/src/sled_path_info_service.rs index 3215fd0e4730..6b9212a3c912 100644 --- a/tvix/store/src/sled_path_info_service.rs +++ b/tvix/store/src/sled_path_info_service.rs @@ -7,7 +7,7 @@ use crate::proto::CalculateNarResponse; use crate::proto::GetPathInfoRequest; use crate::proto::Node; use crate::proto::PathInfo; -use crate::store_path::DIGEST_SIZE; +use nix_compat::store_path::DIGEST_SIZE; use tonic::{Request, Response, Result, Status}; use tracing::{instrument, warn}; diff --git a/tvix/store/src/store_path.rs b/tvix/store/src/store_path.rs deleted file mode 100644 index 5032a73fb19b..000000000000 --- a/tvix/store/src/store_path.rs +++ /dev/null @@ -1,198 +0,0 @@ -use crate::nixbase32::{self, Nixbase32DecodeError}; -use std::fmt; -use thiserror::Error; - -pub const DIGEST_SIZE: usize = 20; -// lazy_static doesn't allow us to call NIXBASE32.encode_len(), so we ran it -// manually and have an assert in the tests. -pub const ENCODED_DIGEST_SIZE: usize = 32; - -// The store dir prefix, without trailing slash. -// That's usually where the Nix store is mounted at. -pub const STORE_DIR: &str = "/nix/store"; -pub const STORE_DIR_WITH_SLASH: &str = "/nix/store/"; - -/// Errors that can occur during the validation of name characters. -#[derive(Debug, PartialEq, Eq, Error)] -pub enum ParseStorePathError { - #[error("Dash is missing between hash and name")] - MissingDash(), - #[error("Hash encoding is invalid: {0}")] - InvalidHashEncoding(Nixbase32DecodeError), - #[error("Invalid name: {0}")] - InvalidName(String), - #[error("Tried to parse an absolute path which was missing the store dir prefix.")] - MissingStoreDir(), -} - -/// Represents a path in the Nix store (a direct child of [STORE_DIR]). -/// -/// It starts with a digest (20 bytes), [struct@NIXBASE32]-encoded, followed by -/// a `-`, and ends with a `name`, which is a string, consisting only of ASCCI -/// alphanumeric characters, or one of the following characters: `-`, `_`, `.`, -/// `+`, `?`, `=`. -/// -/// The name is usually used to describe the pname and version of a package. -/// Derivations paths can also be represented as store paths, they end -/// with .drv. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct StorePath { - pub digest: [u8; DIGEST_SIZE], - pub name: String, -} - -impl StorePath { - pub fn from_string(s: &str) -> Result<StorePath, ParseStorePathError> { - // the whole string needs to be at least: - // - // - 32 characters (encoded hash) - // - 1 dash - // - 1 character for the name - if s.len() < ENCODED_DIGEST_SIZE + 2 { - return Err(ParseStorePathError::InvalidName("".to_string())); - } - - let digest = match nixbase32::decode(s[..ENCODED_DIGEST_SIZE].as_bytes()) { - Ok(decoded) => decoded, - Err(decoder_error) => { - return Err(ParseStorePathError::InvalidHashEncoding(decoder_error)) - } - }; - - if s.as_bytes()[ENCODED_DIGEST_SIZE] != b'-' { - return Err(ParseStorePathError::MissingDash()); - } - - StorePath::validate_name(&s[ENCODED_DIGEST_SIZE + 2..])?; - - Ok(StorePath { - name: s[ENCODED_DIGEST_SIZE + 1..].to_string(), - digest: digest.try_into().expect("size is known"), - }) - } - - /// Construct a [StorePath] from an absolute store path string. - /// That is a string starting with the store prefix (/nix/store) - pub fn from_absolute_path(s: &str) -> Result<StorePath, ParseStorePathError> { - match s.strip_prefix(STORE_DIR_WITH_SLASH) { - Some(s_stripped) => Self::from_string(s_stripped), - None => Err(ParseStorePathError::MissingStoreDir()), - } - } - - /// Converts the [StorePath] to an absolute store path string. - /// That is a string starting with the store prefix (/nix/store) - pub fn to_absolute_path(&self) -> String { - format!("{}/{}", STORE_DIR, self) - } - - /// Checks a given &str to match the restrictions for store path names. - pub fn validate_name(s: &str) -> Result<(), ParseStorePathError> { - for c in s.chars() { - if c.is_ascii_alphanumeric() - || c == '-' - || c == '_' - || c == '.' - || c == '+' - || c == '?' - || c == '=' - { - continue; - } - - return Err(ParseStorePathError::InvalidName(s.to_string())); - } - - Ok(()) - } -} - -impl fmt::Display for StorePath { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}-{}", nixbase32::encode(&self.digest), self.name) - } -} - -#[cfg(test)] -mod tests { - use crate::nixbase32; - use crate::store_path::{DIGEST_SIZE, ENCODED_DIGEST_SIZE}; - - use super::{ParseStorePathError, StorePath}; - - #[test] - fn encoded_digest_size() { - assert_eq!(ENCODED_DIGEST_SIZE, nixbase32::encode_len(DIGEST_SIZE)); - } - - #[test] - fn happy_path() { - let example_nix_path_str = - "00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432"; - let nixpath = - StorePath::from_string(&example_nix_path_str).expect("Error parsing example string"); - - let expected_digest: [u8; DIGEST_SIZE] = [ - 0x8a, 0x12, 0x32, 0x15, 0x22, 0xfd, 0x91, 0xef, 0xbd, 0x60, 0xeb, 0xb2, 0x48, 0x1a, - 0xf8, 0x85, 0x80, 0xf6, 0x16, 0x00, - ]; - - assert_eq!("net-tools-1.60_p20170221182432", nixpath.name); - assert_eq!(nixpath.digest, expected_digest); - - assert_eq!(example_nix_path_str, nixpath.to_string()) - } - - #[test] - fn invalid_hash_length() { - StorePath::from_string("00bgd045z0d4icpbc2yy-net-tools-1.60_p20170221182432") - .expect_err("No error raised."); - } - - #[test] - fn invalid_encoding_hash() { - StorePath::from_string("00bgd045z0d4icpbc2yyz4gx48aku4la-net-tools-1.60_p20170221182432") - .expect_err("No error raised."); - } - - #[test] - fn more_than_just_the_bare_nix_store_path() { - StorePath::from_string( - "00bgd045z0d4icpbc2yyz4gx48aku4la-net-tools-1.60_p20170221182432/bin/arp", - ) - .expect_err("No error raised."); - } - - #[test] - fn no_dash_between_hash_and_name() { - StorePath::from_string("00bgd045z0d4icpbc2yyz4gx48ak44lanet-tools-1.60_p20170221182432") - .expect_err("No error raised."); - } - - #[test] - fn absolute_path() { - let example_nix_path_str = - "00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432"; - let nixpath_expected = StorePath::from_string(&example_nix_path_str).expect("must parse"); - - let nixpath_actual = StorePath::from_absolute_path( - "/nix/store/00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432", - ) - .expect("must parse"); - - assert_eq!(nixpath_expected, nixpath_actual); - - assert_eq!( - "/nix/store/00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432", - nixpath_actual.to_absolute_path(), - ); - } - - #[test] - fn absolute_path_missing_prefix() { - assert_eq!( - ParseStorePathError::MissingStoreDir(), - StorePath::from_absolute_path("foobar-123").expect_err("must fail") - ); - } -} diff --git a/tvix/store/src/tests/pathinfo.rs b/tvix/store/src/tests/pathinfo.rs index a278cf5e8eb0..35a2771c300a 100644 --- a/tvix/store/src/tests/pathinfo.rs +++ b/tvix/store/src/tests/pathinfo.rs @@ -1,8 +1,6 @@ -use crate::{ - proto::{self, Node, PathInfo, ValidatePathInfoError}, - store_path::{ParseStorePathError, StorePath}, -}; +use crate::proto::{self, Node, PathInfo, ValidatePathInfoError}; use lazy_static::lazy_static; +use nix_compat::store_path::{ParseStorePathError, StorePath}; use test_case::test_case; lazy_static! { |