diff options
Diffstat (limited to 'tvix')
| -rw-r--r-- | tvix/Cargo.lock | 173 | ||||
| -rw-r--r-- | tvix/Cargo.nix | 631 | ||||
| -rw-r--r-- | tvix/store/Cargo.toml | 2 | ||||
| -rw-r--r-- | tvix/store/src/pathinfoservice/from_addr.rs | 32 | ||||
| -rw-r--r-- | tvix/store/src/pathinfoservice/mod.rs | 2 | ||||
| -rw-r--r-- | tvix/store/src/pathinfoservice/nix_http.rs | 213 |
6 files changed, 1043 insertions, 10 deletions
diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock index 72da9607466f..2b05ed744e39 100644 --- a/tvix/Cargo.lock +++ b/tvix/Cargo.lock @@ -694,6 +694,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + +[[package]] name = "endian-type" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1065,6 +1074,20 @@ dependencies = [ ] [[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + +[[package]] name = "hyper-timeout" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1141,6 +1164,12 @@ dependencies = [ ] [[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] name = "is-terminal" version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1299,6 +1328,17 @@ dependencies = [ ] [[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] name = "matchit" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1634,6 +1674,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] name = "plotters" version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1960,6 +2006,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" [[package]] +name = "reqwest" +version = "0.11.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-rustls", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "system-configuration", + "tokio", + "tokio-rustls", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", + "winreg", +] + +[[package]] name = "ring" version = "0.16.20" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2210,6 +2298,18 @@ dependencies = [ ] [[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] name = "sha2" version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2392,6 +2492,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] name = "tabwriter" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2977,6 +3098,7 @@ dependencies = [ "pin-project-lite", "prost", "prost-build", + "reqwest", "sha2", "sled", "tempfile", @@ -3002,6 +3124,7 @@ dependencies = [ "vm-memory", "vmm-sys-util", "walkdir", + "xz2", ] [[package]] @@ -3223,6 +3346,18 @@ dependencies = [ ] [[package]] +name = "wasm-bindgen-futures" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] name = "wasm-bindgen-macro" version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3252,6 +3387,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] +name = "wasm-streams" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] name = "web-sys" version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3262,6 +3410,12 @@ dependencies = [ ] [[package]] +name = "webpki-roots" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" + +[[package]] name = "which" version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3436,6 +3590,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] name = "wu-manber" version = "0.1.0" source = "git+https://github.com/tvlfyi/wu-manber.git#0d5b22bea136659f7de60b102a7030e0daaa503d" @@ -3447,6 +3611,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bab77e97b50aee93da431f2cee7cd0f43b4d1da3c408042f2d7d164187774f0a" [[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + +[[package]] name = "yansi" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix index 261ff74c99b2..98a0e67a6272 100644 --- a/tvix/Cargo.nix +++ b/tvix/Cargo.nix @@ -2015,6 +2015,29 @@ rec { }; resolvedDefaultFeatures = [ "default" "use_std" ]; }; + "encoding_rs" = rec { + crateName = "encoding_rs"; + version = "0.8.33"; + edition = "2018"; + sha256 = "1qa5k4a0ipdrxq4xg9amms9r9pnnfn7nfh2i9m3mw0ka563b6s3j"; + authors = [ + "Henri Sivonen <hsivonen@hsivonen.fi>" + ]; + dependencies = [ + { + name = "cfg-if"; + packageId = "cfg-if"; + } + ]; + features = { + "default" = [ "alloc" ]; + "fast-legacy-encode" = [ "fast-hangul-encode" "fast-hanja-encode" "fast-kanji-encode" "fast-gb-hanzi-encode" "fast-big5-hanzi-encode" ]; + "packed_simd" = [ "dep:packed_simd" ]; + "serde" = [ "dep:serde" ]; + "simd-accel" = [ "packed_simd" "packed_simd/into_bits" ]; + }; + resolvedDefaultFeatures = [ "alloc" "default" ]; + }; "endian-type" = rec { crateName = "endian-type"; version = "0.1.2"; @@ -3171,6 +3194,75 @@ rec { }; resolvedDefaultFeatures = [ "client" "default" "full" "h2" "http1" "http2" "runtime" "server" "socket2" "stream" "tcp" ]; }; + "hyper-rustls" = rec { + crateName = "hyper-rustls"; + version = "0.24.2"; + edition = "2021"; + sha256 = "1475j4a2nczz4aajzzsq3hpwg1zacmzbqg393a14j80ff8izsgpc"; + dependencies = [ + { + name = "futures-util"; + packageId = "futures-util"; + usesDefaultFeatures = false; + } + { + name = "http"; + packageId = "http"; + } + { + name = "hyper"; + packageId = "hyper"; + usesDefaultFeatures = false; + features = [ "client" ]; + } + { + name = "rustls"; + packageId = "rustls"; + usesDefaultFeatures = false; + } + { + name = "tokio"; + packageId = "tokio"; + } + { + name = "tokio-rustls"; + packageId = "tokio-rustls"; + usesDefaultFeatures = false; + } + ]; + devDependencies = [ + { + name = "hyper"; + packageId = "hyper"; + features = [ "full" ]; + } + { + name = "rustls"; + packageId = "rustls"; + usesDefaultFeatures = false; + features = [ "tls12" ]; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "io-std" "macros" "net" "rt-multi-thread" ]; + } + ]; + features = { + "acceptor" = [ "hyper/server" "tokio-runtime" ]; + "default" = [ "native-tokio" "http1" "tls12" "logging" "acceptor" ]; + "http1" = [ "hyper/http1" ]; + "http2" = [ "hyper/http2" ]; + "log" = [ "dep:log" ]; + "logging" = [ "log" "tokio-rustls/logging" "rustls/logging" ]; + "native-tokio" = [ "tokio-runtime" "rustls-native-certs" ]; + "rustls-native-certs" = [ "dep:rustls-native-certs" ]; + "tls12" = [ "tokio-rustls/tls12" "rustls/tls12" ]; + "tokio-runtime" = [ "hyper/runtime" ]; + "webpki-roots" = [ "dep:webpki-roots" ]; + "webpki-tokio" = [ "tokio-runtime" "webpki-roots" ]; + }; + }; "hyper-timeout" = rec { crateName = "hyper-timeout"; version = "0.4.1"; @@ -3422,6 +3514,24 @@ rec { }; resolvedDefaultFeatures = [ "close" "default" "hermit-abi" "libc" "windows-sys" ]; }; + "ipnet" = rec { + crateName = "ipnet"; + version = "2.9.0"; + edition = "2018"; + sha256 = "1hzrcysgwf0knf83ahb3535hrkw63mil88iqc6kjaryfblrqylcg"; + authors = [ + "Kris Price <kris@krisprice.nz>" + ]; + features = { + "default" = [ "std" ]; + "heapless" = [ "dep:heapless" ]; + "json" = [ "serde" "schemars" ]; + "schemars" = [ "dep:schemars" ]; + "ser_as_str" = [ "heapless" ]; + "serde" = [ "dep:serde" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "is-terminal" = rec { crateName = "is-terminal"; version = "0.4.7"; @@ -3875,6 +3985,32 @@ rec { }; resolvedDefaultFeatures = [ "std" ]; }; + "lzma-sys" = rec { + crateName = "lzma-sys"; + version = "0.1.20"; + edition = "2018"; + sha256 = "09sxp20waxyglgn3cjz8qjkspb3ryz2fwx4rigkwvrk46ymh9njz"; + authors = [ + "Alex Crichton <alex@alexcrichton.com>" + ]; + dependencies = [ + { + name = "libc"; + packageId = "libc"; + } + ]; + buildDependencies = [ + { + name = "cc"; + packageId = "cc"; + } + { + name = "pkg-config"; + packageId = "pkg-config"; + } + ]; + features = { }; + }; "matchit" = rec { crateName = "matchit"; version = "0.7.0"; @@ -4815,6 +4951,16 @@ rec { ]; }; + "pkg-config" = rec { + crateName = "pkg-config"; + version = "0.3.27"; + edition = "2015"; + sha256 = "0r39ryh1magcq4cz5g9x88jllsnxnhcqr753islvyk4jp9h2h1r6"; + authors = [ + "Alex Crichton <alex@alexcrichton.com>" + ]; + + }; "plotters" = rec { crateName = "plotters"; version = "0.3.4"; @@ -5776,6 +5922,275 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" "unicode" "unicode-age" "unicode-bool" "unicode-case" "unicode-gencat" "unicode-perl" "unicode-script" "unicode-segment" ]; }; + "reqwest" = rec { + crateName = "reqwest"; + version = "0.11.22"; + edition = "2018"; + sha256 = "0nx9mczsf11pcjicfpwad0l8drf2nn72dbpcvp42lv644s4djv04"; + authors = [ + "Sean McArthur <sean@seanmonstar.com>" + ]; + dependencies = [ + { + name = "base64"; + packageId = "base64"; + } + { + name = "bytes"; + packageId = "bytes"; + } + { + name = "encoding_rs"; + packageId = "encoding_rs"; + target = { target, features }: (!("wasm32" == target."arch")); + } + { + name = "futures-core"; + packageId = "futures-core"; + usesDefaultFeatures = false; + } + { + name = "futures-util"; + packageId = "futures-util"; + usesDefaultFeatures = false; + } + { + name = "h2"; + packageId = "h2"; + target = { target, features }: (!("wasm32" == target."arch")); + } + { + name = "http"; + packageId = "http"; + } + { + name = "http-body"; + packageId = "http-body"; + target = { target, features }: (!("wasm32" == target."arch")); + } + { + name = "hyper"; + packageId = "hyper"; + usesDefaultFeatures = false; + target = { target, features }: (!("wasm32" == target."arch")); + features = [ "tcp" "http1" "http2" "client" "runtime" ]; + } + { + name = "hyper-rustls"; + packageId = "hyper-rustls"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (!("wasm32" == target."arch")); + } + { + name = "ipnet"; + packageId = "ipnet"; + target = { target, features }: (!("wasm32" == target."arch")); + } + { + name = "js-sys"; + packageId = "js-sys"; + target = { target, features }: ("wasm32" == target."arch"); + } + { + name = "log"; + packageId = "log"; + target = { target, features }: (!("wasm32" == target."arch")); + } + { + name = "mime"; + packageId = "mime"; + target = { target, features }: (!("wasm32" == target."arch")); + } + { + name = "once_cell"; + packageId = "once_cell"; + target = { target, features }: (!("wasm32" == target."arch")); + } + { + name = "percent-encoding"; + packageId = "percent-encoding"; + target = { target, features }: (!("wasm32" == target."arch")); + } + { + name = "pin-project-lite"; + packageId = "pin-project-lite"; + target = { target, features }: (!("wasm32" == target."arch")); + } + { + name = "rustls"; + packageId = "rustls"; + optional = true; + target = { target, features }: (!("wasm32" == target."arch")); + features = [ "dangerous_configuration" ]; + } + { + name = "rustls-pemfile"; + packageId = "rustls-pemfile"; + optional = true; + target = { target, features }: (!("wasm32" == target."arch")); + } + { + name = "serde"; + packageId = "serde"; + } + { + name = "serde_json"; + packageId = "serde_json"; + optional = true; + } + { + name = "serde_json"; + packageId = "serde_json"; + target = { target, features }: ("wasm32" == target."arch"); + } + { + name = "serde_urlencoded"; + packageId = "serde_urlencoded"; + } + { + name = "system-configuration"; + packageId = "system-configuration"; + target = { target, features }: ("macos" == target."os"); + } + { + name = "tokio"; + packageId = "tokio"; + usesDefaultFeatures = false; + target = { target, features }: (!("wasm32" == target."arch")); + features = [ "net" "time" ]; + } + { + name = "tokio-rustls"; + packageId = "tokio-rustls"; + optional = true; + target = { target, features }: (!("wasm32" == target."arch")); + } + { + name = "tokio-util"; + packageId = "tokio-util"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (!("wasm32" == target."arch")); + features = [ "codec" "io" ]; + } + { + name = "tower-service"; + packageId = "tower-service"; + } + { + name = "url"; + packageId = "url"; + } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + target = { target, features }: ("wasm32" == target."arch"); + } + { + name = "wasm-bindgen-futures"; + packageId = "wasm-bindgen-futures"; + target = { target, features }: ("wasm32" == target."arch"); + } + { + name = "wasm-streams"; + packageId = "wasm-streams"; + optional = true; + target = { target, features }: ("wasm32" == target."arch"); + } + { + name = "web-sys"; + packageId = "web-sys"; + target = { target, features }: ("wasm32" == target."arch"); + features = [ "AbortController" "AbortSignal" "Headers" "Request" "RequestInit" "RequestMode" "Response" "Window" "FormData" "Blob" "BlobPropertyBag" "ServiceWorkerGlobalScope" "RequestCredentials" "File" "ReadableStream" ]; + } + { + name = "webpki-roots"; + packageId = "webpki-roots"; + optional = true; + target = { target, features }: (!("wasm32" == target."arch")); + } + { + name = "winreg"; + packageId = "winreg"; + target = { target, features }: (target."windows" or false); + } + ]; + devDependencies = [ + { + name = "hyper"; + packageId = "hyper"; + usesDefaultFeatures = false; + target = { target, features }: (!("wasm32" == target."arch")); + features = [ "tcp" "stream" "http1" "http2" "client" "server" "runtime" ]; + } + { + name = "serde"; + packageId = "serde"; + target = { target, features }: (!("wasm32" == target."arch")); + features = [ "derive" ]; + } + { + name = "tokio"; + packageId = "tokio"; + usesDefaultFeatures = false; + target = { target, features }: (!("wasm32" == target."arch")); + features = [ "macros" "rt-multi-thread" ]; + } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + target = { target, features }: ("wasm32" == target."arch"); + features = [ "serde-serialize" ]; + } + ]; + features = { + "__rustls" = [ "hyper-rustls" "tokio-rustls" "rustls" "__tls" "rustls-pemfile" ]; + "async-compression" = [ "dep:async-compression" ]; + "blocking" = [ "futures-util/io" "tokio/rt-multi-thread" "tokio/sync" ]; + "brotli" = [ "async-compression" "async-compression/brotli" "tokio-util" ]; + "cookie_crate" = [ "dep:cookie_crate" ]; + "cookie_store" = [ "dep:cookie_store" ]; + "cookies" = [ "cookie_crate" "cookie_store" ]; + "default" = [ "default-tls" ]; + "default-tls" = [ "hyper-tls" "native-tls-crate" "__tls" "tokio-native-tls" ]; + "deflate" = [ "async-compression" "async-compression/zlib" "tokio-util" ]; + "futures-channel" = [ "dep:futures-channel" ]; + "gzip" = [ "async-compression" "async-compression/gzip" "tokio-util" ]; + "h3" = [ "dep:h3" ]; + "h3-quinn" = [ "dep:h3-quinn" ]; + "http3" = [ "rustls-tls-manual-roots" "h3" "h3-quinn" "quinn" "futures-channel" ]; + "hyper-rustls" = [ "dep:hyper-rustls" ]; + "hyper-tls" = [ "dep:hyper-tls" ]; + "json" = [ "serde_json" ]; + "mime_guess" = [ "dep:mime_guess" ]; + "multipart" = [ "mime_guess" ]; + "native-tls" = [ "default-tls" ]; + "native-tls-alpn" = [ "native-tls" "native-tls-crate/alpn" ]; + "native-tls-crate" = [ "dep:native-tls-crate" ]; + "native-tls-vendored" = [ "native-tls" "native-tls-crate/vendored" ]; + "quinn" = [ "dep:quinn" ]; + "rustls" = [ "dep:rustls" ]; + "rustls-native-certs" = [ "dep:rustls-native-certs" ]; + "rustls-pemfile" = [ "dep:rustls-pemfile" ]; + "rustls-tls" = [ "rustls-tls-webpki-roots" ]; + "rustls-tls-manual-roots" = [ "__rustls" ]; + "rustls-tls-native-roots" = [ "rustls-native-certs" "__rustls" ]; + "rustls-tls-webpki-roots" = [ "webpki-roots" "__rustls" ]; + "serde_json" = [ "dep:serde_json" ]; + "socks" = [ "tokio-socks" ]; + "stream" = [ "tokio/fs" "tokio-util" "wasm-streams" ]; + "tokio-native-tls" = [ "dep:tokio-native-tls" ]; + "tokio-rustls" = [ "dep:tokio-rustls" ]; + "tokio-socks" = [ "dep:tokio-socks" ]; + "tokio-util" = [ "dep:tokio-util" ]; + "trust-dns" = [ "trust-dns-resolver" ]; + "trust-dns-resolver" = [ "dep:trust-dns-resolver" ]; + "wasm-streams" = [ "dep:wasm-streams" ]; + "webpki-roots" = [ "dep:webpki-roots" ]; + }; + resolvedDefaultFeatures = [ "__rustls" "__tls" "hyper-rustls" "rustls" "rustls-pemfile" "rustls-tls" "rustls-tls-webpki-roots" "stream" "tokio-rustls" "tokio-util" "wasm-streams" "webpki-roots" ]; + }; "ring" = rec { crateName = "ring"; version = "0.16.20"; @@ -6094,7 +6509,7 @@ rec { "read_buf" = [ "rustversion" ]; "rustversion" = [ "dep:rustversion" ]; }; - resolvedDefaultFeatures = [ "default" "log" "logging" "tls12" ]; + resolvedDefaultFeatures = [ "dangerous_configuration" "default" "log" "logging" "tls12" ]; }; "rustls-native-certs" = rec { crateName = "rustls-native-certs"; @@ -6577,6 +6992,34 @@ rec { }; resolvedDefaultFeatures = [ "serde" ]; }; + "serde_urlencoded" = rec { + crateName = "serde_urlencoded"; + version = "0.7.1"; + edition = "2018"; + sha256 = "1zgklbdaysj3230xivihs30qi5vkhigg323a9m62k8jwf4a1qjfk"; + authors = [ + "Anthony Ramine <n.oxyde@gmail.com>" + ]; + dependencies = [ + { + name = "form_urlencoded"; + packageId = "form_urlencoded"; + } + { + name = "itoa"; + packageId = "itoa"; + } + { + name = "ryu"; + packageId = "ryu"; + } + { + name = "serde"; + packageId = "serde"; + } + ]; + + }; "sha2" = rec { crateName = "sha2"; version = "0.10.6"; @@ -7072,6 +7515,50 @@ rec { "futures-core" = [ "dep:futures-core" ]; }; }; + "system-configuration" = rec { + crateName = "system-configuration"; + version = "0.5.1"; + edition = "2021"; + sha256 = "1rz0r30xn7fiyqay2dvzfy56cvaa3km74hnbz2d72p97bkf3lfms"; + authors = [ + "Mullvad VPN" + ]; + dependencies = [ + { + name = "bitflags"; + packageId = "bitflags 1.3.2"; + } + { + name = "core-foundation"; + packageId = "core-foundation"; + } + { + name = "system-configuration-sys"; + packageId = "system-configuration-sys"; + } + ]; + + }; + "system-configuration-sys" = rec { + crateName = "system-configuration-sys"; + version = "0.5.0"; + edition = "2021"; + sha256 = "1jckxvdr37bay3i9v52izgy52dg690x5xfg3hd394sv2xf4b2px7"; + authors = [ + "Mullvad VPN" + ]; + dependencies = [ + { + name = "core-foundation-sys"; + packageId = "core-foundation-sys"; + } + { + name = "libc"; + packageId = "libc"; + } + ]; + + }; "tabwriter" = rec { crateName = "tabwriter"; version = "1.2.1"; @@ -9140,6 +9627,12 @@ rec { packageId = "prost"; } { + name = "reqwest"; + packageId = "reqwest"; + usesDefaultFeatures = false; + features = [ "rustls-tls" "stream" ]; + } + { name = "sha2"; packageId = "sha2"; } @@ -9237,6 +9730,10 @@ rec { name = "walkdir"; packageId = "walkdir"; } + { + name = "xz2"; + packageId = "xz2"; + } ]; buildDependencies = [ { @@ -9812,6 +10309,39 @@ rec { }; resolvedDefaultFeatures = [ "spans" ]; }; + "wasm-bindgen-futures" = rec { + crateName = "wasm-bindgen-futures"; + version = "0.4.34"; + edition = "2018"; + sha256 = "0m0lnnnhs9ni4dn9vz74prsjz8bdcf8dvnznd5ljch5s279f06gj"; + authors = [ + "The wasm-bindgen Developers" + ]; + dependencies = [ + { + name = "cfg-if"; + packageId = "cfg-if"; + } + { + name = "js-sys"; + packageId = "js-sys"; + } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + } + { + name = "web-sys"; + packageId = "web-sys"; + target = { target, features }: (builtins.elem "atomics" targetFeatures); + features = [ "MessageEvent" "Worker" ]; + } + ]; + features = { + "futures-core" = [ "dep:futures-core" ]; + "futures-core-03-stream" = [ "futures-core" ]; + }; + }; "wasm-bindgen-macro" = rec { crateName = "wasm-bindgen-macro"; version = "0.2.84"; @@ -9884,6 +10414,48 @@ rec { ]; }; + "wasm-streams" = rec { + crateName = "wasm-streams"; + version = "0.3.0"; + edition = "2021"; + sha256 = "1iqa4kmhbsjj8k4q15i1x0x4p3xda0dhbg7zw51mydr4g129sq5l"; + type = [ "cdylib" "rlib" ]; + authors = [ + "Mattias Buelens <mattias@buelens.com>" + ]; + dependencies = [ + { + name = "futures-util"; + packageId = "futures-util"; + features = [ "io" "sink" ]; + } + { + name = "js-sys"; + packageId = "js-sys"; + } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + } + { + name = "wasm-bindgen-futures"; + packageId = "wasm-bindgen-futures"; + } + { + name = "web-sys"; + packageId = "web-sys"; + features = [ "AbortSignal" ]; + } + ]; + devDependencies = [ + { + name = "web-sys"; + packageId = "web-sys"; + features = [ "console" "AbortSignal" "Response" "ReadableStream" "Window" ]; + } + ]; + + }; "web-sys" = rec { crateName = "web-sys"; version = "0.3.61"; @@ -10339,7 +10911,14 @@ rec { "XrViewerPose" = [ "XrPose" ]; "XrWebGlLayer" = [ "EventTarget" "XrLayer" ]; }; - resolvedDefaultFeatures = [ "CanvasRenderingContext2d" "Crypto" "Document" "DomRect" "DomRectReadOnly" "Element" "EventTarget" "HtmlCanvasElement" "HtmlElement" "Node" "Window" ]; + resolvedDefaultFeatures = [ "AbortController" "AbortSignal" "Blob" "BlobPropertyBag" "CanvasRenderingContext2d" "Crypto" "Document" "DomRect" "DomRectReadOnly" "Element" "Event" "EventTarget" "File" "FormData" "Headers" "HtmlCanvasElement" "HtmlElement" "MessageEvent" "Node" "ReadableStream" "Request" "RequestCredentials" "RequestInit" "RequestMode" "Response" "ServiceWorkerGlobalScope" "Window" "Worker" "WorkerGlobalScope" ]; + }; + "webpki-roots" = rec { + crateName = "webpki-roots"; + version = "0.25.2"; + edition = "2018"; + sha256 = "1z13850xvsijjxxvzx1wq3m6pz78ih5q6wjcp7gpgwz4gfspn90l"; + }; "which" = rec { crateName = "which"; @@ -11008,7 +11587,7 @@ rec { "Win32_Web" = [ "Win32" ]; "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; }; - resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_NetworkManagement" "Win32_NetworkManagement_IpHelper" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_Memory" "Win32_System_Pipes" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_WindowsProgramming" "default" ]; + resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_NetworkManagement" "Win32_NetworkManagement_IpHelper" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_Memory" "Win32_System_Pipes" "Win32_System_Registry" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_Time" "Win32_System_WindowsProgramming" "default" ]; }; "windows-targets 0.42.2" = rec { crateName = "windows-targets"; @@ -11269,6 +11848,31 @@ rec { ]; }; + "winreg" = rec { + crateName = "winreg"; + version = "0.50.0"; + edition = "2018"; + sha256 = "1cddmp929k882mdh6i9f2as848f13qqna6czwsqzkh1pqnr5fkjj"; + authors = [ + "Igor Shaula <gentoo90@gmail.com>" + ]; + dependencies = [ + { + name = "cfg-if"; + packageId = "cfg-if"; + } + { + name = "windows-sys"; + packageId = "windows-sys 0.48.0"; + features = [ "Win32_Foundation" "Win32_System_Time" "Win32_System_Registry" "Win32_Security" "Win32_Storage_FileSystem" "Win32_System_Diagnostics_Debug" ]; + } + ]; + features = { + "chrono" = [ "dep:chrono" ]; + "serde" = [ "dep:serde" ]; + "serialization-serde" = [ "transactions" "serde" ]; + }; + }; "wu-manber" = rec { crateName = "wu-manber"; version = "0.1.0"; @@ -11296,6 +11900,27 @@ rec { ]; }; + "xz2" = rec { + crateName = "xz2"; + version = "0.1.7"; + edition = "2018"; + sha256 = "1qk7nzpblizvayyq4xzi4b0zacmmbqr6vb9fc0v1avyp17f4931q"; + authors = [ + "Alex Crichton <alex@alexcrichton.com>" + ]; + dependencies = [ + { + name = "lzma-sys"; + packageId = "lzma-sys"; + } + ]; + features = { + "futures" = [ "dep:futures" ]; + "static" = [ "lzma-sys/static" ]; + "tokio" = [ "tokio-io" "futures" ]; + "tokio-io" = [ "dep:tokio-io" ]; + }; + }; "yansi" = rec { crateName = "yansi"; version = "0.5.1"; diff --git a/tvix/store/Cargo.toml b/tvix/store/Cargo.toml index c55eb80f4dc3..d5ea10a5b70b 100644 --- a/tvix/store/Cargo.toml +++ b/tvix/store/Cargo.toml @@ -32,6 +32,8 @@ tvix-castore = { path = "../castore" } url = "2.4.0" walkdir = "2.4.0" async-recursion = "1.0.5" +reqwest = { version = "0.11.22", features = ["rustls-tls", "stream"], default-features = false } +xz2 = "0.1.7" [dependencies.fuse-backend-rs] optional = true diff --git a/tvix/store/src/pathinfoservice/from_addr.rs b/tvix/store/src/pathinfoservice/from_addr.rs index 0f1f8d5c96a6..e7faf20144bb 100644 --- a/tvix/store/src/pathinfoservice/from_addr.rs +++ b/tvix/store/src/pathinfoservice/from_addr.rs @@ -1,6 +1,9 @@ use crate::proto::path_info_service_client::PathInfoServiceClient; -use super::{GRPCPathInfoService, MemoryPathInfoService, PathInfoService, SledPathInfoService}; +use super::{ + GRPCPathInfoService, MemoryPathInfoService, NixHTTPPathInfoService, PathInfoService, + SledPathInfoService, +}; use std::sync::Arc; use tvix_castore::{blobservice::BlobService, directoryservice::DirectoryService, Error}; @@ -62,6 +65,16 @@ pub async fn from_addr( SledPathInfoService::new(url.path().into(), blob_service, directory_service) .map_err(|e| Error::StorageError(e.to_string()))?, )); + } else if url.scheme() == "nix+http" || url.scheme() == "nix+https" { + // Stringify the URL and remove the nix+ prefix. + // We can't use `url.set_scheme(rest)`, as it disallows + // setting something http(s) that previously wasn't. + let url = Url::parse(url.to_string().strip_prefix("nix+").unwrap()).unwrap(); + Arc::new(NixHTTPPathInfoService::new( + url, + blob_service, + directory_service, + )) } else if url.scheme().starts_with("grpc+") { // schemes starting with grpc+ go to the GRPCPathInfoService. // That's normally grpc+unix for unix sockets, and grpc+http(s) for the HTTP counterparts. @@ -113,6 +126,14 @@ mod tests { #[test_case("memory:///", false; "memory invalid root path")] /// This sets a memory url path to "/foo", which is invalid. #[test_case("memory:///foo", false; "memory invalid root path foo")] + /// Correct Scheme for the cache.nixos.org binary cache. + #[test_case("nix+https://cache.nixos.org", true; "correct nix+https")] + /// Correct Scheme for the cache.nixos.org binary cache (HTTP URL). + #[test_case("nix+http://cache.nixos.org", true; "correct nix+http")] + /// Correct Scheme for Nix HTTP Binary cache, with a subpath. + #[test_case("nix+http://192.0.2.1/foo", true; "correct nix http with subpath")] + /// Correct Scheme for Nix HTTP Binary cache, with a subpath and port. + #[test_case("nix+http://[::1]:8080/foo", true; "correct nix http with subpath and port")] /// Correct scheme to connect to a unix socket. #[test_case("grpc+unix:///path/to/somewhere", true; "grpc valid unix socket")] /// Correct scheme for unix socket, but setting a host too, which is invalid. @@ -127,11 +148,8 @@ mod tests { #[test_case("grpc+http://localhost/some-path", false; "grpc valid invalid host and path")] #[tokio::test] async fn test_from_addr_tokio(uri_str: &str, is_ok: bool) { - assert_eq!( - from_addr(uri_str, gen_blob_service(), gen_directory_service()) - .await - .is_ok(), - is_ok - ) + let resp = from_addr(uri_str, gen_blob_service(), gen_directory_service()).await; + + assert_eq!(resp.is_ok(), is_ok); } } diff --git a/tvix/store/src/pathinfoservice/mod.rs b/tvix/store/src/pathinfoservice/mod.rs index 3fde10179b36..5faa0900a0b0 100644 --- a/tvix/store/src/pathinfoservice/mod.rs +++ b/tvix/store/src/pathinfoservice/mod.rs @@ -1,6 +1,7 @@ mod from_addr; mod grpc; mod memory; +mod nix_http; mod sled; use futures::Stream; @@ -14,6 +15,7 @@ use crate::proto::PathInfo; pub use self::from_addr::from_addr; pub use self::grpc::GRPCPathInfoService; pub use self::memory::MemoryPathInfoService; +pub use self::nix_http::NixHTTPPathInfoService; pub use self::sled::SledPathInfoService; /// The base trait all PathInfo services need to implement. diff --git a/tvix/store/src/pathinfoservice/nix_http.rs b/tvix/store/src/pathinfoservice/nix_http.rs new file mode 100644 index 000000000000..49cd515ce913 --- /dev/null +++ b/tvix/store/src/pathinfoservice/nix_http.rs @@ -0,0 +1,213 @@ +use std::{ + io::{self, BufRead}, + pin::Pin, + sync::Arc, +}; + +use data_encoding::BASE64; +use futures::{Stream, TryStreamExt}; +use nix_compat::{narinfo::NarInfo, nixbase32}; +use reqwest::StatusCode; +use tonic::async_trait; +use tracing::{debug, instrument, warn}; +use tvix_castore::{ + blobservice::BlobService, directoryservice::DirectoryService, proto as castorepb, Error, +}; + +use crate::proto::PathInfo; + +use super::PathInfoService; + +/// NixHTTPPathInfoService acts as a bridge in between the Nix HTTP Binary cache +/// protocol provided by Nix binary caches such as cache.nixos.org, and the Tvix +/// Store Model. +/// It implements the [PathInfoService] trait in an interesting way: +/// Every [PathInfoService::get] fetches the .narinfo and referred NAR file, +/// inserting components into a [BlobService] and [DirectoryService], then +/// returning a [PathInfo] struct with the root. +/// +/// Due to this being quite a costly operation, clients are expected to layer +/// this service with store composition, so they're only ingested once. +/// +/// The client is expected to be (indirectly) using the same [BlobService] and +/// [DirectoryService], so able to fetch referred Directories and Blobs. +/// [PathInfoService::put] and [PathInfoService::nar] are not implemented and +/// return an error if called. +/// TODO: what about reading from nix-cache-info? +pub struct NixHTTPPathInfoService { + base_url: url::Url, + http_client: reqwest::Client, + + blob_service: Arc<dyn BlobService>, + directory_service: Arc<dyn DirectoryService>, +} + +impl NixHTTPPathInfoService { + pub fn new( + base_url: url::Url, + blob_service: Arc<dyn BlobService>, + directory_service: Arc<dyn DirectoryService>, + ) -> Self { + Self { + base_url, + http_client: reqwest::Client::new(), + blob_service, + directory_service, + } + } +} + +#[async_trait] +impl PathInfoService for NixHTTPPathInfoService { + #[instrument(skip_all, err, fields(path.digest=BASE64.encode(&digest)))] + async fn get(&self, digest: [u8; 20]) -> Result<Option<PathInfo>, Error> { + let narinfo_url = self + .base_url + .join(&format!("{}.narinfo", nixbase32::encode(&digest))) + .map_err(|e| { + warn!(e = %e, "unable to join URL"); + io::Error::new(io::ErrorKind::InvalidInput, "unable to join url") + })?; + + debug!(narinfo_url= %narinfo_url, "constructed NARInfo url"); + + let resp = self + .http_client + .get(narinfo_url) + .send() + .await + .map_err(|e| { + warn!(e=%e,"unable to send NARInfo request"); + io::Error::new( + io::ErrorKind::InvalidInput, + "unable to send NARInfo request", + ) + })?; + + // In the case of a 404, return a NotFound. + // We also return a NotFound in case of a 403 - this is to match the behaviour as Nix, + // when querying nix-cache.s3.amazonaws.com directly, rather than cache.nixos.org. + if resp.status() == StatusCode::NOT_FOUND || resp.status() == StatusCode::FORBIDDEN { + return Ok(None); + } + + let narinfo_str = resp.text().await.map_err(|e| { + warn!(e=%e,"unable to decode response as string"); + io::Error::new( + io::ErrorKind::InvalidData, + "unable to decode response as string", + ) + })?; + + // parse the received narinfo + let narinfo = NarInfo::parse(&narinfo_str).map_err(|e| { + warn!(e=%e,"unable to parse response as NarInfo"); + io::Error::new( + io::ErrorKind::InvalidData, + "unable to parse response as NarInfo", + ) + })?; + + // Convert to a (sparse) PathInfo. We still need to populate the node field, + // and for this we need to download the NAR file. + // FUTUREWORK: Keep some database around mapping from narsha256 to + // (unnamed) rootnode, so we can use that (and the name from the + // StorePath) and avoid downloading the same NAR a second time. + let pathinfo: PathInfo = (&narinfo).into(); + + // create a request for the NAR file itself. + let nar_url = self.base_url.join(narinfo.url).map_err(|e| { + warn!(e = %e, "unable to join URL"); + io::Error::new(io::ErrorKind::InvalidInput, "unable to join url") + })?; + debug!(nar_url= %nar_url, "constructed NAR url"); + + let resp = self + .http_client + .get(nar_url.clone()) + .send() + .await + .map_err(|e| { + warn!(e=%e,"unable to send NAR request"); + io::Error::new(io::ErrorKind::InvalidInput, "unable to send NAR request") + })?; + + // if the request is not successful, return an error. + if !resp.status().is_success() { + return Err(Error::StorageError(format!( + "unable to retrieve NAR at {}, status {}", + nar_url, + resp.status() + ))); + } + + // get an AsyncRead of the response body. + let async_r = tokio_util::io::StreamReader::new(resp.bytes_stream().map_err(|e| { + let e = e.without_url(); + warn!(e=%e, "failed to get response body"); + io::Error::new(io::ErrorKind::BrokenPipe, e.to_string()) + })); + let sync_r = std::io::BufReader::new(tokio_util::io::SyncIoBridge::new(async_r)); + + // handle decompression, by wrapping the reader. + let mut sync_r: Box<dyn BufRead + Send> = match narinfo.compression { + Some("none") => Box::new(sync_r), + Some("xz") => Box::new(std::io::BufReader::new(xz2::read::XzDecoder::new(sync_r))), + Some(comp) => { + return Err(Error::InvalidRequest( + format!("unsupported compression: {}", comp).to_string(), + )) + } + None => { + return Err(Error::InvalidRequest( + "unsupported compression: bzip2".to_string(), + )) + } + }; + + let res = tokio::task::spawn_blocking({ + let blob_service = self.blob_service.clone(); + let directory_service = self.directory_service.clone(); + move || crate::nar::read_nar(&mut sync_r, blob_service, directory_service) + }) + .await + .unwrap(); + + match res { + Ok(root_node) => Ok(Some(PathInfo { + node: Some(castorepb::Node { + // set the name of the root node to the digest-name of the store path. + node: Some(root_node.rename(narinfo.store_path.to_string().to_owned().into())), + }), + references: pathinfo.references, + narinfo: pathinfo.narinfo, + })), + Err(e) => Err(e.into()), + } + } + + #[instrument(skip_all, fields(path_info=?_path_info))] + async fn put(&self, _path_info: PathInfo) -> Result<PathInfo, Error> { + Err(Error::InvalidRequest( + "put not supported for this backend".to_string(), + )) + } + + #[instrument(skip_all, fields(root_node=?root_node))] + async fn calculate_nar( + &self, + root_node: &castorepb::node::Node, + ) -> Result<(u64, [u8; 32]), Error> { + Err(Error::InvalidRequest( + "calculate_nar not supported for this backend".to_string(), + )) + } + + fn list(&self) -> Pin<Box<dyn Stream<Item = Result<PathInfo, Error>> + Send>> { + Box::pin(futures::stream::once(async { + Err(Error::InvalidRequest( + "list not supported for this backend".to_string(), + )) + })) + } +} |