diff options
author | Florian Klink <flokli@flokli.de> | 2024-05-14T11·55+0200 |
---|---|---|
committer | clbot <clbot@tvl.fyi> | 2024-07-20T16·53+0000 |
commit | 861cc1f341d6774397f6505027f7d8bcc15291f6 (patch) | |
tree | d1eb3afb6f5ee7939b3828d05b5aea91ea4b0e2a | |
parent | 0244ae6eaffe1dd938748aaf1cfdf5fdab0c0a57 (diff) |
feat(tvix/nar-bridge): init r/8376
This adds an implementation of nar-bridge in Rust. Currently, only the GET parts are implemented. Contrary to the Go variant, this doesn't try to keep a mapping from nar hashes to root node in memory, it simply encodes the root node itself (stripped by its basename) into the URL. This pulls in a more recent version of axum than what we use in tonic, causing two versions of http and hyper, however dealing with `Body::from_stream` in axum 0.6 is much more annoying, and https://github.com/hyperium/tonic/pull/1740 suggests this will be fixed soon. Change-Id: Ia4c2dbda7cd3fdbe47a75f3e33544d19eac6e44e Reviewed-on: https://cl.tvl.fyi/c/depot/+/11898 Autosubmit: flokli <flokli@flokli.de> Reviewed-by: Brian Olsen <me@griff.name> Tested-by: BuildkiteCI
-rw-r--r-- | tvix/Cargo.lock | 268 | ||||
-rw-r--r-- | tvix/Cargo.nix | 885 | ||||
-rw-r--r-- | tvix/Cargo.toml | 1 | ||||
-rw-r--r-- | tvix/nar-bridge/Cargo.toml | 40 | ||||
-rw-r--r-- | tvix/nar-bridge/default.nix | 11 | ||||
-rw-r--r-- | tvix/nar-bridge/src/bin/nar-bridge.rs | 84 | ||||
-rw-r--r-- | tvix/nar-bridge/src/lib.rs | 50 | ||||
-rw-r--r-- | tvix/nar-bridge/src/nar.rs | 77 | ||||
-rw-r--r-- | tvix/nar-bridge/src/narinfo.rs | 131 |
9 files changed, 1460 insertions, 87 deletions
diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock index 7939383353bb..9e58235fae35 100644 --- a/tvix/Cargo.lock +++ b/tvix/Cargo.lock @@ -310,13 +310,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", - "axum-core", + "axum-core 0.3.4", "bitflags 1.3.2", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "itoa", "matchit", "memchr", @@ -325,13 +325,47 @@ dependencies = [ "pin-project-lite", "rustversion", "serde", - "sync_wrapper", + "sync_wrapper 0.1.2", "tower", "tower-layer", "tower-service", ] [[package]] +name = "axum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +dependencies = [ + "async-trait", + "axum-core 0.4.3", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.3.1", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper 1.0.1", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] name = "axum-core" version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -340,8 +374,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.6", "mime", "rustversion", "tower-layer", @@ -349,6 +383,27 @@ dependencies = [ ] [[package]] +name = "axum-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 0.1.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] name = "backtrace" version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -382,7 +437,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12ccba0acd0a82afb2b1fe89181b0776ff18aef5e355030631fe534c8ae8ec73" dependencies = [ "gcp_auth", - "http", + "http 0.2.11", "log", "prost", "prost-build", @@ -1388,7 +1443,7 @@ dependencies = [ "base64", "chrono", "home", - "hyper", + "hyper 0.14.28", "hyper-rustls", "ring", "rustls 0.21.12", @@ -1478,7 +1533,26 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.11", + "indexmap 2.1.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 1.1.0", "indexmap 2.1.0", "slab", "tokio", @@ -1553,13 +1627,47 @@ dependencies = [ ] [[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] name = "http-body" version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.11", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +dependencies = [ + "bytes", + "futures-core", + "http 1.1.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -1597,9 +1705,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.26", + "http 0.2.11", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -1612,14 +1720,34 @@ dependencies = [ ] [[package]] +name = "hyper" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.4", + "http 1.1.0", + "http-body 1.0.0", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", +] + +[[package]] name = "hyper-rustls" version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", - "hyper", + "http 0.2.11", + "hyper 0.14.28", "rustls 0.21.12", "rustls-native-certs 0.6.3", "tokio", @@ -1632,13 +1760,29 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.28", "pin-project-lite", "tokio", "tokio-io-timeout", ] [[package]] +name = "hyper-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +dependencies = [ + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.3.1", + "pin-project-lite", + "socket2", + "tokio", +] + +[[package]] name = "iana-time-zone" version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2095,6 +2239,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] +name = "nar-bridge" +version = "0.1.0" +dependencies = [ + "axum 0.7.5", + "bytes", + "clap", + "data-encoding", + "hex-literal", + "itertools 0.12.0", + "nix-compat", + "prost", + "prost-build", + "rstest", + "serde", + "thiserror", + "tokio", + "tokio-listener", + "tokio-util", + "tonic", + "tonic-build", + "tracing", + "tracing-subscriber", + "tvix-castore", + "tvix-store", + "tvix-tracing", + "url", +] + +[[package]] name = "nibble_vec" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2260,7 +2433,7 @@ dependencies = [ "chrono", "futures", "humantime", - "hyper", + "hyper 0.14.28", "itertools 0.12.0", "md-5", "parking_lot 0.12.2", @@ -2320,7 +2493,7 @@ checksum = "7690dc77bf776713848c4faa6501157469017eaf332baccd4eb1cea928743d94" dependencies = [ "async-trait", "bytes", - "http", + "http 0.2.11", "opentelemetry", ] @@ -2332,7 +2505,7 @@ checksum = "1a016b8d9495c639af2145ac22387dcb88e44118e45320d9238fbf4e7889abcb" dependencies = [ "async-trait", "futures-core", - "http", + "http 0.2.11", "opentelemetry", "opentelemetry-proto", "opentelemetry-semantic-conventions", @@ -2981,10 +3154,10 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.26", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-rustls", "ipnet", "js-sys", @@ -3021,7 +3194,7 @@ checksum = "5a735987236a8e238bf0296c7e351b999c188ccc11477f311b82b55c93984216" dependencies = [ "anyhow", "async-trait", - "http", + "http 0.2.11", "reqwest", "serde", "task-local-extensions", @@ -3372,18 +3545,18 @@ checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" [[package]] name = "serde" -version = "1.0.197" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.197" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", @@ -3402,6 +3575,16 @@ dependencies = [ ] [[package]] +name = "serde_path_to_error" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" +dependencies = [ + "itoa", + "serde", +] + +[[package]] name = "serde_qs" version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3686,6 +3869,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" + +[[package]] name = "system-configuration" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3913,16 +4102,21 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4134661e12ec11c6276be73544a43144a357b08dfab5c41fd226e15b5bc9a6b2" dependencies = [ + "axum 0.7.5", "clap", "document-features", "futures-core", "futures-util", + "hyper 1.3.1", + "hyper-util", "nix 0.26.4", "pin-project", "socket2", "tokio", "tokio-util", "tonic", + "tower", + "tower-service", "tracing", ] @@ -4065,13 +4259,13 @@ checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" dependencies = [ "async-stream", "async-trait", - "axum", + "axum 0.6.20", "base64", "bytes", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.26", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-timeout", "percent-encoding", "pin-project", @@ -4144,8 +4338,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.6", "http-range-header", "pin-project-lite", "tower-layer", @@ -4571,7 +4765,7 @@ dependencies = [ name = "tvix-tracing" version = "0.1.0" dependencies = [ - "http", + "http 0.2.11", "indicatif", "lazy_static", "opentelemetry", diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix index 8a52332aba20..221211c1c842 100644 --- a/tvix/Cargo.nix +++ b/tvix/Cargo.nix @@ -35,6 +35,16 @@ rec { # You can override the features with # workspaceMembers."${crateName}".build.override { features = [ "default" "feature1" ... ]; }. workspaceMembers = { + "nar-bridge" = rec { + packageId = "nar-bridge"; + build = internal.buildRustCrateWithFeatures { + packageId = "nar-bridge"; + }; + + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; "nix-compat" = rec { packageId = "nix-compat"; build = internal.buildRustCrateWithFeatures { @@ -1006,7 +1016,7 @@ rec { ]; }; - "axum" = rec { + "axum 0.6.20" = rec { crateName = "axum"; version = "0.6.20"; edition = "2021"; @@ -1018,7 +1028,7 @@ rec { } { name = "axum-core"; - packageId = "axum-core"; + packageId = "axum-core 0.3.4"; } { name = "bitflags"; @@ -1036,15 +1046,15 @@ rec { } { name = "http"; - packageId = "http"; + packageId = "http 0.2.11"; } { name = "http-body"; - packageId = "http-body"; + packageId = "http-body 0.4.6"; } { name = "hyper"; - packageId = "hyper"; + packageId = "hyper 0.14.28"; features = [ "stream" ]; } { @@ -1077,7 +1087,7 @@ rec { } { name = "sync_wrapper"; - packageId = "sync_wrapper"; + packageId = "sync_wrapper 0.1.2"; } { name = "tower"; @@ -1134,7 +1144,184 @@ rec { "ws" = [ "tokio" "dep:tokio-tungstenite" "dep:sha1" "dep:base64" ]; }; }; - "axum-core" = rec { + "axum 0.7.5" = rec { + crateName = "axum"; + version = "0.7.5"; + edition = "2021"; + sha256 = "1kyb7pzgn60crl9wyq7dhciv40sxdr1mbqx2r4s7g9j253qrlv1s"; + dependencies = [ + { + name = "async-trait"; + packageId = "async-trait"; + } + { + name = "axum-core"; + packageId = "axum-core 0.4.3"; + } + { + name = "bytes"; + packageId = "bytes"; + } + { + name = "futures-util"; + packageId = "futures-util"; + usesDefaultFeatures = false; + features = [ "alloc" ]; + } + { + name = "http"; + packageId = "http 1.1.0"; + } + { + name = "http-body"; + packageId = "http-body 1.0.0"; + } + { + name = "http-body-util"; + packageId = "http-body-util"; + } + { + name = "hyper"; + packageId = "hyper 1.3.1"; + optional = true; + } + { + name = "hyper-util"; + packageId = "hyper-util"; + optional = true; + features = [ "tokio" "server" ]; + } + { + name = "itoa"; + packageId = "itoa"; + } + { + name = "matchit"; + packageId = "matchit"; + } + { + name = "memchr"; + packageId = "memchr"; + } + { + name = "mime"; + packageId = "mime"; + } + { + name = "percent-encoding"; + packageId = "percent-encoding"; + } + { + name = "pin-project-lite"; + packageId = "pin-project-lite"; + } + { + name = "serde"; + packageId = "serde"; + } + { + name = "serde_json"; + packageId = "serde_json"; + optional = true; + features = [ "raw_value" ]; + } + { + name = "serde_path_to_error"; + packageId = "serde_path_to_error"; + optional = true; + } + { + name = "serde_urlencoded"; + packageId = "serde_urlencoded"; + optional = true; + } + { + name = "sync_wrapper"; + packageId = "sync_wrapper 1.0.1"; + } + { + name = "tokio"; + packageId = "tokio"; + rename = "tokio"; + optional = true; + features = [ "time" ]; + } + { + name = "tower"; + packageId = "tower"; + usesDefaultFeatures = false; + features = [ "util" ]; + } + { + name = "tower-layer"; + packageId = "tower-layer"; + } + { + name = "tower-service"; + packageId = "tower-service"; + } + { + name = "tracing"; + packageId = "tracing"; + optional = true; + usesDefaultFeatures = false; + } + ]; + buildDependencies = [ + { + name = "rustversion"; + packageId = "rustversion"; + } + ]; + devDependencies = [ + { + name = "rustversion"; + packageId = "rustversion"; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "tokio"; + packageId = "tokio"; + rename = "tokio"; + features = [ "macros" "rt" "rt-multi-thread" "net" "test-util" ]; + } + { + name = "tower"; + packageId = "tower"; + rename = "tower"; + features = [ "util" "timeout" "limit" "load-shed" "steer" "filter" ]; + } + { + name = "tracing"; + packageId = "tracing"; + } + ]; + features = { + "__private_docs" = [ "tower/full" "dep:tower-http" ]; + "default" = [ "form" "http1" "json" "matched-path" "original-uri" "query" "tokio" "tower-log" "tracing" ]; + "form" = [ "dep:serde_urlencoded" ]; + "http1" = [ "dep:hyper" "hyper?/http1" "hyper-util?/http1" ]; + "http2" = [ "dep:hyper" "hyper?/http2" "hyper-util?/http2" ]; + "json" = [ "dep:serde_json" "dep:serde_path_to_error" ]; + "macros" = [ "dep:axum-macros" ]; + "multipart" = [ "dep:multer" ]; + "query" = [ "dep:serde_urlencoded" ]; + "tokio" = [ "dep:hyper-util" "dep:tokio" "tokio/net" "tokio/rt" "tower/make" "tokio/macros" ]; + "tower-log" = [ "tower/log" ]; + "tracing" = [ "dep:tracing" "axum-core/tracing" ]; + "ws" = [ "dep:hyper" "tokio" "dep:tokio-tungstenite" "dep:sha1" "dep:base64" ]; + }; + resolvedDefaultFeatures = [ "default" "form" "http1" "http2" "json" "matched-path" "original-uri" "query" "tokio" "tower-log" "tracing" ]; + }; + "axum-core 0.3.4" = rec { crateName = "axum-core"; version = "0.3.4"; edition = "2021"; @@ -1156,11 +1343,11 @@ rec { } { name = "http"; - packageId = "http"; + packageId = "http 0.2.11"; } { name = "http-body"; - packageId = "http-body"; + packageId = "http-body 0.4.6"; } { name = "mime"; @@ -1194,6 +1381,85 @@ rec { "tracing" = [ "dep:tracing" ]; }; }; + "axum-core 0.4.3" = rec { + crateName = "axum-core"; + version = "0.4.3"; + edition = "2021"; + sha256 = "1qx28wg4j6qdcdrisqwyaavlzc0zvbsrcwa99zf9456lfbyn6p51"; + dependencies = [ + { + name = "async-trait"; + packageId = "async-trait"; + } + { + name = "bytes"; + packageId = "bytes"; + } + { + name = "futures-util"; + packageId = "futures-util"; + usesDefaultFeatures = false; + features = [ "alloc" ]; + } + { + name = "http"; + packageId = "http 1.1.0"; + } + { + name = "http-body"; + packageId = "http-body 1.0.0"; + } + { + name = "http-body-util"; + packageId = "http-body-util"; + } + { + name = "mime"; + packageId = "mime"; + } + { + name = "pin-project-lite"; + packageId = "pin-project-lite"; + } + { + name = "sync_wrapper"; + packageId = "sync_wrapper 0.1.2"; + } + { + name = "tower-layer"; + packageId = "tower-layer"; + } + { + name = "tower-service"; + packageId = "tower-service"; + } + { + name = "tracing"; + packageId = "tracing"; + optional = true; + usesDefaultFeatures = false; + } + ]; + buildDependencies = [ + { + name = "rustversion"; + packageId = "rustversion"; + } + ]; + devDependencies = [ + { + name = "futures-util"; + packageId = "futures-util"; + usesDefaultFeatures = false; + features = [ "alloc" ]; + } + ]; + features = { + "__private_docs" = [ "dep:tower-http" ]; + "tracing" = [ "dep:tracing" ]; + }; + resolvedDefaultFeatures = [ "tracing" ]; + }; "backtrace" = rec { crateName = "backtrace"; version = "0.3.69"; @@ -1298,7 +1564,7 @@ rec { } { name = "http"; - packageId = "http"; + packageId = "http 0.2.11"; } { name = "log"; @@ -4176,7 +4442,7 @@ rec { } { name = "hyper"; - packageId = "hyper"; + packageId = "hyper 0.14.28"; features = [ "client" "runtime" "http2" ]; } { @@ -4423,7 +4689,7 @@ rec { ]; }; - "h2" = rec { + "h2 0.3.26" = rec { crateName = "h2"; version = "0.3.26"; edition = "2018"; @@ -4458,7 +4724,79 @@ rec { } { name = "http"; - packageId = "http"; + packageId = "http 0.2.11"; + } + { + name = "indexmap"; + packageId = "indexmap 2.1.0"; + features = [ "std" ]; + } + { + name = "slab"; + packageId = "slab"; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "io-util" ]; + } + { + name = "tokio-util"; + packageId = "tokio-util"; + features = [ "codec" "io" ]; + } + { + name = "tracing"; + packageId = "tracing"; + usesDefaultFeatures = false; + features = [ "std" ]; + } + ]; + devDependencies = [ + { + name = "tokio"; + packageId = "tokio"; + features = [ "rt-multi-thread" "macros" "sync" "net" ]; + } + ]; + features = { }; + }; + "h2 0.4.4" = rec { + crateName = "h2"; + version = "0.4.4"; + edition = "2021"; + sha256 = "0sc0ymhiqp4hbz39d405cjbga77wnz2pprbgyc498xs58hlwfvl1"; + authors = [ + "Carl Lerche <me@carllerche.com>" + "Sean McArthur <sean@seanmonstar.com>" + ]; + dependencies = [ + { + name = "bytes"; + packageId = "bytes"; + } + { + name = "fnv"; + packageId = "fnv"; + } + { + name = "futures-core"; + packageId = "futures-core"; + usesDefaultFeatures = false; + } + { + name = "futures-sink"; + packageId = "futures-sink"; + usesDefaultFeatures = false; + } + { + name = "futures-util"; + packageId = "futures-util"; + usesDefaultFeatures = false; + } + { + name = "http"; + packageId = "http 1.1.0"; } { name = "indexmap"; @@ -4646,7 +4984,7 @@ rec { ]; }; - "http" = rec { + "http 0.2.11" = rec { crateName = "http"; version = "0.2.11"; edition = "2018"; @@ -4672,7 +5010,36 @@ rec { ]; }; - "http-body" = rec { + "http 1.1.0" = rec { + crateName = "http"; + version = "1.1.0"; + edition = "2018"; + sha256 = "0n426lmcxas6h75c2cp25m933pswlrfjz10v91vc62vib2sdvf91"; + authors = [ + "Alex Crichton <alex@alexcrichton.com>" + "Carl Lerche <me@carllerche.com>" + "Sean McArthur <sean@seanmonstar.com>" + ]; + dependencies = [ + { + name = "bytes"; + packageId = "bytes"; + } + { + name = "fnv"; + packageId = "fnv"; + } + { + name = "itoa"; + packageId = "itoa"; + } + ]; + features = { + "default" = [ "std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; + "http-body 0.4.6" = rec { crateName = "http-body"; version = "0.4.6"; edition = "2018"; @@ -4689,7 +5056,63 @@ rec { } { name = "http"; - packageId = "http"; + packageId = "http 0.2.11"; + } + { + name = "pin-project-lite"; + packageId = "pin-project-lite"; + } + ]; + + }; + "http-body 1.0.0" = rec { + crateName = "http-body"; + version = "1.0.0"; + edition = "2018"; + sha256 = "0hyn8n3iadrbwq8y0p1rl1275s4nm49bllw5wji29g4aa3dqbb0w"; + authors = [ + "Carl Lerche <me@carllerche.com>" + "Lucio Franco <luciofranco14@gmail.com>" + "Sean McArthur <sean@seanmonstar.com>" + ]; + dependencies = [ + { + name = "bytes"; + packageId = "bytes"; + } + { + name = "http"; + packageId = "http 1.1.0"; + } + ]; + + }; + "http-body-util" = rec { + crateName = "http-body-util"; + version = "0.1.1"; + edition = "2018"; + sha256 = "07agldas2qgcfc05ckiarlmf9vzragbda823nqhrqrc6mjrghx84"; + authors = [ + "Carl Lerche <me@carllerche.com>" + "Lucio Franco <luciofranco14@gmail.com>" + "Sean McArthur <sean@seanmonstar.com>" + ]; + dependencies = [ + { + name = "bytes"; + packageId = "bytes"; + } + { + name = "futures-core"; + packageId = "futures-core"; + } + { + name = "http"; + packageId = "http 1.1.0"; + } + { + name = "http-body"; + packageId = "http-body 1.0.0"; } { name = "pin-project-lite"; @@ -4738,7 +5161,7 @@ rec { ]; }; - "hyper" = rec { + "hyper 0.14.28" = rec { crateName = "hyper"; version = "0.14.28"; edition = "2018"; @@ -4767,16 +5190,16 @@ rec { } { name = "h2"; - packageId = "h2"; + packageId = "h2 0.3.26"; optional = true; } { name = "http"; - packageId = "http"; + packageId = "http 0.2.11"; } { name = "http-body"; - packageId = "http-body"; + packageId = "http-body 0.4.6"; } { name = "httparse"; @@ -4845,6 +5268,104 @@ rec { }; resolvedDefaultFeatures = [ "client" "default" "full" "h2" "http1" "http2" "runtime" "server" "socket2" "stream" "tcp" ]; }; + "hyper 1.3.1" = rec { + crateName = "hyper"; + version = "1.3.1"; + edition = "2021"; + sha256 = "0va9pjqshsr8zc07m9h4j2821hsmd9lw9j416yisjqh8gp8msmzy"; + authors = [ + "Sean McArthur <sean@seanmonstar.com>" + ]; + dependencies = [ + { + name = "bytes"; + packageId = "bytes"; + } + { + name = "futures-channel"; + packageId = "futures-channel"; + optional = true; + } + { + name = "futures-util"; + packageId = "futures-util"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "h2"; + packageId = "h2 0.4.4"; + optional = true; + } + { + name = "http"; + packageId = "http 1.1.0"; + } + { + name = "http-body"; + packageId = "http-body 1.0.0"; + } + { + name = "httparse"; + packageId = "httparse"; + optional = true; + } + { + name = "httpdate"; + packageId = "httpdate"; + optional = true; + } + { + name = "itoa"; + packageId = "itoa"; + optional = true; + } + { + name = "pin-project-lite"; + packageId = "pin-project-lite"; + optional = true; + } + { + name = "smallvec"; + packageId = "smallvec"; + optional = true; + features = [ "const_generics" "const_new" ]; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "sync" ]; + } + ]; + devDependencies = [ + { + name = "futures-channel"; + packageId = "futures-channel"; + features = [ "sink" ]; + } + { + name = "futures-util"; + packageId = "futures-util"; + usesDefaultFeatures = false; + features = [ "alloc" "sink" ]; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "fs" "macros" "net" "io-std" "io-util" "rt" "rt-multi-thread" "sync" "time" "test-util" ]; + } + ]; + features = { + "client" = [ "dep:want" "dep:pin-project-lite" "dep:smallvec" ]; + "ffi" = [ "dep:libc" "dep:http-body-util" "futures-util?/alloc" ]; + "full" = [ "client" "http1" "http2" "server" ]; + "http1" = [ "dep:futures-channel" "dep:futures-util" "dep:httparse" "dep:itoa" ]; + "http2" = [ "dep:futures-channel" "dep:futures-util" "dep:h2" ]; + "server" = [ "dep:httpdate" "dep:pin-project-lite" "dep:smallvec" ]; + "tracing" = [ "dep:tracing" ]; + }; + resolvedDefaultFeatures = [ "default" "http1" "http2" "server" ]; + }; "hyper-rustls" = rec { crateName = "hyper-rustls"; version = "0.24.2"; @@ -4858,11 +5379,11 @@ rec { } { name = "http"; - packageId = "http"; + packageId = "http 0.2.11"; } { name = "hyper"; - packageId = "hyper"; + packageId = "hyper 0.14.28"; usesDefaultFeatures = false; features = [ "client" ]; } @@ -4889,7 +5410,7 @@ rec { devDependencies = [ { name = "hyper"; - packageId = "hyper"; + packageId = "hyper 0.14.28"; features = [ "full" ]; } { @@ -4931,7 +5452,7 @@ rec { dependencies = [ { name = "hyper"; - packageId = "hyper"; + packageId = "hyper 0.14.28"; features = [ "client" ]; } { @@ -4950,7 +5471,7 @@ rec { devDependencies = [ { name = "hyper"; - packageId = "hyper"; + packageId = "hyper 0.14.28"; features = [ "client" "http1" "tcp" ]; } { @@ -4961,6 +5482,82 @@ rec { ]; }; + "hyper-util" = rec { + crateName = "hyper-util"; + version = "0.1.3"; + edition = "2021"; + sha256 = "1akngan7j0n2n0wd25c6952mvqbkj9gp1lcwzyxjc0d37l8yyf6a"; + authors = [ + "Sean McArthur <sean@seanmonstar.com>" + ]; + dependencies = [ + { + name = "bytes"; + packageId = "bytes"; + } + { + name = "futures-util"; + packageId = "futures-util"; + usesDefaultFeatures = false; + } + { + name = "http"; + packageId = "http 1.1.0"; + } + { + name = "http-body"; + packageId = "http-body 1.0.0"; + } + { + name = "hyper"; + packageId = "hyper 1.3.1"; + } + { + name = "pin-project-lite"; + packageId = "pin-project-lite"; + } + { + name = "socket2"; + packageId = "socket2"; + optional = true; + features = [ "all" ]; + } + { + name = "tokio"; + packageId = "tokio"; + optional = true; + features = [ "net" "rt" "time" ]; + } + ]; + devDependencies = [ + { + name = "bytes"; + packageId = "bytes"; + } + { + name = "hyper"; + packageId = "hyper 1.3.1"; + features = [ "full" ]; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "macros" "test-util" ]; + } + ]; + features = { + "client" = [ "hyper/client" "dep:tracing" "dep:futures-channel" "dep:tower" "dep:tower-service" ]; + "client-legacy" = [ "client" ]; + "full" = [ "client" "client-legacy" "server" "server-auto" "service" "http1" "http2" "tokio" ]; + "http1" = [ "hyper/http1" ]; + "http2" = [ "hyper/http2" ]; + "server" = [ "hyper/server" ]; + "server-auto" = [ "server" "http1" "http2" ]; + "service" = [ "dep:tower" "dep:tower-service" ]; + "tokio" = [ "dep:tokio" "dep:socket2" ]; + }; + resolvedDefaultFeatures = [ "default" "http1" "http2" "server" "server-auto" "tokio" ]; + }; "iana-time-zone" = rec { crateName = "iana-time-zone"; version = "0.1.60"; @@ -6260,6 +6857,130 @@ rec { "serde_impl" = [ "serde" ]; }; }; + "nar-bridge" = rec { + crateName = "nar-bridge"; + version = "0.1.0"; + edition = "2021"; + crateBin = [ + { + name = "nar-bridge"; + path = "src/bin/nar-bridge.rs"; + requiredFeatures = [ ]; + } + ]; + src = lib.cleanSourceWith { filter = sourceFilter; src = ./nar-bridge; }; + dependencies = [ + { + name = "axum"; + packageId = "axum 0.7.5"; + features = [ "http2" ]; + } + { + name = "bytes"; + packageId = "bytes"; + } + { + name = "clap"; + packageId = "clap"; + features = [ "derive" "env" ]; + } + { + name = "data-encoding"; + packageId = "data-encoding"; + } + { + name = "itertools"; + packageId = "itertools 0.12.0"; + } + { + name = "nix-compat"; + packageId = "nix-compat"; + features = [ "async" ]; + } + { + name = "prost"; + packageId = "prost"; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "thiserror"; + packageId = "thiserror"; + } + { + name = "tokio"; + packageId = "tokio"; + } + { + name = "tokio-listener"; + packageId = "tokio-listener"; + features = [ "axum07" "clap" "multi-listener" "sd_listen" ]; + } + { + name = "tokio-util"; + packageId = "tokio-util"; + features = [ "io" "io-util" "compat" ]; + } + { + name = "tonic"; + packageId = "tonic"; + features = [ "tls" "tls-roots" ]; + } + { + name = "tracing"; + packageId = "tracing"; + } + { + name = "tracing-subscriber"; + packageId = "tracing-subscriber"; + } + { + name = "tvix-castore"; + packageId = "tvix-castore"; + } + { + name = "tvix-store"; + packageId = "tvix-store"; + } + { + name = "tvix-tracing"; + packageId = "tvix-tracing"; + features = [ "tonic" ]; + } + { + name = "url"; + packageId = "url"; + } + ]; + buildDependencies = [ + { + name = "prost-build"; + packageId = "prost-build"; + } + { + name = "tonic-build"; + packageId = "tonic-build"; + } + ]; + devDependencies = [ + { + name = "hex-literal"; + packageId = "hex-literal"; + } + { + name = "rstest"; + packageId = "rstest"; + } + ]; + features = { + "default" = [ "otlp" ]; + "otlp" = [ "tvix-tracing/otlp" ]; + }; + resolvedDefaultFeatures = [ "default" "otlp" ]; + }; "nibble_vec" = rec { crateName = "nibble_vec"; version = "0.1.0"; @@ -6796,7 +7517,7 @@ rec { } { name = "hyper"; - packageId = "hyper"; + packageId = "hyper 0.14.28"; optional = true; usesDefaultFeatures = false; } @@ -6890,7 +7611,7 @@ rec { devDependencies = [ { name = "hyper"; - packageId = "hyper"; + packageId = "hyper 0.14.28"; features = [ "server" ]; } { @@ -7020,7 +7741,7 @@ rec { } { name = "http"; - packageId = "http"; + packageId = "http 0.2.11"; usesDefaultFeatures = false; } { @@ -7053,7 +7774,7 @@ rec { } { name = "http"; - packageId = "http"; + packageId = "http 0.2.11"; optional = true; usesDefaultFeatures = false; } @@ -9022,21 +9743,21 @@ rec { } { name = "h2"; - packageId = "h2"; + packageId = "h2 0.3.26"; target = { target, features }: (!("wasm32" == target."arch" or null)); } { name = "http"; - packageId = "http"; + packageId = "http 0.2.11"; } { name = "http-body"; - packageId = "http-body"; + packageId = "http-body 0.4.6"; target = { target, features }: (!("wasm32" == target."arch" or null)); } { name = "hyper"; - packageId = "hyper"; + packageId = "hyper 0.14.28"; usesDefaultFeatures = false; target = { target, features }: (!("wasm32" == target."arch" or null)); features = [ "tcp" "http1" "http2" "client" "runtime" ]; @@ -9191,7 +9912,7 @@ rec { devDependencies = [ { name = "hyper"; - packageId = "hyper"; + packageId = "hyper 0.14.28"; usesDefaultFeatures = false; target = { target, features }: (!("wasm32" == target."arch" or null)); features = [ "tcp" "stream" "http1" "http2" "client" "server" "runtime" ]; @@ -9282,7 +10003,7 @@ rec { } { name = "http"; - packageId = "http"; + packageId = "http 0.2.11"; } { name = "reqwest"; @@ -10378,9 +11099,9 @@ rec { }; "serde" = rec { crateName = "serde"; - version = "1.0.197"; + version = "1.0.204"; edition = "2018"; - sha256 = "1qjcxqd3p4yh5cmmax9q4ics1zy34j5ij32cvjj5dc5rw5rwic9z"; + sha256 = "04kwpwqz559xlhxkggmm8rjxqgywy5swam3kscwsicnbw1cgaxmw"; authors = [ "Erick Tryzelaar <erick.tryzelaar@gmail.com>" "David Tolnay <dtolnay@gmail.com>" @@ -10412,9 +11133,9 @@ rec { }; "serde_derive" = rec { crateName = "serde_derive"; - version = "1.0.197"; + version = "1.0.204"; edition = "2015"; - sha256 = "02v1x0sdv8qy06lpr6by4ar1n3jz3hmab15cgimpzhgd895v7c3y"; + sha256 = "08p25262mbmhsr2cg0508d5b1wvljj956rvpg0v3qgg6gc8pxkg0"; procMacro = true; authors = [ "Erick Tryzelaar <erick.tryzelaar@gmail.com>" @@ -10481,7 +11202,27 @@ rec { "preserve_order" = [ "indexmap" "std" ]; "std" = [ "serde/std" ]; }; - resolvedDefaultFeatures = [ "alloc" "default" "std" ]; + resolvedDefaultFeatures = [ "alloc" "default" "raw_value" "std" ]; + }; + "serde_path_to_error" = rec { + crateName = "serde_path_to_error"; + version = "0.1.16"; + edition = "2021"; + sha256 = "19hlz2359l37ifirskpcds7sxg0gzpqvfilibs7whdys0128i6dg"; + authors = [ + "David Tolnay <dtolnay@gmail.com>" + ]; + dependencies = [ + { + name = "itoa"; + packageId = "itoa"; + } + { + name = "serde"; + packageId = "serde"; + } + ]; + }; "serde_qs" = rec { crateName = "serde_qs"; @@ -10970,6 +11711,7 @@ rec { "drain_keep_rest" = [ "drain_filter" ]; "serde" = [ "dep:serde" ]; }; + resolvedDefaultFeatures = [ "const_generics" "const_new" ]; }; "smol_str" = rec { crateName = "smol_str"; @@ -11332,7 +12074,7 @@ rec { }; resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "full" "parsing" "printing" "proc-macro" "quote" "visit" "visit-mut" ]; }; - "sync_wrapper" = rec { + "sync_wrapper 0.1.2" = rec { crateName = "sync_wrapper"; version = "0.1.2"; edition = "2018"; @@ -11345,6 +12087,19 @@ rec { "futures-core" = [ "dep:futures-core" ]; }; }; + "sync_wrapper 1.0.1" = rec { + crateName = "sync_wrapper"; + version = "1.0.1"; + edition = "2018"; + sha256 = "150k6lwvr4nl237ngsz8fj5j78k712m4bggrfyjsidllraz5l1m7"; + authors = [ + "Actyx AG <developer@actyx.io>" + ]; + features = { + "futures" = [ "futures-core" ]; + "futures-core" = [ "dep:futures-core" ]; + }; + }; "system-configuration" = rec { crateName = "system-configuration"; version = "0.5.1"; @@ -11979,6 +12734,12 @@ rec { sha256 = "1cm6r5dmpq96s8gw9dgsinq5g8s466j48dg7dckwc4gc28g6cd21"; dependencies = [ { + name = "axum"; + packageId = "axum 0.7.5"; + rename = "axum07"; + optional = true; + } + { name = "clap"; packageId = "clap"; optional = true; @@ -11999,6 +12760,19 @@ rec { optional = true; } { + name = "hyper"; + packageId = "hyper 1.3.1"; + rename = "hyper1"; + optional = true; + features = [ "server" ]; + } + { + name = "hyper-util"; + packageId = "hyper-util"; + optional = true; + features = [ "server" "server-auto" ]; + } + { name = "nix"; packageId = "nix 0.26.4"; optional = true; @@ -12034,6 +12808,17 @@ rec { optional = true; } { + name = "tower"; + packageId = "tower"; + optional = true; + features = [ "util" ]; + } + { + name = "tower-service"; + packageId = "tower-service"; + optional = true; + } + { name = "tracing"; packageId = "tracing"; } @@ -12069,7 +12854,7 @@ rec { "unix_path_tools" = [ "nix" ]; "user_facing_default" = [ "inetd" "unix" "unix_path_tools" "sd_listen" "socket_options" ]; }; - resolvedDefaultFeatures = [ "clap" "default" "inetd" "multi-listener" "nix" "sd_listen" "socket2" "socket_options" "tokio-util" "tonic011" "unix" "unix_path_tools" "user_facing_default" ]; + resolvedDefaultFeatures = [ "axum07" "clap" "default" "inetd" "multi-listener" "nix" "sd_listen" "socket2" "socket_options" "tokio-util" "tonic011" "unix" "unix_path_tools" "user_facing_default" ]; }; "tokio-macros" = rec { crateName = "tokio-macros"; @@ -12534,7 +13319,7 @@ rec { } { name = "axum"; - packageId = "axum"; + packageId = "axum 0.6.20"; optional = true; usesDefaultFeatures = false; } @@ -12548,20 +13333,20 @@ rec { } { name = "h2"; - packageId = "h2"; + packageId = "h2 0.3.26"; optional = true; } { name = "http"; - packageId = "http"; + packageId = "http 0.2.11"; } { name = "http-body"; - packageId = "http-body"; + packageId = "http-body 0.4.6"; } { name = "hyper"; - packageId = "hyper"; + packageId = "hyper 0.14.28"; optional = true; features = [ "full" ]; } @@ -12900,11 +13685,11 @@ rec { } { name = "http"; - packageId = "http"; + packageId = "http 0.2.11"; } { name = "http-body"; - packageId = "http-body"; + packageId = "http-body 0.4.6"; } { name = "http-range-header"; @@ -14618,7 +15403,7 @@ rec { dependencies = [ { name = "http"; - packageId = "http"; + packageId = "http 0.2.11"; optional = true; } { diff --git a/tvix/Cargo.toml b/tvix/Cargo.toml index ed5c6d0d8b5c..53b9134a567b 100644 --- a/tvix/Cargo.toml +++ b/tvix/Cargo.toml @@ -25,6 +25,7 @@ members = [ "eval", "eval/builtin-macros", "glue", + "nar-bridge", "nix-compat", "serde", "store", diff --git a/tvix/nar-bridge/Cargo.toml b/tvix/nar-bridge/Cargo.toml new file mode 100644 index 000000000000..920b81c45cbe --- /dev/null +++ b/tvix/nar-bridge/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "nar-bridge" +version = "0.1.0" +edition = "2021" + +[dependencies] +axum = { version = "0.7.5", features = ["http2"] } +bytes = "1.4.0" +clap = { version = "4.0", features = ["derive", "env"] } +data-encoding = "2.3.3" +itertools = "0.12.0" +prost = "0.12.1" +nix-compat = { path = "../nix-compat", features = ["async"] } +thiserror = "1.0.56" +tokio = { version = "1.32.0" } +tokio-listener = { version = "0.4.2", features = [ "axum07", "clap", "multi-listener", "sd_listen" ] } +tokio-util = { version = "0.7.9", features = ["io", "io-util", "compat"] } +tonic = { version = "0.11.0", features = ["tls", "tls-roots"] } +tvix-castore = { path = "../castore" } +tvix-store = { path = "../store" } +tvix-tracing = { path = "../tracing", features = ["tonic"] } +tracing = "0.1.37" +tracing-subscriber = "0.3.16" +url = "2.4.0" +serde = { version = "1.0.204", features = ["derive"] } + +[build-dependencies] +prost-build = "0.12.1" +tonic-build = "0.11.0" + +[features] +default = ["otlp"] +otlp = ["tvix-tracing/otlp"] + +[dev-dependencies] +hex-literal = "0.4.1" +rstest = "0.19.0" + +[lints] +workspace = true diff --git a/tvix/nar-bridge/default.nix b/tvix/nar-bridge/default.nix new file mode 100644 index 000000000000..3e116a1fc02b --- /dev/null +++ b/tvix/nar-bridge/default.nix @@ -0,0 +1,11 @@ +{ depot, lib, ... }: + +(depot.tvix.crates.workspaceMembers.nar-bridge.build.override { + runTests = true; +}).overrideAttrs (old: rec { + meta.ci.targets = lib.filter (x: lib.hasPrefix "with-features" x || x == "no-features") (lib.attrNames passthru); + passthru = (depot.tvix.utils.mkFeaturePowerset { + inherit (old) crateName; + features = [ "otlp" ]; + }); +}) diff --git a/tvix/nar-bridge/src/bin/nar-bridge.rs b/tvix/nar-bridge/src/bin/nar-bridge.rs new file mode 100644 index 000000000000..e24068ca2300 --- /dev/null +++ b/tvix/nar-bridge/src/bin/nar-bridge.rs @@ -0,0 +1,84 @@ +use clap::Parser; +use nar_bridge::AppState; +use tracing::info; + +/// Expose the Nix HTTP Binary Cache protocol for a tvix-store. +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +struct Cli { + #[arg(long, env, default_value = "grpc+http://[::1]:8000")] + blob_service_addr: String, + + #[arg(long, env, default_value = "grpc+http://[::1]:8000")] + directory_service_addr: String, + + #[arg(long, env, default_value = "grpc+http://[::1]:8000")] + path_info_service_addr: String, + + /// The priority to announce at the `nix-cache-info` endpoint. + /// A lower number means it's *more preferred. + #[arg(long, env, default_value_t = 39)] + priority: u64, + + /// The address to listen on. + #[clap(flatten)] + listen_args: tokio_listener::ListenerAddressLFlag, + + #[cfg(feature = "otlp")] + /// Whether to configure OTLP. Set --otlp=false to disable. + #[arg(long, default_missing_value = "true", default_value = "true", num_args(0..=1), require_equals(true), action(clap::ArgAction::Set))] + otlp: bool, +} + +#[tokio::main] +async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> { + let cli = Cli::parse(); + + let _tracing_handle = { + #[allow(unused_mut)] + let mut builder = tvix_tracing::TracingBuilder::default(); + #[cfg(feature = "otlp")] + { + if cli.otlp { + builder = builder.enable_otlp("tvix.store"); + } + } + builder.build()? + }; + + // initialize stores + let (blob_service, directory_service, path_info_service, _nar_calculation_service) = + tvix_store::utils::construct_services( + cli.blob_service_addr, + cli.directory_service_addr, + cli.path_info_service_addr, + ) + .await?; + + let state = AppState::new(blob_service, directory_service, path_info_service.into()); + + let app = nar_bridge::gen_router(cli.priority).with_state(state); + + let listen_address = &cli.listen_args.listen_address.unwrap_or_else(|| { + "[::]:8000" + .parse() + .expect("invalid fallback listen address") + }); + + let listener = tokio_listener::Listener::bind( + listen_address, + &Default::default(), + &cli.listen_args.listener_options, + ) + .await?; + + info!(listen_address=%listen_address, "starting daemon"); + + tokio_listener::axum07::serve( + listener, + app.into_make_service_with_connect_info::<tokio_listener::SomeSocketAddrClonable>(), + ) + .await?; + + Ok(()) +} diff --git a/tvix/nar-bridge/src/lib.rs b/tvix/nar-bridge/src/lib.rs new file mode 100644 index 000000000000..5d9ec43c1cde --- /dev/null +++ b/tvix/nar-bridge/src/lib.rs @@ -0,0 +1,50 @@ +use axum::routing::head; +use axum::{routing::get, Router}; +use std::sync::Arc; +use tvix_castore::blobservice::BlobService; +use tvix_castore::directoryservice::DirectoryService; +use tvix_store::pathinfoservice::PathInfoService; + +mod nar; +mod narinfo; + +#[derive(Clone)] +pub struct AppState { + blob_service: Arc<dyn BlobService>, + directory_service: Arc<dyn DirectoryService>, + path_info_service: Arc<dyn PathInfoService>, +} + +impl AppState { + pub fn new( + blob_service: Arc<dyn BlobService>, + directory_service: Arc<dyn DirectoryService>, + path_info_service: Arc<dyn PathInfoService>, + ) -> Self { + Self { + blob_service, + directory_service, + path_info_service, + } + } +} + +pub fn gen_router(priority: u64) -> Router<AppState> { + Router::new() + .route("/", get(root)) + .route("/nar/tvix-castore/:root_node_enc", get(nar::get)) + .route("/:narinfo_str", get(narinfo::get)) + .route("/:narinfo_str", head(narinfo::head)) + .route("/nix-cache-info", get(move || nix_cache_info(priority))) +} + +async fn root() -> &'static str { + "Hello from nar-bridge" +} + +async fn nix_cache_info(priority: u64) -> String { + format!( + "StoreDir: /nix/store\nWantMassQuery: 1\nPriority: {}\n", + priority + ) +} diff --git a/tvix/nar-bridge/src/nar.rs b/tvix/nar-bridge/src/nar.rs new file mode 100644 index 000000000000..63cd86e49969 --- /dev/null +++ b/tvix/nar-bridge/src/nar.rs @@ -0,0 +1,77 @@ +use axum::body::Body; +use axum::extract::Query; +use axum::http::StatusCode; +use axum::response::Response; +use bytes::Bytes; +use data_encoding::BASE64URL_NOPAD; +use serde::Deserialize; +use tokio_util::io::ReaderStream; +use tracing::{instrument, warn}; + +use crate::AppState; + +#[derive(Debug, Deserialize)] +pub(crate) struct GetNARParams { + #[serde(rename = "narsize")] + nar_size: u64, +} + +#[instrument(skip(blob_service, directory_service))] +pub async fn get( + axum::extract::Path(root_node_enc): axum::extract::Path<String>, + axum::extract::Query(GetNARParams { nar_size }): Query<GetNARParams>, + axum::extract::State(AppState { + blob_service, + directory_service, + .. + }): axum::extract::State<AppState>, +) -> Result<Response, StatusCode> { + use prost::Message; + // b64decode the root node passed *by the user* + let root_node_proto = BASE64URL_NOPAD + .decode(root_node_enc.as_bytes()) + .map_err(|e| { + warn!(err=%e, "unable to decode root node b64"); + StatusCode::NOT_FOUND + })?; + + // check the proto size to be somewhat reasonable before parsing it. + if root_node_proto.len() > 4096 { + warn!("rejected too large root node"); + return Err(StatusCode::BAD_REQUEST); + } + + // parse the proto + let root_node: tvix_castore::proto::Node = Message::decode(Bytes::from(root_node_enc)) + .map_err(|e| { + warn!(err=%e, "unable to decode root node proto"); + StatusCode::NOT_FOUND + })?; + + // validate it. + let root_node = root_node + .validate() + .map_err(|e| { + warn!(err=%e, "root node validation failed"); + StatusCode::BAD_REQUEST + })? + .to_owned(); + + let (w, r) = tokio::io::duplex(1024 * 8); + + // spawn a task rendering the NAR to the client + tokio::spawn(async move { + if let Err(e) = + tvix_store::nar::write_nar(w, &root_node, blob_service, directory_service).await + { + warn!(err=%e, "failed to write out NAR"); + } + }); + + Ok(Response::builder() + .status(StatusCode::OK) + .header("cache-control", "max-age=31536000, immutable") + .header("content-length", nar_size) + .body(Body::from_stream(ReaderStream::new(r))) + .unwrap()) +} diff --git a/tvix/nar-bridge/src/narinfo.rs b/tvix/nar-bridge/src/narinfo.rs new file mode 100644 index 000000000000..7b6c1bdfdccf --- /dev/null +++ b/tvix/nar-bridge/src/narinfo.rs @@ -0,0 +1,131 @@ +use axum::http::StatusCode; +use nix_compat::nixbase32; +use tracing::{instrument, warn, Span}; +use tvix_castore::proto::node::Node; + +use crate::AppState; + +#[instrument(skip(path_info_service))] +pub async fn head( + axum::extract::Path(narinfo_str): axum::extract::Path<String>, + axum::extract::State(AppState { + path_info_service, .. + }): axum::extract::State<AppState>, +) -> Result<&'static str, StatusCode> { + let digest = parse_narinfo_str(&narinfo_str)?; + Span::current().record("path_info.digest", &narinfo_str[0..32]); + + if path_info_service + .get(digest) + .await + .map_err(|e| { + warn!(err=%e, "failed to get PathInfo"); + StatusCode::INTERNAL_SERVER_ERROR + })? + .is_some() + { + Ok("") + } else { + warn!("PathInfo not found"); + Err(StatusCode::NOT_FOUND) + } +} + +#[instrument(skip(path_info_service))] +pub async fn get( + axum::extract::Path(narinfo_str): axum::extract::Path<String>, + axum::extract::State(AppState { + path_info_service, .. + }): axum::extract::State<AppState>, +) -> Result<String, StatusCode> { + let digest = parse_narinfo_str(&narinfo_str)?; + Span::current().record("path_info.digest", &narinfo_str[0..32]); + + // fetch the PathInfo + let path_info = path_info_service + .get(digest) + .await + .map_err(|e| { + warn!(err=%e, "failed to get PathInfo"); + StatusCode::INTERNAL_SERVER_ERROR + })? + .ok_or(StatusCode::NOT_FOUND)?; + + let store_path = path_info.validate().map_err(|e| { + warn!(err=%e, "invalid PathInfo"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + let mut narinfo = path_info.to_narinfo(store_path).ok_or_else(|| { + warn!(path_info=?path_info, "PathInfo contained no NAR data"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + // encode the (unnamed) root node in the NAR url itself. + let root_node = path_info + .node + .as_ref() + .and_then(|n| n.node.as_ref()) + .expect("root node must not be none") + .clone() + .rename("".into()); + + let mut buf = Vec::new(); + Node::encode(&root_node, &mut buf); + + let url = format!( + "nar/tvix-castore/{}?narsize={}", + data_encoding::BASE64URL_NOPAD.encode(&buf), + narinfo.nar_size, + ); + + narinfo.url = &url; + + Ok(narinfo.to_string()) +} + +/// Parses a `3mzh8lvgbynm9daj7c82k2sfsfhrsfsy.narinfo` string and returns the +/// nixbase32-decoded digest. +fn parse_narinfo_str(s: &str) -> Result<[u8; 20], StatusCode> { + if !s.is_char_boundary(32) { + warn!("invalid string, no char boundary at 32"); + return Err(StatusCode::NOT_FOUND); + } + + Ok(match s.split_at(32) { + (hash_str, ".narinfo") => { + // we know this is 32 bytes + let hash_str_fixed: [u8; 32] = hash_str.as_bytes().try_into().unwrap(); + nixbase32::decode_fixed(hash_str_fixed).map_err(|e| { + warn!(err=%e, "invalid digest"); + StatusCode::NOT_FOUND + })? + } + _ => { + warn!("invalid string"); + return Err(StatusCode::NOT_FOUND); + } + }) +} + +#[cfg(test)] +mod test { + use super::parse_narinfo_str; + use hex_literal::hex; + + #[test] + fn success() { + assert_eq!( + hex!("8a12321522fd91efbd60ebb2481af88580f61600"), + parse_narinfo_str("00bgd045z0d4icpbc2yyz4gx48ak44la.narinfo").unwrap() + ); + } + + #[test] + fn failure() { + assert!(parse_narinfo_str("00bgd045z0d4icpbc2yyz4gx48ak44la").is_err()); + assert!(parse_narinfo_str("/00bgd045z0d4icpbc2yyz4gx48ak44la").is_err()); + assert!(parse_narinfo_str("000000").is_err()); + assert!(parse_narinfo_str("00bgd045z0d4icpbc2yyz4gx48ak44l🦊.narinfo").is_err()); + } +} |