From da6cbb4a459d02111c44a67d3d0dd7e654abff23 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Wed, 13 Sep 2023 14:20:21 +0200 Subject: refactor(tvix/store/blobsvc): make BlobStore async We previously kept the trait of a BlobService sync. This however had some annoying consequences: - It became more and more complicated to track when we're in a context with an async runtime in the context or not, producing bugs like https://b.tvl.fyi/issues/304 - The sync trait shielded away async clients from async worloads, requiring manual block_on code inside the gRPC client code, and spawn_blocking calls in consumers of the trait, even if they were async (like the gRPC server) - We had to write our own custom glue code (SyncReadIntoAsyncRead) to convert a sync io::Read into a tokio::io::AsyncRead, which already existed in tokio internally, but upstream ia hesitant to expose. This now makes the BlobService trait async (via the async_trait macro, like we already do in various gRPC parts), and replaces the sync readers and writers with their async counterparts. Tests interacting with a BlobService now need to have an async runtime available, the easiest way for this is to mark the test functions with the tokio::test macro, allowing us to directly .await in the test function. In places where we don't have an async runtime available from context (like tvix-cli), we can pass one down explicitly. Now that we don't provide a sync interface anymore, the (sync) FUSE library now holds a pointer to a tokio runtime handle, and needs to at least have 2 threads available when talking to a blob service (which is why some of the tests now use the multi_thread flavor). The FUSE tests got a bit more verbose, as we couldn't use the setup_and_mount function accepting a callback anymore. We can hopefully move some of the test fixture setup to rstest in the future to make this less repetitive. Co-Authored-By: Connor Brewster Change-Id: Ia0501b606e32c852d0108de9c9016b21c94a3c05 Reviewed-on: https://cl.tvl.fyi/c/depot/+/9329 Reviewed-by: Connor Brewster Tested-by: BuildkiteCI Reviewed-by: raitobezarius --- tvix/Cargo.nix | 283 +++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 255 insertions(+), 28 deletions(-) (limited to 'tvix/Cargo.nix') diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix index 4089008aaad6..e967b345e26f 100644 --- a/tvix/Cargo.nix +++ b/tvix/Cargo.nix @@ -129,6 +129,50 @@ rec { # inject test dependencies into the build crates = { + "addr2line" = rec { + crateName = "addr2line"; + version = "0.21.0"; + edition = "2018"; + sha256 = "1jx0k3iwyqr8klqbzk6kjvr496yd94aspis10vwsj5wy7gib4c4a"; + dependencies = [ + { + name = "gimli"; + packageId = "gimli"; + usesDefaultFeatures = false; + features = [ "read" ]; + } + ]; + features = { + "alloc" = [ "dep:alloc" ]; + "compiler_builtins" = [ "dep:compiler_builtins" ]; + "core" = [ "dep:core" ]; + "cpp_demangle" = [ "dep:cpp_demangle" ]; + "default" = [ "rustc-demangle" "cpp_demangle" "std-object" "fallible-iterator" "smallvec" "memmap2" ]; + "fallible-iterator" = [ "dep:fallible-iterator" ]; + "memmap2" = [ "dep:memmap2" ]; + "object" = [ "dep:object" ]; + "rustc-demangle" = [ "dep:rustc-demangle" ]; + "rustc-dep-of-std" = [ "core" "alloc" "compiler_builtins" "gimli/rustc-dep-of-std" ]; + "smallvec" = [ "dep:smallvec" ]; + "std" = [ "gimli/std" ]; + "std-object" = [ "std" "object" "object/std" "object/compression" "gimli/endian-reader" ]; + }; + }; + "adler" = rec { + crateName = "adler"; + version = "1.0.2"; + edition = "2015"; + sha256 = "1zim79cvzd5yrkzl3nyfx0avijwgk9fqv3yrscdy1cc79ih02qpj"; + authors = [ + "Jonas Schievink " + ]; + features = { + "compiler_builtins" = [ "dep:compiler_builtins" ]; + "core" = [ "dep:core" ]; + "default" = [ "std" ]; + "rustc-dep-of-std" = [ "core" "compiler_builtins" ]; + }; + }; "aho-corasick" = rec { crateName = "aho-corasick"; version = "1.0.1"; @@ -618,6 +662,67 @@ rec { "tracing" = [ "dep:tracing" ]; }; }; + "backtrace" = rec { + crateName = "backtrace"; + version = "0.3.69"; + edition = "2018"; + sha256 = "0dsq23dhw4pfndkx2nsa1ml2g31idm7ss7ljxp8d57avygivg290"; + authors = [ + "The Rust Project Developers" + ]; + dependencies = [ + { + name = "addr2line"; + packageId = "addr2line"; + usesDefaultFeatures = false; + target = { target, features }: (!((target."windows" or false) && ("msvc" == target."env") && (!("uwp" == target."vendor")))); + } + { + name = "cfg-if"; + packageId = "cfg-if"; + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: (!((target."windows" or false) && ("msvc" == target."env") && (!("uwp" == target."vendor")))); + } + { + name = "miniz_oxide"; + packageId = "miniz_oxide"; + usesDefaultFeatures = false; + target = { target, features }: (!((target."windows" or false) && ("msvc" == target."env") && (!("uwp" == target."vendor")))); + } + { + name = "object"; + packageId = "object"; + usesDefaultFeatures = false; + target = { target, features }: (!((target."windows" or false) && ("msvc" == target."env") && (!("uwp" == target."vendor")))); + features = [ "read_core" "elf" "macho" "pe" "unaligned" "archive" ]; + } + { + name = "rustc-demangle"; + packageId = "rustc-demangle"; + } + ]; + buildDependencies = [ + { + name = "cc"; + packageId = "cc"; + } + ]; + features = { + "cpp_demangle" = [ "dep:cpp_demangle" ]; + "default" = [ "std" ]; + "rustc-serialize" = [ "dep:rustc-serialize" ]; + "serde" = [ "dep:serde" ]; + "serialize-rustc" = [ "rustc-serialize" ]; + "serialize-serde" = [ "serde" ]; + "verify-winapi" = [ "winapi/dbghelp" "winapi/handleapi" "winapi/libloaderapi" "winapi/memoryapi" "winapi/minwindef" "winapi/processthreadsapi" "winapi/synchapi" "winapi/tlhelp32" "winapi/winbase" "winapi/winnt" ]; + "winapi" = [ "dep:winapi" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "base64 0.10.1" = rec { crateName = "base64"; version = "0.10.1"; @@ -2586,6 +2691,23 @@ rec { }; resolvedDefaultFeatures = [ "std" ]; }; + "gimli" = rec { + crateName = "gimli"; + version = "0.28.0"; + edition = "2018"; + sha256 = "1h7hcl3chfvd2gfrrxjymnwj7anqxjslvz20kcargkvsya2dgf3g"; + features = { + "default" = [ "read-all" "write" ]; + "endian-reader" = [ "read" "dep:stable_deref_trait" ]; + "fallible-iterator" = [ "dep:fallible-iterator" ]; + "read" = [ "read-core" ]; + "read-all" = [ "read" "std" "fallible-iterator" "endian-reader" ]; + "rustc-dep-of-std" = [ "dep:core" "dep:alloc" "dep:compiler_builtins" ]; + "std" = [ "fallible-iterator?/std" "stable_deref_trait?/std" ]; + "write" = [ "dep:indexmap" ]; + }; + resolvedDefaultFeatures = [ "read" "read-core" ]; + }; "glob" = rec { crateName = "glob"; version = "0.3.1"; @@ -2925,7 +3047,7 @@ rec { } { name = "socket2"; - packageId = "socket2"; + packageId = "socket2 0.4.9"; optional = true; features = [ "all" ]; } @@ -3570,9 +3692,9 @@ rec { }; "libc" = rec { crateName = "libc"; - version = "0.2.144"; + version = "0.2.148"; edition = "2015"; - sha256 = "1qfzrwhncsradwvdzd8vsj4mc31fh0rb5rvny3884rwa48fcq01b"; + sha256 = "16rn9l8s5sj9n2jb2pw13ghqwa5nvjggkh9q3lp6vs1jfghp3p4w"; authors = [ "The Rust Project Developers" ]; @@ -3722,6 +3844,32 @@ rec { ]; }; + "miniz_oxide" = rec { + crateName = "miniz_oxide"; + version = "0.7.1"; + edition = "2018"; + sha256 = "1ivl3rbbdm53bzscrd01g60l46lz5krl270487d8lhjvwl5hx0g7"; + authors = [ + "Frommi " + "oyvindln " + ]; + dependencies = [ + { + name = "adler"; + packageId = "adler"; + usesDefaultFeatures = false; + } + ]; + features = { + "alloc" = [ "dep:alloc" ]; + "compiler_builtins" = [ "dep:compiler_builtins" ]; + "core" = [ "dep:core" ]; + "default" = [ "with-alloc" ]; + "rustc-dep-of-std" = [ "core" "alloc" "compiler_builtins" "adler/rustc-dep-of-std" ]; + "simd" = [ "simd-adler32" ]; + "simd-adler32" = [ "dep:simd-adler32" ]; + }; + }; "mio" = rec { crateName = "mio"; version = "0.8.6"; @@ -3762,7 +3910,7 @@ rec { features = { "os-ext" = [ "os-poll" "windows-sys/Win32_System_Pipes" "windows-sys/Win32_Security" ]; }; - resolvedDefaultFeatures = [ "default" "net" "os-ext" "os-poll" ]; + resolvedDefaultFeatures = [ "net" "os-ext" "os-poll" ]; }; "multimap" = rec { crateName = "multimap"; @@ -4035,6 +4183,38 @@ rec { ]; }; + "object" = rec { + crateName = "object"; + version = "0.32.1"; + edition = "2018"; + sha256 = "1c02x4kvqpnl3wn7gz9idm4jrbirbycyqjgiw6lm1g9k77fzkxcw"; + dependencies = [ + { + name = "memchr"; + packageId = "memchr"; + usesDefaultFeatures = false; + } + ]; + features = { + "all" = [ "read" "write" "std" "compression" "wasm" ]; + "alloc" = [ "dep:alloc" ]; + "compiler_builtins" = [ "dep:compiler_builtins" ]; + "compression" = [ "dep:flate2" "dep:ruzstd" "std" ]; + "core" = [ "dep:core" ]; + "default" = [ "read" "compression" ]; + "doc" = [ "read_core" "write_std" "std" "compression" "archive" "coff" "elf" "macho" "pe" "wasm" "xcoff" ]; + "pe" = [ "coff" ]; + "read" = [ "read_core" "archive" "coff" "elf" "macho" "pe" "xcoff" "unaligned" ]; + "rustc-dep-of-std" = [ "core" "compiler_builtins" "alloc" "memchr/rustc-dep-of-std" ]; + "std" = [ "memchr/std" ]; + "unstable-all" = [ "all" "unstable" ]; + "wasm" = [ "dep:wasmparser" ]; + "write" = [ "write_std" "coff" "elf" "macho" "pe" "xcoff" ]; + "write_core" = [ "dep:crc32fast" "dep:indexmap" "dep:hashbrown" ]; + "write_std" = [ "write_core" "std" "indexmap?/std" "crc32fast?/std" ]; + }; + resolvedDefaultFeatures = [ "archive" "coff" "elf" "macho" "pe" "read_core" "unaligned" ]; + }; "once_cell" = rec { crateName = "once_cell"; version = "1.17.1"; @@ -4316,9 +4496,9 @@ rec { }; "pin-project-lite" = rec { crateName = "pin-project-lite"; - version = "0.2.9"; + version = "0.2.13"; edition = "2018"; - sha256 = "05n1z851l356hpgqadw4ar64mjanaxq1qlwqsf2k05ziq8xax9z0"; + sha256 = "0n0bwr5qxlf0mhn2xkl36sy55118s9qmvx2yl5f3ixkb007lbywa"; }; "pin-utils" = rec { @@ -5518,6 +5698,20 @@ rec { "serde1" = [ "serde" "text-size/serde" ]; }; }; + "rustc-demangle" = rec { + crateName = "rustc-demangle"; + version = "0.1.23"; + edition = "2015"; + sha256 = "0xnbk2bmyzshacjm2g1kd4zzv2y2az14bw3sjccq5qkpmsfvn9nn"; + authors = [ + "Alex Crichton " + ]; + features = { + "compiler_builtins" = [ "dep:compiler_builtins" ]; + "core" = [ "dep:core" ]; + "rustc-dep-of-std" = [ "core" "compiler_builtins" ]; + }; + }; "rustc-hash" = rec { crateName = "rustc-hash"; version = "1.1.0"; @@ -6242,7 +6436,7 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "socket2" = rec { + "socket2 0.4.9" = rec { crateName = "socket2"; version = "0.4.9"; edition = "2018"; @@ -6267,6 +6461,31 @@ rec { features = { }; resolvedDefaultFeatures = [ "all" ]; }; + "socket2 0.5.4" = rec { + crateName = "socket2"; + version = "0.5.4"; + edition = "2021"; + sha256 = "17lqx8w2b3nysrkdbdz8y7fkikz5v77c052q57lxwajmxchfhca0"; + authors = [ + "Alex Crichton " + "Thomas de Zeeuw " + ]; + dependencies = [ + { + name = "libc"; + packageId = "libc"; + target = { target, features }: (target."unix" or false); + } + { + name = "windows-sys"; + packageId = "windows-sys 0.48.0"; + target = { target, features }: (target."windows" or false); + features = [ "Win32_Foundation" "Win32_Networking_WinSock" "Win32_System_IO" "Win32_System_Threading" "Win32_System_WindowsProgramming" ]; + } + ]; + features = { }; + resolvedDefaultFeatures = [ "all" ]; + }; "ssri" = rec { crateName = "ssri"; version = "7.0.0"; @@ -6909,13 +7128,18 @@ rec { }; "tokio" = rec { crateName = "tokio"; - version = "1.28.0"; + version = "1.32.0"; edition = "2021"; - sha256 = "0vqk7dkmvadzqrxwlgja04wlf4s8iymjk6yvcshs7r9lh6zqdiy3"; + sha256 = "1yck1349q23l22bgxcbqd3wkaffw2vmkf7z26m3wgmkcxmvn1v8p"; authors = [ "Tokio Contributors " ]; dependencies = [ + { + name = "backtrace"; + packageId = "backtrace"; + target = { target, features }: (target."tokio_taskdump" or false); + } { name = "bytes"; packageId = "bytes"; @@ -6931,6 +7155,7 @@ rec { name = "mio"; packageId = "mio"; optional = true; + usesDefaultFeatures = false; } { name = "num_cpus"; @@ -6949,9 +7174,9 @@ rec { } { name = "socket2"; - packageId = "socket2"; + packageId = "socket2 0.5.4"; optional = true; - target = { target, features }: (!(("wasm32" == target."arch") || ("wasm64" == target."arch"))); + target = { target, features }: (!(builtins.elem "wasm" target."family")); features = [ "all" ]; } { @@ -6959,12 +7184,6 @@ rec { packageId = "tokio-macros"; optional = true; } - { - name = "windows-sys"; - packageId = "windows-sys 0.48.0"; - target = { target, features }: (target."docsrs" or false); - features = [ "Win32_Foundation" "Win32_Security_Authorization" ]; - } { name = "windows-sys"; packageId = "windows-sys 0.48.0"; @@ -6972,12 +7191,6 @@ rec { target = { target, features }: (target."windows" or false); } ]; - buildDependencies = [ - { - name = "autocfg"; - packageId = "autocfg"; - } - ]; devDependencies = [ { name = "libc"; @@ -6986,8 +7199,14 @@ rec { } { name = "socket2"; - packageId = "socket2"; - target = { target, features }: (!(("wasm32" == target."arch") || ("wasm64" == target."arch"))); + packageId = "socket2 0.5.4"; + target = { target, features }: (!(builtins.elem "wasm" target."family")); + } + { + name = "windows-sys"; + packageId = "windows-sys 0.48.0"; + target = { target, features }: (target."windows" or false); + features = [ "Win32_Foundation" "Win32_Security_Authorization" ]; } ]; features = { @@ -7010,7 +7229,7 @@ rec { "tracing" = [ "dep:tracing" ]; "windows-sys" = [ "dep:windows-sys" ]; }; - resolvedDefaultFeatures = [ "bytes" "default" "io-std" "io-util" "libc" "macros" "mio" "net" "num_cpus" "rt" "rt-multi-thread" "signal" "signal-hook-registry" "socket2" "sync" "time" "tokio-macros" "windows-sys" ]; + resolvedDefaultFeatures = [ "bytes" "default" "fs" "io-std" "io-util" "libc" "macros" "mio" "net" "num_cpus" "rt" "rt-multi-thread" "signal" "signal-hook-registry" "socket2" "sync" "time" "tokio-macros" "windows-sys" ]; }; "tokio-io-timeout" = rec { crateName = "tokio-io-timeout"; @@ -8073,6 +8292,10 @@ rec { name = "thiserror"; packageId = "thiserror"; } + { + name = "tokio"; + packageId = "tokio"; + } { name = "tracing"; packageId = "tracing"; @@ -8354,6 +8577,10 @@ rec { name = "nix-compat"; packageId = "nix-compat"; } + { + name = "pin-project-lite"; + packageId = "pin-project-lite"; + } { name = "prost"; packageId = "prost"; @@ -8386,7 +8613,7 @@ rec { { name = "tokio"; packageId = "tokio"; - features = [ "net" "rt-multi-thread" "signal" ]; + features = [ "fs" "net" "rt-multi-thread" "signal" ]; } { name = "tokio-stream"; @@ -10007,7 +10234,7 @@ rec { "Win32_Web" = [ "Win32" ]; "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; }; - resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_NetworkManagement" "Win32_NetworkManagement_IpHelper" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authorization" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_Pipes" "Win32_System_SystemServices" "Win32_System_Threading" "default" ]; + resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_NetworkManagement" "Win32_NetworkManagement_IpHelper" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_Pipes" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_WindowsProgramming" "default" ]; }; "windows-targets 0.42.2" = rec { crateName = "windows-targets"; -- cgit 1.4.1