diff options
253 files changed, 15155 insertions, 13452 deletions
diff --git a/default.nix b/default.nix index 0ffbdbabaa..015fdd1ead 100644 --- a/default.nix +++ b/default.nix @@ -110,6 +110,13 @@ readTree.fix (self: (readDepot { # xanthous and related targets are disabled until cl/9186 is submitted self.users.aspen.xanthous self.users.aspen.system.system.mugwumpSystem + + # Temporarily disabled after cl/11289. Hopefully these failures are transient + # and will disappear with the next channel bump. + self.users.wpcarro.nixos.avaSystem + self.users.wpcarro.nixos.kyokoSystem + self.users.wpcarro.nixos.marcusSystem + self.users.wpcarro.nixos.tarascoSystem ]; # List of all buildable targets, for CI purposes. diff --git a/nix/buildGo/default.nix b/nix/buildGo/default.nix index 9e0a5d6d87..c93642a127 100644 --- a/nix/buildGo/default.nix +++ b/nix/buildGo/default.nix @@ -42,8 +42,6 @@ let xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs)); - pathToName = p: replaceStrings [ "/" ] [ "_" ] (toString p); - # Add an `overrideGo` attribute to a function result that works # similar to `overrideAttrs`, but is used specifically for the # arguments passed to Go builders. diff --git a/nix/buildGo/external/main.go b/nix/buildGo/external/main.go index a77c43b371..4402a8eb86 100644 --- a/nix/buildGo/external/main.go +++ b/nix/buildGo/external/main.go @@ -10,7 +10,6 @@ import ( "flag" "fmt" "go/build" - "io/ioutil" "log" "os" "path" @@ -74,8 +73,8 @@ func findGoDirs(at string) ([]string, error) { } goDirs := []string{} - for k, _ := range dirSet { - goDirs = append(goDirs, k) + for goDir := range dirSet { + goDirs = append(goDirs, goDir) } return goDirs, nil @@ -148,7 +147,7 @@ func analysePackage(root, source, importpath string, stdlib map[string]bool) (pk } func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) { - f, err := ioutil.ReadFile(from) + f, err := os.ReadFile(from) if err != nil { return } diff --git a/nix/buildkite/default.nix b/nix/buildkite/default.nix index 6e158ae6c6..9abba9408a 100644 --- a/nix/buildkite/default.nix +++ b/nix/buildkite/default.nix @@ -27,8 +27,27 @@ let inherit (pkgs) lib runCommand writeText; inherit (depot.nix.readTree) mkLabel; + + inherit (depot.nix) dependency-analyzer; in rec { + # Create a unique key for the buildkite pipeline based on the given derivation + # or drvPath. A consequence of using such keys is that every derivation may + # only be exposed as a single, unique step in the pipeline. + keyForDrv = drvOrPath: + let + drvPath = + if lib.isDerivation drvOrPath then drvOrPath.drvPath + else if lib.isString drvOrPath then drvOrPath + else builtins.throw "keyForDrv: expected string or derivation"; + + # Only use the drv hash to prevent escaping problems. Buildkite also has a + # limit of 100 characters on keys. + in + "drv-" + (builtins.substring 0 32 + (builtins.baseNameOf (unsafeDiscardStringContext drvPath)) + ); + # Given an arbitrary attribute path generate a Nix expression which obtains # this from the root of depot (assumed to be ./.). Attributes may be any # Nix strings suitable as attribute names, not just Nix literal-safe strings. @@ -47,6 +66,9 @@ rec { # Create build command for an attribute path pointing to a derivation. mkBuildCommand = { attrPath, drvPath, outLink ? "result" }: concatStringsSep " " [ + # If the nix build fails, the Nix command's exit status should be used. + "set -o pipefail;" + # First try to realise the drvPath of the target so we don't evaluate twice. # Nix has no concept of depending on a derivation file without depending on # at least one of its `outPath`s, so we need to discard the string context @@ -55,7 +77,7 @@ rec { # To make this more uniform with how nix-build(1) works, we call realpath(1) # on nix-store(1)'s output since it has the habit of printing the path of the # out link, not the store path. - "(nix-store --realise '${drvPath}' --add-root '${outLink}' --indirect | xargs realpath)" + "(nix-store --realise '${drvPath}' --add-root '${outLink}' --indirect | xargs -r realpath)" # Since we don't gcroot the derivation files, they may be deleted by the # garbage collector. In that case we can reevaluate and build the attribute @@ -70,15 +92,28 @@ rec { target.__readTree ++ lib.optionals (target ? __subtarget) [ target.__subtarget ]; + # Given a derivation (identified by drvPath) that is part of the list of + # targets passed to mkPipeline, determine all derivations that it depends on + # and are also part of the pipeline. Finally, return the keys of the steps + # that build them. This is used to populate `depends_on` in `mkStep`. + # + # See //nix/dependency-analyzer for documentation on the structure of `targetDepMap`. + getTargetPipelineDeps = targetDepMap: drvPath: + # Sanity check: We should only call this function on targets explicitly + # passed to mkPipeline. Thus it should have been passed as a โknownโ drv to + # dependency-analyzer. + assert targetDepMap.${drvPath}.known; + builtins.map keyForDrv targetDepMap.${drvPath}.knownDeps; + # Create a pipeline step from a single target. - mkStep = { headBranch, parentTargetMap, target, cancelOnBuildFailing }: + mkStep = { headBranch, parentTargetMap, targetDepMap, target, cancelOnBuildFailing }: let label = mkLabel target; drvPath = unsafeDiscardStringContext target.drvPath; in { label = ":nix: " + label; - key = hashString "sha1" label; + key = keyForDrv target; skip = shouldSkip { inherit label drvPath parentTargetMap; }; command = mkBuildCommand { attrPath = targetAttrPath target; @@ -90,7 +125,9 @@ rec { # Add a dependency on the initial static pipeline step which # always runs. This allows build steps uploaded in batches to # start running before all batches have been uploaded. - depends_on = [ ":init:" ] ++ lib.optionals (target ? meta.ci.buildkiteExtraDeps) target.meta.ci.buildkiteExtraDeps; + depends_on = [ ":init:" ] + ++ getTargetPipelineDeps targetDepMap drvPath + ++ lib.optionals (target ? meta.ci.buildkiteExtraDeps) target.meta.ci.buildkiteExtraDeps; } // lib.optionalAttrs (target ? meta.timeout) { timeout_in_minutes = target.meta.timeout / 60; # Additional arguments to set on the step. @@ -193,12 +230,15 @@ rec { # logic/optimisation depends on knowing whether is executing. buildEnabled = elem "build" enabledPhases; + # Dependency relations between the `drvTargets`. See also //nix/dependency-analyzer. + targetDepMap = dependency-analyzer (dependency-analyzer.drvsToPaths drvTargets); + # Convert a target into all of its steps, separated by build # phase (as phases end up in different chunks). targetToSteps = target: let mkStepArgs = { - inherit headBranch parentTargetMap target cancelOnBuildFailing; + inherit headBranch parentTargetMap targetDepMap target cancelOnBuildFailing; }; step = mkStep mkStepArgs; @@ -376,7 +416,7 @@ rec { prompt = lib.throwIf (prompt != false && phase == "build") '' In step '${label}' (from ${parentLabel}): - The 'prompt' feature can only be used by steps in the "release" + The 'prompt' feature can not be used by steps in the "build" phase, because CI builds should not be gated on manual human approvals. '' @@ -391,7 +431,7 @@ rec { commandScriptLink = "nix-buildkite-extra-step-command-script"; step = { - key = hashString "sha1" "${cfg.label}-${cfg.parentLabel}"; + key = "extra-step-" + hashString "sha1" "${cfg.label}-${cfg.parentLabel}"; label = ":gear: ${cfg.label} (from ${cfg.parentLabel})"; skip = let diff --git a/nix/dependency-analyzer/default.nix b/nix/dependency-analyzer/default.nix index 54ff72912e..2ec8d7b5b9 100644 --- a/nix/dependency-analyzer/default.nix +++ b/nix/dependency-analyzer/default.nix @@ -16,30 +16,35 @@ let # # TODO(sterni): clean this up and expose it directDrvDeps = - if lib.versionAtLeast builtins.nixVersion "2.6" - then - # Since https://github.com/NixOS/nix/pull/1643, Nix apparently ยปpreserves - # string contextยซ through a readFile invocation. This has the side effect - # that it becomes possible to query the actual references a store path has. - # Not a 100% sure this is intended, but _very_ convenient for us here. - drvPath: - # if the passed path is not a derivation we can't necessarily get its - # dependencies, since it may not be representable as a Nix string due to - # NUL bytes, e.g. compressed patch files imported into the Nix store. - if builtins.match "^.+\\.drv$" drvPath == null - then [ ] - else builtins.attrNames (builtins.getContext (builtins.readFile drvPath)) - else - # For Nix < 2.6 we have to rely on HACK, namely grepping for quoted store - # path references in the file. In the future this should be replaced by - # a proper derivation parser. - drvPath: builtins.concatLists ( - builtins.filter builtins.isList ( - builtins.split - "\"(${lib.escapeRegex builtins.storeDir}/[[:alnum:]+._?=-]+.drv)\"" - (builtins.readFile drvPath) - ) - ); + let + getDeps = + if lib.versionAtLeast builtins.nixVersion "2.6" + then + # Since https://github.com/NixOS/nix/pull/1643, Nix apparently ยปpreserves + # string contextยซ through a readFile invocation. This has the side effect + # that it becomes possible to query the actual references a store path has. + # Not a 100% sure this is intended, but _very_ convenient for us here. + drvPath: + builtins.attrNames (builtins.getContext (builtins.readFile drvPath)) + else + # For Nix < 2.6 we have to rely on HACK, namely grepping for quoted + # store path references in the file. In the future this should be + # replaced by a proper derivation parser. + drvPath: builtins.concatLists ( + builtins.filter builtins.isList ( + builtins.split + "\"(${lib.escapeRegex builtins.storeDir}/[[:alnum:]+._?=-]+.drv)\"" + (builtins.readFile drvPath) + ) + ); + in + drvPath: + # if the passed path is not a derivation we can't necessarily get its + # dependencies, since it may not be representable as a Nix string due to + # NUL bytes, e.g. compressed patch files imported into the Nix store. + if builtins.match "^.+\\.drv$" drvPath == null + then [ ] + else getDeps drvPath; # Maps a list of derivation to the list of corresponding `drvPath`s. # diff --git a/nix/nix-1p/README.md b/nix/nix-1p/README.md index 45643032d0..309eddb51e 100644 --- a/nix/nix-1p/README.md +++ b/nix/nix-1p/README.md @@ -1,3 +1,8 @@ +> [!TIP] +> Are you interested in hacking on Nix projects for a week, together +> with other Nix users? Do you have time at the end of August? Great, +> come join us at [Volga Sprint](https://volgasprint.org/)! + Nix - A One Pager ================= diff --git a/ops/glesys/dns-nixery-dev.tf b/ops/glesys/dns-nixery-dev.tf index 0ea5d313d0..42bcec7e21 100644 --- a/ops/glesys/dns-nixery-dev.tf +++ b/ops/glesys/dns-nixery-dev.tf @@ -12,7 +12,7 @@ resource "glesys_dnsdomain_record" "nixery_dev_apex_A" { domain = glesys_dnsdomain.nixery_dev.id host = "@" type = "A" - data = "51.250.48.185" # nixery-01.tvl.fyi + data = "51.250.51.78" # nixery-01.tvl.fyi } resource "glesys_dnsdomain_record" "nixery_dev_NS1" { diff --git a/ops/glesys/dns-tvl-fyi.tf b/ops/glesys/dns-tvl-fyi.tf index e448bc4486..9d7972c412 100644 --- a/ops/glesys/dns-tvl-fyi.tf +++ b/ops/glesys/dns-tvl-fyi.tf @@ -57,7 +57,7 @@ resource "glesys_dnsdomain_record" "tvl_fyi_nixery-01_A" { domain = glesys_dnsdomain.tvl_fyi.id host = "nixery-01" type = "A" - data = "51.250.48.185" + data = "51.250.51.78" } # Explicit records for all services running on whitby diff --git a/ops/machines/all-systems.nix b/ops/machines/all-systems.nix index 449d41a5f7..c4382fbddb 100644 --- a/ops/machines/all-systems.nix +++ b/ops/machines/all-systems.nix @@ -16,6 +16,7 @@ yeren mugwump ogopogo + lusca ]) ++ (with depot.users.wpcarro.nixos; [ diff --git a/ops/machines/nixery-01/default.nix b/ops/machines/nixery-01/default.nix index c7c8fd4b5e..c99db214d8 100644 --- a/ops/machines/nixery-01/default.nix +++ b/ops/machines/nixery-01/default.nix @@ -28,4 +28,13 @@ in }; services.depot.nixery.enable = true; + + # Automatically collect garbage from the Nix store. + services.depot.automatic-gc = { + enable = true; + interval = "1 hour"; + diskThreshold = 25; # GiB + maxFreed = 150; # GiB + preserveGenerations = "30d"; + }; } diff --git a/ops/modules/automatic-gc.nix b/ops/modules/automatic-gc.nix index ad53a63f7f..003f160919 100644 --- a/ops/modules/automatic-gc.nix +++ b/ops/modules/automatic-gc.nix @@ -13,6 +13,11 @@ let gcScript = pkgs.writeShellScript "automatic-nix-gc" '' set -ueo pipefail + if [ -e /run/stop-automatic-gc ]; then + echo "GC is disabled through /run/stop-automatic-gc" + exit 0 + fi + readonly MIN_THRESHOLD_KIB="${toString (GiBtoKiB cfg.diskThreshold)}" readonly MAX_FREED_BYTES="${toString (GiBtoBytes cfg.maxFreed)}" readonly GEN_THRESHOLD="${cfg.preserveGenerations}" diff --git a/ops/modules/josh.nix b/ops/modules/josh.nix index 4591ebf0f0..3c37d0fec3 100644 --- a/ops/modules/josh.nix +++ b/ops/modules/josh.nix @@ -26,7 +26,7 @@ in DynamicUser = true; StateDirectory = "josh"; Restart = "always"; - ExecStart = "${depot.third_party.josh}/bin/josh-proxy --no-background --local /var/lib/josh --port ${toString cfg.port} --remote https://cl.tvl.fyi/ --require-auth"; + ExecStart = "${pkgs.josh}/bin/josh-proxy --no-background --local /var/lib/josh --port ${toString cfg.port} --remote https://cl.tvl.fyi/ --require-auth"; }; }; }; diff --git a/ops/modules/www/tazj.in.nix b/ops/modules/www/tazj.in.nix index 3b80222e0d..47eefca2a6 100644 --- a/ops/modules/www/tazj.in.nix +++ b/ops/modules/www/tazj.in.nix @@ -33,6 +33,11 @@ return 302 https://predlozhnik.ru; } + # redirect for easier entry on a TV + location = /tv { + return 302 https://tazj.in/blobs/play.html; + } + # Temporary place for serving static files. location /blobs/ { alias /var/lib/tazjins-blobs/; diff --git a/ops/users/default.nix b/ops/users/default.nix index 34e0ab85c3..c54a681dce 100644 --- a/ops/users/default.nix +++ b/ops/users/default.nix @@ -224,4 +224,9 @@ email = "espes@pequalsnp.com"; password = "{ARGON2}$argon2id$v=19$m=19456,t=2,p=1$eXeFrbNxuKn/JCpQr5VmxA$NtMNBceNg/JtqMfHk/qHxEHsEVsTWmHJbpq4ve/+XYg"; } + { + username = "caralice"; + email = "tvl@alice-carroll.pet"; + password = "{ARGON2}$argon2id$v=19$m=19456,t=2,p=1$mt/0RzKw4RHxm7ybpMHP5Q$P/SDBMv5si9D98NFO/eZgh2+InlByqYxqAvQWhl+p0c"; + } ] diff --git a/third_party/josh/default.nix b/third_party/josh/default.nix deleted file mode 100644 index 9750780d1f..0000000000 --- a/third_party/josh/default.nix +++ /dev/null @@ -1,49 +0,0 @@ -# https://github.com/josh-project/josh -{ depot, pkgs, ... }: - -let - # TODO(sterni): switch to pkgs.josh as soon as that commit is released - rev = "1586eab06284ce668779c87f00a1fb5fa9763be0"; - src = pkgs.fetchFromGitHub { - owner = "josh-project"; - repo = "josh"; - inherit rev; - hash = "sha256-94QrHcVHiEMCpBZJ5sghwtVNLNm4gdG8X85OetoGRD0="; - }; - - - naersk = pkgs.callPackage depot.third_party.sources.naersk { - inherit (pkgs) rustc cargo; - }; - version = "git-${builtins.substring 0 8 rev}"; -in -naersk.buildPackage { - pname = "josh"; - inherit src version; - JOSH_VERSION = version; - - buildInputs = with pkgs; [ - libgit2 - openssl - pkg-config - ]; - - dontStrip = true; - cargoBuildOptions = x: x ++ [ - "-p" - "josh-filter" - "-p" - "josh-proxy" - ]; - - overrideMain = x: { - preBuild = x.preBuild or "" + '' - echo 'debug = true' >> Cargo.toml - ''; - - nativeBuildInputs = (x.nativeBuildInputs or [ ]) ++ [ pkgs.makeWrapper ]; - postInstall = '' - wrapProgram $out/bin/josh-proxy --prefix PATH : "${pkgs.git}/bin" - ''; - }; -} diff --git a/third_party/lisp/cl-change-case.nix b/third_party/lisp/cl-change-case.nix new file mode 100644 index 0000000000..b66368a9b6 --- /dev/null +++ b/third_party/lisp/cl-change-case.nix @@ -0,0 +1,22 @@ +{ depot, pkgs, ... }: + +let src = with pkgs; srcOnly lispPackages.cl-change-case; +in depot.nix.buildLisp.library { + name = "cl-change-case"; + + deps = with depot.third_party.lisp; [ cl-ppcre cl-ppcre.unicode ]; + + srcs = [ (src + "/src/cl-change-case.lisp") ]; + + tests = { + name = "cl-change-case-tests"; + srcs = [ (src + "/t/cl-change-case.lisp") ]; + deps = [ + depot.third_party.lisp.fiveam + ]; + + expression = '' + (5am:run! :cl-change-case) + ''; + }; +} diff --git a/third_party/lisp/cl-ppcre.nix b/third_party/lisp/cl-ppcre.nix index 561e306191..7cb99db639 100644 --- a/third_party/lisp/cl-ppcre.nix +++ b/third_party/lisp/cl-ppcre.nix @@ -24,4 +24,16 @@ in depot.nix.buildLisp.library { "scanner.lisp" "api.lisp" ]; + + passthru = { + unicode = depot.nix.buildLisp.library { + name = "cl-ppcre-unicode"; + deps = with depot.third_party.lisp; [ cl-ppcre cl-unicode ]; + + srcs = map (f: src + ("/cl-ppcre-unicode/" + f)) [ + "packages.lisp" + "resolver.lisp" + ]; + }; + }; } diff --git a/third_party/lisp/mime4cl/default.nix b/third_party/lisp/mime4cl/default.nix index 99b23c91aa..af015a257b 100644 --- a/third_party/lisp/mime4cl/default.nix +++ b/third_party/lisp/mime4cl/default.nix @@ -10,6 +10,7 @@ depot.nix.buildLisp.library { depot.third_party.lisp.npg depot.third_party.lisp.trivial-gray-streams depot.third_party.lisp.qbase64 + { sbcl = depot.nix.buildLisp.bundled "sb-posix"; } ]; srcs = [ diff --git a/third_party/lisp/mime4cl/ex-sclf.lisp b/third_party/lisp/mime4cl/ex-sclf.lisp index 7951b44f4d..1719732fb3 100644 --- a/third_party/lisp/mime4cl/ex-sclf.lisp +++ b/third_party/lisp/mime4cl/ex-sclf.lisp @@ -32,6 +32,8 @@ (defpackage :mime4cl-ex-sclf (:use :common-lisp) + (:import-from :sb-posix :stat :stat-size) + (:export #:aif #:awhen @@ -62,8 +64,6 @@ #:save-file-excursion #:read-file - #:unix-file-stat - #:unix-stat #:file-size #:promise @@ -275,51 +275,12 @@ ELEMENT-TYPE." #-sbcl (let (#+cmu (lisp::*ignore-wildcards* t)) (namestring pathname))) -(defstruct (unix-file-stat (:conc-name stat-)) - device - inode - links - atime - mtime - ctime - size - blksize - blocks - uid - gid - mode) - -(defun unix-stat (pathname) - ;; this could be different depending on the unix systems - (multiple-value-bind (ok? device inode mode links uid gid rdev - size atime mtime ctime - blksize blocks) - (#+cmu unix:unix-lstat - #+sbcl sb-unix:unix-lstat - ;; TODO(sterni): ECL, CCL - (if (stringp pathname) - pathname - (native-namestring pathname))) - (declare (ignore rdev)) - (when ok? - (make-unix-file-stat :device device - :inode inode - :links links - :atime atime - :mtime mtime - :ctime ctime - :size size - :blksize blksize - :blocks blocks - :uid uid - :gid gid - :mode mode)))) - ;; FILE-LENGTH is a bit idiosyncratic in this respect. Besides, Unix ;; allows to get to know the file size without being able to open a ;; file; just ask politely. (defun file-size (pathname) - (stat-size (unix-stat pathname))) + #+sbcl (stat-size (unix-stat pathname)) + #-sbcl (error "nyi")) ;; LAZY diff --git a/third_party/lisp/str.nix b/third_party/lisp/str.nix new file mode 100644 index 0000000000..556f9cc307 --- /dev/null +++ b/third_party/lisp/str.nix @@ -0,0 +1,49 @@ +{ depot, pkgs, ... }: + +let + inherit (depot.nix) buildLisp; + src = with pkgs; srcOnly lispPackages.str; +in +buildLisp.library { + name = "str"; + + deps = with depot.third_party.lisp; [ + { + sbcl = buildLisp.bundled "uiop"; + default = buildLisp.bundled "asdf"; + } + cl-ppcre + cl-ppcre.unicode + cl-change-case + ]; + + srcs = [ + (pkgs.runCommand "str.lisp" { } '' + substitute ${src}/str.lisp $out \ + --replace-fail \ + '(asdf:component-version (asdf:find-system "str"))' \ + '"${pkgs.lispPackages.str.meta.version}"' + '') + ]; + + brokenOn = [ + "ccl" # In REPLACE-USING: Shouldn't assign to variable I + ]; + + tests = { + name = "str-test"; + srcs = [ (src + "/test/test-str.lisp") ]; + deps = [ + { + sbcl = depot.nix.buildLisp.bundled "uiop"; + default = depot.nix.buildLisp.bundled "asdf"; + } + depot.third_party.lisp.prove + depot.third_party.lisp.fiveam + ]; + + expression = '' + (fiveam:run! 'str::test-str) + ''; + }; +} diff --git a/third_party/nix-snapshotter/default.nix b/third_party/nix-snapshotter/default.nix deleted file mode 100644 index 58b30af25b..0000000000 --- a/third_party/nix-snapshotter/default.nix +++ /dev/null @@ -1,13 +0,0 @@ -# Imports the stable Nix build definitions for nix-snapshotter, a -# plugin to bring native support for Nix images to containerd -{ lib, pkgs, ... }: - -let - src = pkgs.fetchFromGitHub { - owner = "pdtpartners"; - repo = "nix-snapshotter"; - sha256 = "11sfy3kf046p8kacp7yh8ijjpp6php6q8wxlbya1v5q53h3980v1"; - rev = "6eb21bd3429535646da4aa396bb0c1f81a9b72c6"; - }; -in -pkgs.callPackage "${src}/package.nix" { } diff --git a/third_party/nixpkgs/default.nix b/third_party/nixpkgs/default.nix index 0db42f1540..b79a963e5c 100644 --- a/third_party/nixpkgs/default.nix +++ b/third_party/nixpkgs/default.nix @@ -13,6 +13,9 @@ , depotOverlays ? true , localSystem ? externalArgs.localSystem or builtins.currentSystem , crossSystem ? externalArgs.crossSystem or localSystem + # additional overlays to be applied. + # Useful when calling this file in a view exported from depot. +, additionalOverlays ? [ ] , ... }: @@ -65,10 +68,9 @@ import nixpkgsSrc (commonNixpkgsArgs // { stableOverlay ] ++ (if depotOverlays then [ depot.third_party.overlays.haskell - depot.third_party.overlays.emacs depot.third_party.overlays.tvl depot.third_party.overlays.ecl-static depot.third_party.overlays.dhall (import depot.third_party.sources.rust-overlay) - ] else [ ]); + ] else [ ] ++ additionalOverlays); }) diff --git a/third_party/overlays/emacs.nix b/third_party/overlays/emacs.nix deleted file mode 100644 index 341feb5015..0000000000 --- a/third_party/overlays/emacs.nix +++ /dev/null @@ -1,4 +0,0 @@ -# Emacs overlay from https://github.com/nix-community/emacs-overlay -{ depot, ... }: - -import depot.third_party.sources.emacs-overlay diff --git a/third_party/overlays/haskell/default.nix b/third_party/overlays/haskell/default.nix index eaf3d2e85c..dc1201ec43 100644 --- a/third_party/overlays/haskell/default.nix +++ b/third_party/overlays/haskell/default.nix @@ -12,15 +12,38 @@ in { haskellPackages = super.haskellPackages.override { overrides = hsSelf: hsSuper: { + punycode = haskellLib.appendPatch + (self.fetchpatch { + name = "punycode-mtl-2.3.patch"; + url = "https://github.com/litherum/punycode/pull/5/commits/41e55c8b7cef14563e6d04a7190dbabff5a77886.patch"; + sha256 = "03kgmy4z36jv16ffp5jrig2gr8ydc8cl1iscc7difisaq88mxvqc"; + }) + hsSuper.punycode; + + # Build with deprecated ansi-wl-pprint is broken now, use HEAD which switched to + # prettyprinter + tmp-postgres = haskellLib.overrideSrc + { + version = "unstable-2023-08-08"; + src = self.fetchFromGitHub { + owner = "jfischoff"; + repo = "tmp-postgres"; + rev = "7f2467a6d6d5f6db7eed59919a6773fe006cf22b"; + sha256 = "0l1gdx5s8ximgawd3yzfy47pv5pgwqmjqp8hx5rbrq68vr04wkbl"; + }; + } + (hsSuper.tmp-postgres.override { + ansi-wl-pprint = hsSelf.prettyprinter; + }); ihp-hsx = lib.pipe hsSuper.ihp-hsx [ (haskellLib.overrideSrc { - version = "1.1.0"; + version = "unstable-2023-03-28"; src = "${self.fetchFromGitHub { owner = "digitallyinduced"; repo = "ihp"; - rev = "b5d47963c998ccd779aa5c3d46484338fd621f0d"; - sha256 = "sha256-M22W8VX4sRaeU2yVraR0S2t2VOwWGmoteD/M8TahdoE="; + rev = "ab4ecd05f4e7b6b3c4b74b82d39fc6c5cc48766b"; + sha256 = "1fj5q9lygnmvqqv2fwqdj12sv63gkdfv5ha6fi190sv07dp9n9an"; }}/ihp-hsx"; }) haskellLib.doJailbreak diff --git a/third_party/overlays/patches/cbtemulator-uds.patch b/third_party/overlays/patches/cbtemulator-uds.patch new file mode 100644 index 0000000000..a19255306f --- /dev/null +++ b/third_party/overlays/patches/cbtemulator-uds.patch @@ -0,0 +1,140 @@ +commit 1397e10225d8c6fd079a86fccd58fb5d0f4200bc +Author: Florian Klink <flokli@flokli.de> +Date: Fri Mar 29 10:06:34 2024 +0100 + + feat(bigtable/emulator): allow listening on Unix Domain Sockets + + cbtemulator listening on unix domain sockets is much easier than trying + to allocate free TCP ports, especially if many cbtemulators are run at + the same time in integration tests. + + This adds an additional flag, address, which has priority if it's set, + rather than host:port. + + `NewServer` already takes a `laddr string`, so we simply check for it to + contain slashes, and if so, listen on unix, rather than TCP. + +diff --git a/bigtable/bttest/inmem.go b/bigtable/bttest/inmem.go +index 556abc2a85..33e4bf2667 100644 +--- a/bttest/inmem.go ++++ b/bttest/inmem.go +@@ -40,6 +40,7 @@ import ( + "math" + "math/rand" + "net" ++ "os" + "regexp" + "sort" + "strings" +@@ -106,7 +107,15 @@ type server struct { + // The Server will be listening for gRPC connections, without TLS, + // on the provided address. The resolved address is named by the Addr field. + func NewServer(laddr string, opt ...grpc.ServerOption) (*Server, error) { +- l, err := net.Listen("tcp", laddr) ++ var l net.Listener ++ var err error ++ ++ // If the address contains slashes, listen on a unix domain socket instead. ++ if strings.Contains(laddr, "/") { ++ l, err = net.Listen("unix", laddr) ++ } else { ++ l, err = net.Listen("tcp", laddr) ++ } + if err != nil { + return nil, err + } +diff --git a/bigtable/cmd/emulator/cbtemulator.go b/bigtable/cmd/emulator/cbtemulator.go +index 144c09ffb1..deaf69b717 100644 +--- a/cmd/emulator/cbtemulator.go ++++ b/cmd/emulator/cbtemulator.go +@@ -27,8 +27,9 @@ import ( + ) + + var ( +- host = flag.String("host", "localhost", "the address to bind to on the local machine") +- port = flag.Int("port", 9000, "the port number to bind to on the local machine") ++ host = flag.String("host", "localhost", "the address to bind to on the local machine") ++ port = flag.Int("port", 9000, "the port number to bind to on the local machine") ++ address = flag.String("address", "", "address:port number or unix socket path to listen on. Has priority over host/port") + ) + + const ( +@@ -42,7 +43,15 @@ func main() { + grpc.MaxRecvMsgSize(maxMsgSize), + grpc.MaxSendMsgSize(maxMsgSize), + } +- srv, err := bttest.NewServer(fmt.Sprintf("%s:%d", *host, *port), opts...) ++ ++ var laddr string ++ if *address != "" { ++ laddr = *address ++ } else { ++ laddr = fmt.Sprintf("%s:%d", *host, *port) ++ } ++ ++ srv, err := bttest.NewServer(laddr, opts...) + if err != nil { + log.Fatalf("failed to start emulator: %v", err) + } +commit ce16f843d6c93159d86b3807c6d9ff66e43aac67 +Author: Florian Klink <flokli@flokli.de> +Date: Fri Mar 29 11:53:15 2024 +0100 + + feat(bigtable): clean up unix socket on close + + Call srv.Close when receiving an interrupt, and delete the unix domain + socket in that function. + +diff --git a/bigtable/bttest/inmem.go b/bigtable/bttest/inmem.go +index 33e4bf2667..0dc96024b1 100644 +--- a/bttest/inmem.go ++++ b/bttest/inmem.go +@@ -148,6 +148,11 @@ func (s *Server) Close() { + + s.srv.Stop() + s.l.Close() ++ ++ // clean up unix socket ++ if strings.Contains(s.Addr, "/") { ++ _ = os.Remove(s.Addr) ++ } + } + + func (s *server) CreateTable(ctx context.Context, req *btapb.CreateTableRequest) (*btapb.Table, error) { +diff --git a/bigtable/cmd/emulator/cbtemulator.go b/bigtable/cmd/emulator/cbtemulator.go +index deaf69b717..5a9e8f7a8c 100644 +--- a/cmd/emulator/cbtemulator.go ++++ b/cmd/emulator/cbtemulator.go +@@ -18,9 +18,12 @@ cbtemulator launches the in-memory Cloud Bigtable server on the given address. + package main + + import ( ++ "context" + "flag" + "fmt" + "log" ++ "os" ++ "os/signal" + + "cloud.google.com/go/bigtable/bttest" + "google.golang.org/grpc" +@@ -51,11 +54,18 @@ func main() { + laddr = fmt.Sprintf("%s:%d", *host, *port) + } + ++ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) ++ defer stop() ++ + srv, err := bttest.NewServer(laddr, opts...) + if err != nil { + log.Fatalf("failed to start emulator: %v", err) + } + + fmt.Printf("Cloud Bigtable emulator running on %s\n", srv.Addr) +- select {} ++ select { ++ case <-ctx.Done(): ++ srv.Close() ++ stop() ++ } + } diff --git a/third_party/overlays/tvl.nix b/third_party/overlays/tvl.nix index 83b1fcf63e..b54e899b88 100644 --- a/third_party/overlays/tvl.nix +++ b/third_party/overlays/tvl.nix @@ -8,7 +8,14 @@ depot.nix.readTree.drvTargets { # flaky tests, long painful build, see https://github.com/NixOS/nixpkgs/pull/266443 withAWS = false; }); - nix = self.nix_2_3; + nix = self.nix_2_3 // { + # avoid duplicate pipeline step + meta = self.nix_2_3.meta or { } // { + ci = self.nix_2_3.meta.ci or { } // { + skip = true; + }; + }; + }; nix_latest = super.nix.override ({ # flaky tests, long painful build, see https://github.com/NixOS/nixpkgs/pull/266443 withAWS = false; @@ -83,44 +90,19 @@ depot.nix.readTree.drvTargets { }; })); - crate2nix = super.rustPlatform.buildRustPackage rec { - pname = "crate2nix"; - version = "0.13.0"; - - src = super.fetchFromGitHub { - owner = "nix-community"; - repo = "crate2nix"; - rev = "ceb06eb7e76afb9e01a5f069aae136f97df72730"; - hash = "sha256-JTMe8GViCQt51WUiaaoIPmWtwEeeYrl6pBxo2DNuKig="; - }; + # https://github.com/googleapis/google-cloud-go/pull/9665 + cbtemulator = super.cbtemulator.overrideAttrs (old: { + patches = old.patches or [ ] ++ [ + ./patches/cbtemulator-uds.patch + ]; + }); - patches = [ + crate2nix = super.crate2nix.overrideAttrs (old: { + patches = old.patches or [ ] ++ [ + # https://github.com/nix-community/crate2nix/pull/301 ./patches/crate2nix-tests-debug.patch - ./patches/crate2nix-run-tests-in-build-source.patch ]; - - sourceRoot = "${src.name}/crate2nix"; - - cargoHash = "sha256-dhlSXY1CJE+JJt+6Y7W1MVMz36nwr6ny543py1TcjyY="; - - nativeBuildInputs = [ super.makeWrapper ]; - - # Tests use nix(1), which tries (and fails) to set up /nix/var inside the - # sandbox - doCheck = false; - - postFixup = '' - wrapProgram $out/bin/crate2nix \ - --suffix PATH ":" ${lib.makeBinPath (with self; [ cargo nix_latest nix-prefetch-git ])} - - rm -rf $out/lib $out/bin/crate2nix.d - mkdir -p \ - $out/share/bash-completion/completions \ - $out/share/zsh/vendor-completions - $out/bin/crate2nix completions -s 'bash' -o $out/share/bash-completion/completions - $out/bin/crate2nix completions -s 'zsh' -o $out/share/zsh/vendor-completions - ''; - }; + }); evans = super.evans.overrideAttrs (old: { patches = old.patches or [ ] ++ [ diff --git a/third_party/rust-crates/default.nix b/third_party/rust-crates/default.nix index 4a98d2f548..697e47cdde 100644 --- a/third_party/rust-crates/default.nix +++ b/third_party/rust-crates/default.nix @@ -304,8 +304,8 @@ depot.nix.readTree.drvTargets rec{ libgit2-sys = buildRustCrate { pname = "libgit2-sys"; - version = "0.16.1+1.7.1"; - sha256 = "05ci61iw5nqhilxmmdpdc5ra8zpawablh2ap1g0lbgzvzmrdncb0"; + version = "0.16.2+1.7.2"; + sha256 = "0bs446idbmg8s13jvb0ck6qmrskcdn2mp3d4mn9ggxbmiw4ryd3g"; dependencies = [ libc libz-sys diff --git a/third_party/sources/sources.json b/third_party/sources/sources.json index 248fa4df73..109451ff51 100644 --- a/third_party/sources/sources.json +++ b/third_party/sources/sources.json @@ -11,28 +11,16 @@ "url": "https://github.com/ryantm/agenix/archive/0.15.0.tar.gz", "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz" }, - "emacs-overlay": { - "branch": "master", - "description": "Bleeding edge emacs overlay [maintainer=@adisbladis] ", - "homepage": "", - "owner": "nix-community", - "repo": "emacs-overlay", - "rev": "15f5f7d5eb82b1d2e7a4f754b11480b91aea9e91", - "sha256": "14fzs2b0h2lsbygrla6bx0a1r8jf2q4y1rl226ia7kcl5sakgzi9", - "type": "tarball", - "url": "https://github.com/nix-community/emacs-overlay/archive/15f5f7d5eb82b1d2e7a4f754b11480b91aea9e91.tar.gz", - "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz" - }, "home-manager": { "branch": "master", "description": "Manage a user environment using Nix [maintainer=@rycee] ", "homepage": "https://nix-community.github.io/home-manager/", "owner": "nix-community", "repo": "home-manager", - "rev": "206f457fffdb9a73596a4cb2211a471bd305243d", - "sha256": "0imc472hq7mmqhf5h8rhgfplw6575rl15fni0dc4wpypv9jccia9", + "rev": "c1609d584a6b5e9e6a02010f51bd368cb4782f8e", + "sha256": "112r86p3iah1xahwlp82yd3gvh10wkf271za5h7v3jsqv08c6gkr", "type": "tarball", - "url": "https://github.com/nix-community/home-manager/archive/206f457fffdb9a73596a4cb2211a471bd305243d.tar.gz", + "url": "https://github.com/nix-community/home-manager/archive/c1609d584a6b5e9e6a02010f51bd368cb4782f8e.tar.gz", "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz" }, "impermanence": { @@ -53,10 +41,10 @@ "homepage": "", "owner": "nmattia", "repo": "naersk", - "rev": "aeb58d5e8faead8980a807c840232697982d47b9", - "sha256": "185wg4p67krrjd8dx5h9pc381z7677nfzsdyp54kg3niqcf5wdzx", + "rev": "c5037590290c6c7dae2e42e7da1e247e54ed2d49", + "sha256": "1ql5ziwfrpmc8cxhgflmdy2z06z4dsdfzjwb2vv9bag6a2chrvq8", "type": "tarball", - "url": "https://github.com/nmattia/naersk/archive/aeb58d5e8faead8980a807c840232697982d47b9.tar.gz", + "url": "https://github.com/nmattia/naersk/archive/c5037590290c6c7dae2e42e7da1e247e54ed2d49.tar.gz", "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz" }, "napalm": { @@ -77,10 +65,10 @@ "homepage": "", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d691274a972b3165335d261cc4671335f5c67de9", - "sha256": "05lkkw0a63v1zan5g31blhckw6zf66p4gfqkpp01mp7fqwykmzx4", + "rev": "7bb2ccd8cdc44c91edba16c48d2c8f331fb3d856", + "sha256": "0ijqx995jw9i16f28whyjdll9b0nydmyl4n91bci2cgryxms7f8f", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/d691274a972b3165335d261cc4671335f5c67de9.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/7bb2ccd8cdc44c91edba16c48d2c8f331fb3d856.tar.gz", "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz" }, "nixpkgs-stable": { @@ -89,10 +77,10 @@ "homepage": "", "owner": "NixOS", "repo": "nixpkgs", - "rev": "8ac30a39abc5ea67037dfbf090d6e89f187c6e50", - "sha256": "0g8iy5qgb6qp2nq9p0yqhh1w5yp60v90h8zlkgxqhx4dj2fw9vy6", + "rev": "dd37924974b9202f8226ed5d74a252a9785aedf8", + "sha256": "1nxd4dqci8rs94a7cypx30axgj778p2wydkx16q298n29crkflbw", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/8ac30a39abc5ea67037dfbf090d6e89f187c6e50.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/dd37924974b9202f8226ed5d74a252a9785aedf8.tar.gz", "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz" }, "rust-overlay": { @@ -101,10 +89,10 @@ "homepage": "", "owner": "oxalica", "repo": "rust-overlay", - "rev": "50db54295d3922a3b7a40d580b84d75150b36c34", - "sha256": "0v8indbzb97dk9qk4srrxq8z5ds614kazi72zcxjhjzdxc8mklj7", + "rev": "2a42c742ab04b61d9b2f1edf392842cf9f27ebfd", + "sha256": "1wpkca75ysb2ssycc0dshd1m76q8iqhzrrbr6xmfmkkcj1p333nk", "type": "tarball", - "url": "https://github.com/oxalica/rust-overlay/archive/50db54295d3922a3b7a40d580b84d75150b36c34.tar.gz", + "url": "https://github.com/oxalica/rust-overlay/archive/2a42c742ab04b61d9b2f1edf392842cf9f27ebfd.tar.gz", "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz" }, "rustsec-advisory-db": { @@ -113,10 +101,10 @@ "homepage": "https://rustsec.org", "owner": "RustSec", "repo": "advisory-db", - "rev": "369d98c1b95b7b56d0859605916d7b81a7d1f1c4", - "sha256": "1npq87rjn606fhrpaxkphn8spdr1xafpy79nvrc3dnlvq163sr5n", + "rev": "35e7459a331d3e0c585e56dabd03006b9b354088", + "sha256": "1j8c0vzwg6b9lxmdy2a40pvwsy2kncv455spbjbxsj10p2vmy5fl", "type": "tarball", - "url": "https://github.com/RustSec/advisory-db/archive/369d98c1b95b7b56d0859605916d7b81a7d1f1c4.tar.gz", + "url": "https://github.com/RustSec/advisory-db/archive/35e7459a331d3e0c585e56dabd03006b9b354088.tar.gz", "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz" } } diff --git a/tools/depot-deps.nix b/tools/depot-deps.nix index 480b8c2f7c..c2f1cd302c 100644 --- a/tools/depot-deps.nix +++ b/tools/depot-deps.nix @@ -11,6 +11,7 @@ depot.nix.lazy-deps { gerrit-update.attr = "tools.gerrit-update"; gerrit.attr = "tools.gerrit-cli"; hash-password.attr = "tools.hash-password"; + josh-filter.attr = "third_party.nixpkgs.josh"; mg.attr = "tools.magrathea"; nint.attr = "nix.nint"; niv.attr = "third_party.nixpkgs.niv"; diff --git a/tools/releases/default.nix b/tools/releases/default.nix index d38ca0b5d8..0df07bbc9c 100644 --- a/tools/releases/default.nix +++ b/tools/releases/default.nix @@ -17,7 +17,7 @@ in command = pkgs.writeShellScript "${sanitizeDerivationName filter}-push" '' set -e - export PATH="${makeBinPath [ pkgs.git depot.third_party.josh ]}:$PATH" + export PATH="${makeBinPath [ pkgs.git pkgs.josh ]}:$PATH" echo 'Filtering depot through ${filter}' josh-filter '${filter}' diff --git a/tools/when/default.nix b/tools/when/default.nix new file mode 100644 index 0000000000..1aee5e1ea8 --- /dev/null +++ b/tools/when/default.nix @@ -0,0 +1,6 @@ +{ depot, ... }: + +depot.nix.buildGo.program { + name = "when"; + srcs = [ ./when.go ]; +} diff --git a/tools/when/when.go b/tools/when/when.go new file mode 100644 index 0000000000..a2ac494e8c --- /dev/null +++ b/tools/when/when.go @@ -0,0 +1,206 @@ +package main + +import ( + "fmt" + "os" + "strconv" + "strings" + "time" +) + +const usage = `usage: when <time> + +This program converts the given time into various formats (currently a local +timestamp, UTC timestamp, and UNIX epoch). It tries to guess what the input is. + +Some valid queries: + + 2024-01-05 + 1715079241 + tomorrow 5PM + -22h + -7h10m + Mar 15 + Sep 3 18:00 + +For now a single timestamp and a single duration (which is added either to the +current time, or the given time) is supported.` + +func printTime(t time.Time) { + fmt.Println("Local:", t.Format("Mon 02 January 2006 at 15:04:05 MST")) + fmt.Println("UTC: ", t.UTC().Format(time.RFC3339)) + fmt.Println("UNIX: ", t.Unix()) +} + +type FieldSet uint8 + +const ( + SetYear FieldSet = 1 << iota + SetDay + SetMonth + SetHour + SetMinute + SetSecond + SetLocation +) + +const ( + SetDate = SetYear | SetDay | SetMonth + SetClock = SetHour | SetMinute | SetSecond +) + +// mergeTimes returns a new time.Time with all fields in this overridden with the +// specified fields from that. +func mergeTimes(this time.Time, that time.Time, set FieldSet) time.Time { + year, month, day := this.Date() + hour, min, sec := this.Clock() + loc := this.Location() + + if set&SetYear == SetYear { + year = that.Year() + } + if set&SetMonth == SetMonth { + month = that.Month() + } + if set&SetDay == SetDay { + day = that.Day() + } + if set&SetHour == SetHour { + hour = that.Hour() + } + if set&SetMinute == SetMinute { + min = that.Minute() + } + if set&SetSecond == SetSecond { + sec = that.Second() + } + if set&SetLocation == SetLocation { + loc = that.Location() + } + + return time.Date(year, month, day, hour, min, sec, 0, loc) +} + +func parseTime(input string) (time.Time, error) { + // try unix times + if i, err := strconv.ParseInt(input, 10, 64); err == nil { + if i < 9999999999 { + return time.Unix(i, 0), nil + } + if i < 9999999999999 { + return time.UnixMilli(i), nil + } + } + + // try simple date/time formats + if t, err := time.Parse(time.DateOnly, input); err == nil { + return t, nil + } + + if t, err := time.Parse(time.Kitchen, input); err == nil { + now := time.Now() + return mergeTimes(now, t, SetClock), nil + } + + if t, err := time.Parse(time.TimeOnly, input); err == nil { + now := time.Now() + return mergeTimes(now, t, SetClock), nil + } + + if t, err := time.Parse("15:04", input); err == nil { + now := time.Now() + return mergeTimes(now, t, SetClock), nil + } + + if t, err := time.Parse("3PM", input); err == nil { + now := time.Now() + return mergeTimes(now, t, SetClock), nil + } + + if t, err := time.Parse(time.DateTime, input); err == nil { + return t, nil + } + + if t, err := time.Parse(time.Stamp, input); err == nil { + now := time.Now() + return mergeTimes(t, now, SetYear|SetLocation), nil + } + + if t, err := time.Parse("Jan _2 15:04", input); err == nil { + now := time.Now() + return mergeTimes(t, now, SetYear|SetLocation), nil + } + + if t, err := time.Parse("Jan _2", input); err == nil { + now := time.Now() + return mergeTimes(t, now, SetYear|SetLocation), nil + } + + return time.Time{}, fmt.Errorf("could not parse time: %q", input) +} + +func parseDuration(input string) (time.Duration, error) { + // some simple rewriting + switch input { + case "yesterday": + input = "-24h" + case "tomorrow": + input = "24h" + case "today", "now": + return time.Duration(0), nil + } + + // TODO: days, months, weeks, ... + return time.ParseDuration(input) +} + +func main() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, usage) + os.Exit(1) + } + + var d time.Duration + var t time.Time + var err error + var haveTime, haveDuration bool + + // Try to parse entire input as one full thing, before getting more + // clever. + if t, err = parseTime(strings.Join(os.Args[1:], " ")); err == nil { + printTime(t) + return + } + + for _, arg := range os.Args[1:] { + if !haveTime { + if t, err = parseTime(arg); err == nil { + haveTime = true + continue + } + } + + if !haveDuration { + if d, err = parseDuration(arg); err == nil { + haveDuration = true + continue + } + } + } + + if err != nil { + fmt.Fprintln(os.Stderr, "Not sure what you want, try another time.") + os.Exit(1) + } + + if haveTime && haveDuration { + printTime(t.Add(d)) + } else if haveTime { + printTime(t) + } else if haveDuration { + printTime(time.Now().Add(d)) + } else { + fmt.Fprintln(os.Stderr, "Not sure what you want, try another time.") + os.Exit(1) + } +} diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock index 4e95d39b6a..8385528e8f 100644 --- a/tvix/Cargo.lock +++ b/tvix/Cargo.lock @@ -120,10 +120,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] +name = "async-channel" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28243a43d821d11341ab73c80bed182dc015c514b951616cf79bd4af39af0c3" +dependencies = [ + "concurrent-queue", + "event-listener 5.2.0", + "event-listener-strategy 0.5.0", + "futures-core", + "pin-project-lite", +] + +[[package]] name = "async-compression" -version = "0.4.6" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" +checksum = "4e9eabd7a98fe442131a17c316bd9349c43695e49e730c3c8e12cfb5f4da2693" dependencies = [ "bzip2", "flate2", @@ -132,17 +145,83 @@ dependencies = [ "pin-project-lite", "tokio", "xz2", + "zstd", + "zstd-safe", ] [[package]] -name = "async-recursion" -version = "1.0.5" +name = "async-io" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" +checksum = "dcccb0f599cfa2f8ace422d3555572f47424da5648a4382a9dd0310ff8210884" dependencies = [ - "proc-macro2 1.0.76", - "quote 1.0.35", - "syn 2.0.48", + "async-lock 3.3.0", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "tracing", + "windows-sys 0.52.0", +] + +[[package]] +name = "async-lock" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" +dependencies = [ + "event-listener 2.5.3", +] + +[[package]] +name = "async-lock" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" +dependencies = [ + "event-listener 4.0.3", + "event-listener-strategy 0.4.0", + "pin-project-lite", +] + +[[package]] +name = "async-process" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451e3cf68011bd56771c79db04a9e333095ab6349f7e47592b788e9b98720cc8" +dependencies = [ + "async-channel", + "async-io", + "async-lock 3.3.0", + "async-signal", + "blocking", + "cfg-if", + "event-listener 5.2.0", + "futures-lite", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "async-signal" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e47d90f65a225c4527103a8d747001fc56e375203592b25ad103e1ca13124c5" +dependencies = [ + "async-io", + "async-lock 2.8.0", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix", + "signal-hook-registry", + "slab", + "windows-sys 0.48.0", ] [[package]] @@ -162,12 +241,18 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "syn 2.0.48", ] [[package]] +name = "async-task" +version = "4.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" + +[[package]] name = "async-tempfile" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -183,12 +268,18 @@ version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "syn 2.0.48", ] [[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -322,6 +413,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] +name = "bigtable_rs" +version = "0.2.9" +source = "git+https://github.com/flokli/bigtable_rs?rev=0af404741dfc40eb9fa99cf4d4140a09c5c20df7#0af404741dfc40eb9fa99cf4d4140a09c5c20df7" +dependencies = [ + "gcp_auth", + "http 0.2.11", + "log", + "prost 0.12.3", + "prost-build", + "prost-types", + "prost-wkt", + "prost-wkt-build", + "prost-wkt-types", + "serde", + "serde_with", + "thiserror", + "tokio", + "tonic 0.11.0", + "tonic-build", + "tower", +] + +[[package]] name = "bit-set" version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -379,6 +493,22 @@ dependencies = [ ] [[package]] +name = "blocking" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" +dependencies = [ + "async-channel", + "async-lock 3.3.0", + "async-task", + "fastrand", + "futures-io", + "futures-lite", + "piper", + "tracing", +] + +[[package]] name = "bstr" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -468,8 +598,10 @@ checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" dependencies = [ "android-tzdata", "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-targets 0.52.0", ] @@ -529,8 +661,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "syn 2.0.48", ] @@ -574,6 +706,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] +name = "concurrent-queue" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" +dependencies = [ + "crossbeam-utils", +] + +[[package]] name = "const-oid" version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -734,8 +875,43 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "darling" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.48", +] + +[[package]] +name = "darling_macro" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" +dependencies = [ + "darling_core", + "quote", "syn 2.0.48", ] @@ -756,6 +932,16 @@ dependencies = [ ] [[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", + "serde", +] + +[[package]] name = "diff" version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -880,7 +1066,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba7795da175654fe16979af73f81f26a8ea27638d8d9823d317016888a63dc4c" dependencies = [ "num-traits", - "quote 1.0.35", + "quote", "syn 2.0.48", ] @@ -891,6 +1077,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] +name = "erased-serde" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b73807008a3c7f171cc40312f37d95ef0396e048b5848d775f54b1a4dd4a0d3" +dependencies = [ + "serde", +] + +[[package]] name = "errno" version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -911,6 +1106,54 @@ dependencies = [ ] [[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "event-listener" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b5fb89194fa3cad959b833185b3063ba881dbfc7030680b314250779fb4cc91" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" +dependencies = [ + "event-listener 4.0.3", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feedafcaa9b749175d5ac357452a9d41ea2911da598fde46ce1fe02c37751291" +dependencies = [ + "event-listener 5.2.0", + "pin-project-lite", +] + +[[package]] name = "fastcdc" version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1067,13 +1310,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] +name = "futures-lite" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] name = "futures-macro" version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "syn 2.0.48", ] @@ -1123,6 +1379,31 @@ dependencies = [ ] [[package]] +name = "gcp_auth" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de2c71ea685b88a1aa50e9fb66fe0e1cb29d755f58cca41fb8c91ef604d4f4d4" +dependencies = [ + "async-trait", + "base64", + "chrono", + "home", + "hyper 0.14.28", + "hyper-rustls", + "ring", + "rustls 0.21.10", + "rustls-pemfile 1.0.4", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", + "tracing-futures", + "url", + "which 5.0.0", +] + +[[package]] name = "genawaiter" version = "0.99.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1239,6 +1520,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" [[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] name = "hex-literal" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1381,6 +1668,7 @@ dependencies = [ "http 0.2.11", "hyper 0.14.28", "rustls 0.21.10", + "rustls-native-certs 0.6.3", "tokio", "tokio-rustls 0.24.1", ] @@ -1437,6 +1725,12 @@ dependencies = [ ] [[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] name = "idna" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1478,6 +1772,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", + "serde", ] [[package]] @@ -1488,6 +1783,7 @@ checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", "hashbrown 0.14.3", + "serde", ] [[package]] @@ -1500,6 +1796,12 @@ dependencies = [ ] [[package]] +name = "inventory" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f958d3d68f4167080a18141e10381e7634563984a537f2a49a30fd8e53ac5767" + +[[package]] name = "ipnet" version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1876,18 +2178,17 @@ dependencies = [ "ed25519-dalek", "enum-primitive-derive", "futures", - "futures-util", "glob", "hex-literal", "lazy_static", "nom", "num-traits", + "pin-project-lite", "pretty_assertions", + "rstest", "serde", "serde_json", "sha2", - "test-case", - "test-generator", "thiserror", "tokio", "tokio-test", @@ -1924,6 +2225,12 @@ dependencies = [ ] [[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] name = "num-traits" version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2104,6 +2411,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] +name = "parking" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" + +[[package]] name = "parking_lot" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2188,8 +2501,8 @@ version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "syn 2.0.48", ] @@ -2206,6 +2519,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] +name = "piper" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + +[[package]] name = "pkcs8" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2256,6 +2580,26 @@ dependencies = [ ] [[package]] +name = "polling" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30054e72317ab98eddd8561db0f6524df3367636884b7b21b703e4b280a84a14" +dependencies = [ + "cfg-if", + "concurrent-queue", + "pin-project-lite", + "rustix", + "tracing", + "windows-sys 0.52.0", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] name = "ppv-lite86" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2277,21 +2621,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ - "proc-macro2 1.0.76", + "proc-macro2", "syn 2.0.48", ] [[package]] name = "proc-macro2" -version = "0.4.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" -dependencies = [ - "unicode-xid", -] - -[[package]] -name = "proc-macro2" version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" @@ -2355,10 +2690,12 @@ dependencies = [ "prettyplease", "prost 0.12.3", "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", "regex", "syn 2.0.48", "tempfile", - "which", + "which 4.4.2", ] [[package]] @@ -2369,8 +2706,8 @@ checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", "itertools 0.10.5", - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -2382,8 +2719,8 @@ checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" dependencies = [ "anyhow", "itertools 0.11.0", - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "syn 2.0.48", ] @@ -2397,6 +2734,72 @@ dependencies = [ ] [[package]] +name = "prost-wkt" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d8ef9c3f0f1dab910d2b7e2c24a8e4322e122eba6d7a1921eeebcebbc046c40" +dependencies = [ + "chrono", + "inventory", + "prost 0.12.3", + "serde", + "serde_derive", + "serde_json", + "typetag", +] + +[[package]] +name = "prost-wkt-build" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b31cae9a54ca84fee1504740a82eebf2479532905e106f63ca0c3bc8d780321" +dependencies = [ + "heck", + "prost 0.12.3", + "prost-build", + "prost-types", + "quote", +] + +[[package]] +name = "prost-wkt-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435be4a8704091b4c5fb1d79799de7f2dbff53af05edf29385237f8cf7ab37ee" +dependencies = [ + "chrono", + "prost 0.12.3", + "prost-build", + "prost-types", + "prost-wkt", + "prost-wkt-build", + "regex", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "pulldown-cmark" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" +dependencies = [ + "bitflags 2.4.2", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "10.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0194e6e1966c23cc5fd988714f85b18d548d773e81965413555d96569931833d" +dependencies = [ + "pulldown-cmark", +] + +[[package]] name = "quick-error" version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2414,20 +2817,11 @@ dependencies = [ [[package]] name = "quote" -version = "0.6.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" -dependencies = [ - "proc-macro2 0.4.30", -] - -[[package]] -name = "quote" version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ - "proc-macro2 1.0.76", + "proc-macro2", ] [[package]] @@ -2676,9 +3070,9 @@ dependencies = [ [[package]] name = "rstest" -version = "0.18.2" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97eeab2f3c0a199bc4be135c36c924b6590b88c377d416494288c14f2db30199" +checksum = "9d5316d2a1479eeef1ea21e7f9ddc67c191d497abc8fc3ba2467857abbb68330" dependencies = [ "futures", "futures-timer", @@ -2688,14 +3082,14 @@ dependencies = [ [[package]] name = "rstest_macros" -version = "0.18.2" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d428f8247852f894ee1be110b375111b586d4fa431f6c46e64ba5a0dcccbe605" +checksum = "04a9df72cc1f67020b0d63ad9bfe4a323e459ea7eb68e03bd9824db49f9a4c25" dependencies = [ "cfg-if", "glob", - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "regex", "relative-path", "rustc_version", @@ -2709,7 +3103,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88530b681abe67924d42cca181d070e3ac20e0740569441a9e35a7cedd2b34a4" dependencies = [ - "quote 1.0.35", + "quote", "rand", "rustc_version", "syn 2.0.48", @@ -2958,21 +3352,21 @@ checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" [[package]] name = "serde" -version = "1.0.195" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.195" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "syn 2.0.48", ] @@ -2998,6 +3392,17 @@ dependencies = [ ] [[package]] +name = "serde_qs" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0431a35568651e363364210c91983c1da5eb29404d9f0928b67d4ebcfa7d330c" +dependencies = [ + "percent-encoding", + "serde", + "thiserror", +] + +[[package]] name = "serde_spanned" version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3019,6 +3424,36 @@ dependencies = [ ] [[package]] +name = "serde_with" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee80b0e361bbf88fd2f6e242ccd19cfda072cb0faa6ae694ecee08199938569a" +dependencies = [ + "base64", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.1.0", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6561dc161a9224638a31d876ccdfefbc1df91d3f3a8342eddb35f055d48c7655" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] name = "sha1" version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3124,8 +3559,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" dependencies = [ "heck", - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -3179,8 +3614,8 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "104842d6278bf64aa9d2f182ba4bde31e8aec7a131d29b7f444bb9b344a09e2a" dependencies = [ - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "structmeta-derive", "syn 1.0.109", ] @@ -3191,8 +3626,8 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24420be405b590e2d746d83b01f09af673270cf80e9b003a5fa7b651c58c7d93" dependencies = [ - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "syn 1.0.109", ] @@ -3204,23 +3639,12 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "0.15.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" -dependencies = [ - "proc-macro2 0.4.30", - "quote 0.6.13", - "unicode-xid", -] - -[[package]] -name = "syn" version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "unicode-ident", ] @@ -3230,8 +3654,8 @@ version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "unicode-ident", ] @@ -3294,57 +3718,13 @@ dependencies = [ ] [[package]] -name = "test-case" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" -dependencies = [ - "test-case-macros", -] - -[[package]] -name = "test-case-core" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adcb7fd841cd518e279be3d5a3eb0636409487998a4aff22f3de87b81e88384f" -dependencies = [ - "cfg-if", - "proc-macro2 1.0.76", - "quote 1.0.35", - "syn 2.0.48", -] - -[[package]] -name = "test-case-macros" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" -dependencies = [ - "proc-macro2 1.0.76", - "quote 1.0.35", - "syn 2.0.48", - "test-case-core", -] - -[[package]] -name = "test-generator" -version = "0.3.0" -source = "git+https://github.com/JamesGuthrie/test-generator.git?rev=82e799979980962aec1aa324ec6e0e4cad781f41#82e799979980962aec1aa324ec6e0e4cad781f41" -dependencies = [ - "glob", - "proc-macro2 0.4.30", - "quote 0.6.13", - "syn 0.15.44", -] - -[[package]] name = "test-strategy" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62d6408d1406657be2f9d1701fbae379331d30d2f6e92050710edb0d34eeb480" dependencies = [ - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "structmeta", "syn 1.0.109", ] @@ -3370,8 +3750,8 @@ version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "syn 2.0.48", ] @@ -3386,6 +3766,37 @@ dependencies = [ ] [[package]] +name = "time" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] name = "tinytemplate" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3463,8 +3874,8 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "syn 2.0.48", ] @@ -3654,9 +4065,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4ef6dd70a610078cb4e338a0f79d06bc759ff1b22d2120c2ff02ae264ba9c2" dependencies = [ "prettyplease", - "proc-macro2 1.0.76", + "proc-macro2", "prost-build", - "quote 1.0.35", + "quote", "syn 2.0.48", ] @@ -3723,8 +4134,8 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "syn 2.0.48", ] @@ -3739,6 +4150,16 @@ dependencies = [ ] [[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] name = "tracing-log" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3813,7 +4234,7 @@ dependencies = [ "itertools 0.12.0", "prost 0.12.3", "prost-build", - "test-case", + "rstest", "thiserror", "tokio", "tokio-listener", @@ -3830,8 +4251,10 @@ dependencies = [ name = "tvix-castore" version = "0.1.0" dependencies = [ + "async-process", "async-stream", "async-tempfile", + "bigtable_rs", "blake3", "bstr", "bytes", @@ -3845,18 +4268,22 @@ dependencies = [ "libc", "object_store", "parking_lot 0.12.1", + "petgraph", "pin-project-lite", "prost 0.12.3", "prost-build", "rstest", "rstest_reuse", + "serde", + "serde_qs", + "serde_with", "sled", "tempfile", - "test-case", "thiserror", "tokio", "tokio-retry", "tokio-stream", + "tokio-tar", "tokio-util", "tonic 0.11.0", "tonic-build", @@ -3871,6 +4298,7 @@ dependencies = [ "vm-memory", "vmm-sys-util", "walkdir", + "xattr", "zstd", ] @@ -3883,7 +4311,6 @@ dependencies = [ "dirs", "nix-compat", "rustyline", - "test-case", "thiserror", "tokio", "tracing", @@ -3938,8 +4365,8 @@ dependencies = [ name = "tvix-eval-builtin-macros" version = "0.0.1" dependencies = [ - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "syn 1.0.109", "tvix-eval", ] @@ -3949,7 +4376,6 @@ name = "tvix-glue" version = "0.1.0" dependencies = [ "async-compression", - "async-recursion", "bstr", "bytes", "criterion", @@ -3958,6 +4384,7 @@ dependencies = [ "hex-literal", "lazy_static", "magic", + "md-5", "nix 0.27.1", "nix-compat", "pin-project", @@ -3966,9 +4393,9 @@ dependencies = [ "rstest", "serde", "serde_json", + "sha1", "sha2", "tempfile", - "test-case", "thiserror", "tokio", "tokio-tar", @@ -3978,6 +4405,7 @@ dependencies = [ "tvix-castore", "tvix-eval", "tvix-store", + "url", "walkdir", "wu-manber", ] @@ -3996,8 +4424,10 @@ name = "tvix-store" version = "0.1.0" dependencies = [ "anyhow", - "async-recursion", + "async-compression", + "async-process", "async-stream", + "bigtable_rs", "blake3", "bstr", "bytes", @@ -4016,10 +4446,13 @@ dependencies = [ "reqwest", "rstest", "rstest_reuse", + "serde", + "serde_json", + "serde_qs", + "serde_with", "sha2", "sled", "tempfile", - "test-case", "thiserror", "tokio", "tokio-listener", @@ -4036,7 +4469,6 @@ dependencies = [ "tvix-castore", "url", "walkdir", - "xz2", ] [[package]] @@ -4046,12 +4478,45 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] +name = "typetag" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "661d18414ec032a49ece2d56eee03636e43c4e8d577047ab334c0ba892e29aaf" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac73887f47b9312552aa90ef477927ff014d63d1920ca8037c6c1951eab64bb1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] name = "unarray" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] +name = "unicase" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" +dependencies = [ + "version_check", +] + +[[package]] name = "unicode-bidi" version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4085,12 +4550,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] -name = "unicode-xid" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" - -[[package]] name = "untrusted" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4271,8 +4730,8 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "syn 2.0.48", "wasm-bindgen-shared", ] @@ -4295,7 +4754,7 @@ version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" dependencies = [ - "quote 1.0.35", + "quote", "wasm-bindgen-macro-support", ] @@ -4305,8 +4764,8 @@ version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ - "proc-macro2 1.0.76", - "quote 1.0.35", + "proc-macro2", + "quote", "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -4364,6 +4823,19 @@ dependencies = [ ] [[package]] +name = "which" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bf3ea8596f3a0dd5980b46430f2058dfe2c36a27ccfbb1845d6fbfcd9ba6e14" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] name = "winapi" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix index e6920342d7..0efb86c66d 100644 --- a/tvix/Cargo.nix +++ b/tvix/Cargo.nix @@ -1,4 +1,4 @@ -# This file was @generated by crate2nix 0.13.0 with the command: +# This file was @generated by crate2nix 0.14.0 with the command: # "generate" "--all-features" # See https://github.com/kolloch/crate2nix for more info. @@ -412,11 +412,51 @@ rec { "zeroize" = [ "dep:zeroize" ]; }; }; + "async-channel" = rec { + crateName = "async-channel"; + version = "2.2.0"; + edition = "2021"; + sha256 = "1hzhkbrlmgbrrwb1d5aba5f03p42s6z80g5p38s127c27nj470pj"; + authors = [ + "Stjepan Glavina <stjepang@gmail.com>" + ]; + dependencies = [ + { + name = "concurrent-queue"; + packageId = "concurrent-queue"; + usesDefaultFeatures = false; + } + { + name = "event-listener"; + packageId = "event-listener 5.2.0"; + usesDefaultFeatures = false; + } + { + name = "event-listener-strategy"; + packageId = "event-listener-strategy 0.5.0"; + usesDefaultFeatures = false; + } + { + name = "futures-core"; + packageId = "futures-core"; + usesDefaultFeatures = false; + } + { + name = "pin-project-lite"; + packageId = "pin-project-lite"; + } + ]; + features = { + "default" = [ "std" ]; + "std" = [ "concurrent-queue/std" "event-listener/std" "event-listener-strategy/std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "async-compression" = rec { crateName = "async-compression"; - version = "0.4.6"; + version = "0.4.9"; edition = "2018"; - sha256 = "0b6874q56g1cx8ivs9j89d757rsh9kyrrwlp1852094jjrmg85m1"; + sha256 = "14r6vbsbbkqjiqy0qwwywjakdi29jfyidhqp389l5r4gm7bsp7jf"; authors = [ "Wim Looman <wim@nemo157.com>" "Allen Bui <fairingrey@gmail.com>" @@ -456,6 +496,27 @@ rec { packageId = "xz2"; optional = true; } + { + name = "zstd"; + packageId = "zstd"; + rename = "libzstd"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "zstd-safe"; + packageId = "zstd-safe"; + optional = true; + usesDefaultFeatures = false; + } + ]; + devDependencies = [ + { + name = "tokio"; + packageId = "tokio"; + usesDefaultFeatures = false; + features = [ "io-util" "macros" "rt-multi-thread" "io-std" ]; + } ]; features = { "all" = [ "all-implementations" "all-algorithms" ]; @@ -478,33 +539,252 @@ rec { "zstd-safe" = [ "dep:zstd-safe" ]; "zstdmt" = [ "zstd" "zstd-safe/zstdmt" ]; }; - resolvedDefaultFeatures = [ "bzip2" "flate2" "gzip" "tokio" "xz" "xz2" ]; + resolvedDefaultFeatures = [ "bzip2" "flate2" "gzip" "libzstd" "tokio" "xz" "xz2" "zstd" "zstd-safe" ]; + }; + "async-io" = rec { + crateName = "async-io"; + version = "2.3.2"; + edition = "2021"; + sha256 = "110847w0ycfhklm3i928avd28x7lf9amblr2wjngi8ngk7sv1k6w"; + authors = [ + "Stjepan Glavina <stjepang@gmail.com>" + ]; + dependencies = [ + { + name = "async-lock"; + packageId = "async-lock 3.3.0"; + } + { + name = "cfg-if"; + packageId = "cfg-if"; + } + { + name = "concurrent-queue"; + packageId = "concurrent-queue"; + } + { + name = "futures-io"; + packageId = "futures-io"; + usesDefaultFeatures = false; + features = [ "std" ]; + } + { + name = "futures-lite"; + packageId = "futures-lite"; + usesDefaultFeatures = false; + } + { + name = "parking"; + packageId = "parking"; + } + { + name = "polling"; + packageId = "polling"; + } + { + name = "rustix"; + packageId = "rustix"; + usesDefaultFeatures = false; + features = [ "fs" "net" "std" ]; + } + { + name = "slab"; + packageId = "slab"; + } + { + name = "tracing"; + packageId = "tracing"; + usesDefaultFeatures = false; + } + { + name = "windows-sys"; + packageId = "windows-sys 0.52.0"; + target = { target, features }: (target."windows" or false); + features = [ "Win32_Foundation" ]; + } + ]; + }; - "async-recursion" = rec { - crateName = "async-recursion"; - version = "1.0.5"; + "async-lock 2.8.0" = rec { + crateName = "async-lock"; + version = "2.8.0"; edition = "2018"; - sha256 = "1l2vlgyaa9a2dd0y1vbqyppzsvpdr1y4rar4gn1qi68pl5dmmmaz"; - procMacro = true; + sha256 = "0asq5xdzgp3d5m82y5rg7a0k9q0g95jy6mgc7ivl334x7qlp4wi8"; authors = [ - "Robert Usher <266585+dcchut@users.noreply.github.com>" + "Stjepan Glavina <stjepang@gmail.com>" ]; dependencies = [ { - name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + name = "event-listener"; + packageId = "event-listener 2.5.3"; + } + ]; + + }; + "async-lock 3.3.0" = rec { + crateName = "async-lock"; + version = "3.3.0"; + edition = "2021"; + sha256 = "0yxflkfw46rad4lv86f59b5z555dlfmg1riz1n8830rgi0qb8d6h"; + authors = [ + "Stjepan Glavina <stjepang@gmail.com>" + ]; + dependencies = [ + { + name = "event-listener"; + packageId = "event-listener 4.0.3"; usesDefaultFeatures = false; } { - name = "quote"; - packageId = "quote 1.0.35"; + name = "event-listener-strategy"; + packageId = "event-listener-strategy 0.4.0"; usesDefaultFeatures = false; } { - name = "syn"; - packageId = "syn 2.0.48"; + name = "pin-project-lite"; + packageId = "pin-project-lite"; + } + ]; + features = { + "default" = [ "std" ]; + "std" = [ "event-listener/std" "event-listener-strategy/std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; + "async-process" = rec { + crateName = "async-process"; + version = "2.1.0"; + edition = "2021"; + sha256 = "1j0cfac9p3kq5dclfzlz6jv5l29kwflh9nvr3ivmdg8ih3v3q7j5"; + authors = [ + "Stjepan Glavina <stjepang@gmail.com>" + ]; + dependencies = [ + { + name = "async-channel"; + packageId = "async-channel"; + target = { target, features }: (target."windows" or false); + } + { + name = "async-io"; + packageId = "async-io"; + target = { target, features }: (target."unix" or false); + } + { + name = "async-lock"; + packageId = "async-lock 3.3.0"; + } + { + name = "async-signal"; + packageId = "async-signal"; + target = { target, features }: (target."unix" or false); + } + { + name = "blocking"; + packageId = "blocking"; + target = { target, features }: (target."windows" or false); + } + { + name = "cfg-if"; + packageId = "cfg-if"; + } + { + name = "event-listener"; + packageId = "event-listener 5.2.0"; + } + { + name = "futures-lite"; + packageId = "futures-lite"; + } + { + name = "rustix"; + packageId = "rustix"; + usesDefaultFeatures = false; + target = { target, features }: (target."unix" or false); + features = [ "std" "fs" ]; + } + { + name = "windows-sys"; + packageId = "windows-sys 0.52.0"; usesDefaultFeatures = false; - features = [ "full" "parsing" "printing" "proc-macro" "clone-impls" ]; + target = { target, features }: (target."windows" or false); + features = [ "Win32_Foundation" "Win32_System_Threading" ]; + } + ]; + devDependencies = [ + { + name = "async-io"; + packageId = "async-io"; + } + ]; + + }; + "async-signal" = rec { + crateName = "async-signal"; + version = "0.2.5"; + edition = "2018"; + sha256 = "1i9466hiqghhmljjnn83a8vnxi8z013xga03f59c89d2cl7xjiwy"; + authors = [ + "John Nunley <dev@notgull.net>" + ]; + dependencies = [ + { + name = "async-io"; + packageId = "async-io"; + target = { target, features }: (target."unix" or false); + } + { + name = "async-lock"; + packageId = "async-lock 2.8.0"; + target = { target, features }: (target."windows" or false); + } + { + name = "atomic-waker"; + packageId = "atomic-waker"; + target = { target, features }: (target."windows" or false); + } + { + name = "cfg-if"; + packageId = "cfg-if"; + } + { + name = "futures-core"; + packageId = "futures-core"; + } + { + name = "futures-io"; + packageId = "futures-io"; + target = { target, features }: (target."unix" or false); + } + { + name = "rustix"; + packageId = "rustix"; + usesDefaultFeatures = false; + target = { target, features }: (target."unix" or false); + features = [ "process" "std" ]; + } + { + name = "signal-hook-registry"; + packageId = "signal-hook-registry"; + target = { target, features }: (target."unix" or false); + } + { + name = "slab"; + packageId = "slab"; + target = { target, features }: (target."windows" or false); + } + { + name = "windows-sys"; + packageId = "windows-sys 0.48.0"; + usesDefaultFeatures = false; + target = { target, features }: (target."windows" or false); + features = [ "Win32_Foundation" "Win32_System_Console" ]; + } + ]; + devDependencies = [ + { + name = "async-io"; + packageId = "async-io"; } ]; @@ -545,11 +825,11 @@ rec { dependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "syn"; @@ -559,6 +839,20 @@ rec { ]; }; + "async-task" = rec { + crateName = "async-task"; + version = "4.7.0"; + edition = "2018"; + sha256 = "16975vx6aqy5yf16fs9xz5vx1zq8mwkzfmykvcilc1j7b6c6xczv"; + authors = [ + "Stjepan Glavina <stjepang@gmail.com>" + ]; + features = { + "default" = [ "std" ]; + "portable-atomic" = [ "dep:portable-atomic" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "async-tempfile" = rec { crateName = "async-tempfile"; version = "0.4.0"; @@ -605,11 +899,11 @@ rec { dependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "syn"; @@ -619,6 +913,19 @@ rec { ]; }; + "atomic-waker" = rec { + crateName = "atomic-waker"; + version = "1.1.2"; + edition = "2018"; + sha256 = "1h5av1lw56m0jf0fd3bchxq8a30xv0b4wv8s4zkp4s0i7mfvs18m"; + authors = [ + "Stjepan Glavina <stjepang@gmail.com>" + "Contributors to futures-rs" + ]; + features = { + "portable-atomic" = [ "dep:portable-atomic" ]; + }; + }; "autocfg" = rec { crateName = "autocfg"; version = "1.1.0"; @@ -1162,6 +1469,94 @@ rec { }; resolvedDefaultFeatures = [ "alloc" ]; }; + "bigtable_rs" = rec { + crateName = "bigtable_rs"; + version = "0.2.9"; + edition = "2021"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/flokli/bigtable_rs"; + rev = "0af404741dfc40eb9fa99cf4d4140a09c5c20df7"; + sha256 = "1njjam1lx2xlnm7a41lga8601vmjgqz0fvc77x24gd04pc7avxll"; + }; + authors = [ + "Fuyang Liu <liufuyang@users.noreply.github.com>" + ]; + dependencies = [ + { + name = "gcp_auth"; + packageId = "gcp_auth"; + } + { + name = "http"; + packageId = "http 0.2.11"; + } + { + name = "log"; + packageId = "log"; + } + { + name = "prost"; + packageId = "prost 0.12.3"; + } + { + name = "prost-types"; + packageId = "prost-types"; + } + { + name = "prost-wkt"; + packageId = "prost-wkt"; + } + { + name = "prost-wkt-types"; + packageId = "prost-wkt-types"; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde_with"; + packageId = "serde_with"; + features = [ "base64" ]; + } + { + name = "thiserror"; + packageId = "thiserror"; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "rt-multi-thread" ]; + } + { + name = "tonic"; + packageId = "tonic 0.11.0"; + features = [ "tls" "transport" ]; + } + { + name = "tower"; + packageId = "tower"; + } + ]; + buildDependencies = [ + { + name = "prost-build"; + packageId = "prost-build"; + } + { + name = "prost-wkt-build"; + packageId = "prost-wkt-build"; + } + { + name = "tonic-build"; + packageId = "tonic-build"; + features = [ "cleanup-markdown" ]; + } + ]; + + }; "bit-set" = rec { crateName = "bit-set"; version = "0.5.3"; @@ -1317,6 +1712,61 @@ rec { ]; }; + "blocking" = rec { + crateName = "blocking"; + version = "1.5.1"; + edition = "2018"; + sha256 = "064i3d6b8ln34fgdw49nmx9m36bwi3r3nv8c9xhcrpf4ilz92dva"; + authors = [ + "Stjepan Glavina <stjepang@gmail.com>" + ]; + dependencies = [ + { + name = "async-channel"; + packageId = "async-channel"; + } + { + name = "async-lock"; + packageId = "async-lock 3.3.0"; + target = { target, features }: (!(builtins.elem "wasm" target."family")); + } + { + name = "async-task"; + packageId = "async-task"; + } + { + name = "fastrand"; + packageId = "fastrand"; + } + { + name = "futures-io"; + packageId = "futures-io"; + usesDefaultFeatures = false; + features = [ "std" ]; + } + { + name = "futures-lite"; + packageId = "futures-lite"; + usesDefaultFeatures = false; + } + { + name = "piper"; + packageId = "piper"; + } + { + name = "tracing"; + packageId = "tracing"; + usesDefaultFeatures = false; + } + ]; + devDependencies = [ + { + name = "futures-lite"; + packageId = "futures-lite"; + } + ]; + + }; "bstr" = rec { crateName = "bstr"; version = "1.9.0"; @@ -1544,6 +1994,12 @@ rec { features = [ "fallback" ]; } { + name = "js-sys"; + packageId = "js-sys"; + optional = true; + target = { target, features }: (("wasm32" == target."arch" or null) && (!(("emscripten" == target."os" or null) || ("wasi" == target."os" or null)))); + } + { name = "num-traits"; packageId = "num-traits"; usesDefaultFeatures = false; @@ -1555,6 +2011,12 @@ rec { usesDefaultFeatures = false; } { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + optional = true; + target = { target, features }: (("wasm32" == target."arch" or null) && (!(("emscripten" == target."os" or null) || ("wasi" == target."os" or null)))); + } + { name = "windows-targets"; packageId = "windows-targets 0.52.0"; optional = true; @@ -1584,7 +2046,7 @@ rec { "winapi" = [ "windows-targets" ]; "windows-targets" = [ "dep:windows-targets" ]; }; - resolvedDefaultFeatures = [ "alloc" "android-tzdata" "clock" "iana-time-zone" "now" "serde" "std" "winapi" "windows-targets" ]; + resolvedDefaultFeatures = [ "alloc" "android-tzdata" "clock" "default" "iana-time-zone" "js-sys" "now" "oldtime" "serde" "std" "wasm-bindgen" "wasmbind" "winapi" "windows-targets" ]; }; "ciborium" = rec { crateName = "ciborium"; @@ -1744,11 +2206,11 @@ rec { } { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "syn"; @@ -1838,6 +2300,30 @@ rec { sha256 = "1ix7w85kwvyybwi2jdkl3yva2r2bvdcc3ka2grjfzfgrapqimgxc"; }; + "concurrent-queue" = rec { + crateName = "concurrent-queue"; + version = "2.4.0"; + edition = "2018"; + sha256 = "0qvk23ynj311adb4z7v89wk3bs65blps4n24q8rgl23vjk6lhq6i"; + authors = [ + "Stjepan Glavina <stjepang@gmail.com>" + "Taiki Endo <te316e89@gmail.com>" + "John Nunley <jtnunley01@gmail.com>" + ]; + dependencies = [ + { + name = "crossbeam-utils"; + packageId = "crossbeam-utils"; + usesDefaultFeatures = false; + } + ]; + features = { + "default" = [ "std" ]; + "loom" = [ "dep:loom" ]; + "portable-atomic" = [ "dep:portable-atomic" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "const-oid" = rec { crateName = "const-oid"; version = "0.9.6"; @@ -2298,11 +2784,11 @@ rec { dependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "syn"; @@ -2312,6 +2798,98 @@ rec { ]; }; + "darling" = rec { + crateName = "darling"; + version = "0.20.8"; + edition = "2018"; + sha256 = "14a38qsi9104kvk1z11rqj0bnz1866dyhnvgvbgzz17d2g6nzqsl"; + authors = [ + "Ted Driggs <ted.driggs@outlook.com>" + ]; + dependencies = [ + { + name = "darling_core"; + packageId = "darling_core"; + } + { + name = "darling_macro"; + packageId = "darling_macro"; + } + ]; + features = { + "default" = [ "suggestions" ]; + "diagnostics" = [ "darling_core/diagnostics" ]; + "suggestions" = [ "darling_core/suggestions" ]; + }; + resolvedDefaultFeatures = [ "default" "suggestions" ]; + }; + "darling_core" = rec { + crateName = "darling_core"; + version = "0.20.8"; + edition = "2018"; + sha256 = "03x7s149p06xfwcq0lgkk4yxh6jf7jckny18nzp1yyk87b1g2b4w"; + authors = [ + "Ted Driggs <ted.driggs@outlook.com>" + ]; + dependencies = [ + { + name = "fnv"; + packageId = "fnv"; + } + { + name = "ident_case"; + packageId = "ident_case"; + } + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "strsim"; + packageId = "strsim"; + optional = true; + } + { + name = "syn"; + packageId = "syn 2.0.48"; + features = [ "full" "extra-traits" ]; + } + ]; + features = { + "strsim" = [ "dep:strsim" ]; + "suggestions" = [ "strsim" ]; + }; + resolvedDefaultFeatures = [ "strsim" "suggestions" ]; + }; + "darling_macro" = rec { + crateName = "darling_macro"; + version = "0.20.8"; + edition = "2018"; + sha256 = "0gwkz0cjfy3fgcc1zmm7azzhj5qpja34s0cklcria4l38sjyss56"; + procMacro = true; + authors = [ + "Ted Driggs <ted.driggs@outlook.com>" + ]; + dependencies = [ + { + name = "darling_core"; + packageId = "darling_core"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "syn"; + packageId = "syn 2.0.48"; + } + ]; + + }; "data-encoding" = rec { crateName = "data-encoding"; version = "2.5.0"; @@ -2361,6 +2939,39 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "oid" "std" "zeroize" ]; }; + "deranged" = rec { + crateName = "deranged"; + version = "0.3.11"; + edition = "2021"; + sha256 = "1d1ibqqnr5qdrpw8rclwrf1myn3wf0dygl04idf4j2s49ah6yaxl"; + authors = [ + "Jacob Pratt <jacob@jhpratt.dev>" + ]; + dependencies = [ + { + name = "powerfmt"; + packageId = "powerfmt"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "serde"; + packageId = "serde"; + optional = true; + usesDefaultFeatures = false; + } + ]; + features = { + "default" = [ "std" ]; + "num" = [ "dep:num-traits" ]; + "powerfmt" = [ "dep:powerfmt" ]; + "quickcheck" = [ "dep:quickcheck" "alloc" ]; + "rand" = [ "dep:rand" ]; + "serde" = [ "dep:serde" ]; + "std" = [ "alloc" ]; + }; + resolvedDefaultFeatures = [ "alloc" "powerfmt" "serde" "std" ]; + }; "diff" = rec { crateName = "diff"; version = "0.1.13"; @@ -2709,7 +3320,7 @@ rec { } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "syn"; @@ -2725,6 +3336,28 @@ rec { sha256 = "1malmx5f4lkfvqasz319lq6gb3ddg19yzf9s8cykfsgzdmyq0hsl"; }; + "erased-serde" = rec { + crateName = "erased-serde"; + version = "0.4.4"; + edition = "2021"; + sha256 = "1lx0si6iljzmfpblhn4b0ip3kw2yv4vjyca0riqz3ix311q80wrb"; + authors = [ + "David Tolnay <dtolnay@gmail.com>" + ]; + dependencies = [ + { + name = "serde"; + packageId = "serde"; + usesDefaultFeatures = false; + } + ]; + features = { + "alloc" = [ "serde/alloc" ]; + "default" = [ "std" ]; + "std" = [ "alloc" "serde/std" ]; + }; + resolvedDefaultFeatures = [ "alloc" ]; + }; "errno" = rec { crateName = "errno"; version = "0.3.8"; @@ -2786,6 +3419,136 @@ rec { ]; features = { }; }; + "event-listener 2.5.3" = rec { + crateName = "event-listener"; + version = "2.5.3"; + edition = "2018"; + sha256 = "1q4w3pndc518crld6zsqvvpy9lkzwahp2zgza9kbzmmqh9gif1h2"; + authors = [ + "Stjepan Glavina <stjepang@gmail.com>" + ]; + + }; + "event-listener 4.0.3" = rec { + crateName = "event-listener"; + version = "4.0.3"; + edition = "2021"; + sha256 = "0vk4smw1vf871vi76af1zn7w69jg3zmpjddpby2qq91bkg21bck7"; + authors = [ + "Stjepan Glavina <stjepang@gmail.com>" + ]; + dependencies = [ + { + name = "concurrent-queue"; + packageId = "concurrent-queue"; + usesDefaultFeatures = false; + } + { + name = "parking"; + packageId = "parking"; + optional = true; + target = { target, features }: (!(builtins.elem "wasm" target."family")); + } + { + name = "pin-project-lite"; + packageId = "pin-project-lite"; + } + ]; + features = { + "default" = [ "std" ]; + "parking" = [ "dep:parking" ]; + "portable-atomic" = [ "portable-atomic-util" "portable_atomic_crate" ]; + "portable-atomic-util" = [ "dep:portable-atomic-util" ]; + "portable_atomic_crate" = [ "dep:portable_atomic_crate" ]; + "std" = [ "concurrent-queue/std" "parking" ]; + }; + resolvedDefaultFeatures = [ "parking" "std" ]; + }; + "event-listener 5.2.0" = rec { + crateName = "event-listener"; + version = "5.2.0"; + edition = "2021"; + sha256 = "14fcnjgpfl22645nhc3hzkdq3a1v0srqacc3kfassg7sjj8vhprb"; + authors = [ + "Stjepan Glavina <stjepang@gmail.com>" + ]; + dependencies = [ + { + name = "concurrent-queue"; + packageId = "concurrent-queue"; + usesDefaultFeatures = false; + } + { + name = "parking"; + packageId = "parking"; + optional = true; + target = { target, features }: (!(builtins.elem "wasm" target."family")); + } + { + name = "pin-project-lite"; + packageId = "pin-project-lite"; + } + ]; + features = { + "default" = [ "std" ]; + "parking" = [ "dep:parking" ]; + "portable-atomic" = [ "portable-atomic-util" "portable_atomic_crate" ]; + "portable-atomic-util" = [ "dep:portable-atomic-util" ]; + "portable_atomic_crate" = [ "dep:portable_atomic_crate" ]; + "std" = [ "concurrent-queue/std" "parking" ]; + }; + resolvedDefaultFeatures = [ "default" "parking" "std" ]; + }; + "event-listener-strategy 0.4.0" = rec { + crateName = "event-listener-strategy"; + version = "0.4.0"; + edition = "2018"; + sha256 = "1lwprdjqp2ibbxhgm9khw7s7y7k4xiqj5i5yprqiks6mnrq4v3lm"; + authors = [ + "John Nunley <dev@notgull.net>" + ]; + dependencies = [ + { + name = "event-listener"; + packageId = "event-listener 4.0.3"; + usesDefaultFeatures = false; + } + { + name = "pin-project-lite"; + packageId = "pin-project-lite"; + } + ]; + features = { + "default" = [ "std" ]; + "std" = [ "event-listener/std" ]; + }; + resolvedDefaultFeatures = [ "std" ]; + }; + "event-listener-strategy 0.5.0" = rec { + crateName = "event-listener-strategy"; + version = "0.5.0"; + edition = "2021"; + sha256 = "148jflvjrq0zrr3dx3srv88jksj1klm4amy3b9fifjdpm75azvgy"; + authors = [ + "John Nunley <dev@notgull.net>" + ]; + dependencies = [ + { + name = "event-listener"; + packageId = "event-listener 5.2.0"; + usesDefaultFeatures = false; + } + { + name = "pin-project-lite"; + packageId = "pin-project-lite"; + } + ]; + features = { + "default" = [ "std" ]; + "std" = [ "event-listener/std" ]; + }; + resolvedDefaultFeatures = [ "std" ]; + }; "fastcdc" = rec { crateName = "fastcdc"; version = "3.1.0"; @@ -3280,6 +4043,53 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; + "futures-lite" = rec { + crateName = "futures-lite"; + version = "2.3.0"; + edition = "2021"; + sha256 = "19gk4my8zhfym6gwnpdjiyv2hw8cc098skkbkhryjdaf0yspwljj"; + authors = [ + "Stjepan Glavina <stjepang@gmail.com>" + "Contributors to futures-rs" + ]; + dependencies = [ + { + name = "fastrand"; + packageId = "fastrand"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "futures-core"; + packageId = "futures-core"; + usesDefaultFeatures = false; + } + { + name = "futures-io"; + packageId = "futures-io"; + optional = true; + } + { + name = "parking"; + packageId = "parking"; + optional = true; + } + { + name = "pin-project-lite"; + packageId = "pin-project-lite"; + } + ]; + features = { + "default" = [ "race" "std" ]; + "fastrand" = [ "dep:fastrand" ]; + "futures-io" = [ "dep:futures-io" ]; + "memchr" = [ "dep:memchr" ]; + "parking" = [ "dep:parking" ]; + "race" = [ "fastrand" ]; + "std" = [ "alloc" "fastrand/std" "futures-io" "parking" ]; + }; + resolvedDefaultFeatures = [ "alloc" "default" "fastrand" "futures-io" "parking" "race" "std" ]; + }; "futures-macro" = rec { crateName = "futures-macro"; version = "0.3.30"; @@ -3289,11 +4099,11 @@ rec { dependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "syn"; @@ -3441,6 +4251,100 @@ rec { ]; }; + "gcp_auth" = rec { + crateName = "gcp_auth"; + version = "0.10.0"; + edition = "2021"; + sha256 = "1m7lsh2gc7n9p0gs9k2qbxsrvchw1vz6dyz9a2ma322vd3m72b6y"; + dependencies = [ + { + name = "async-trait"; + packageId = "async-trait"; + } + { + name = "base64"; + packageId = "base64"; + } + { + name = "chrono"; + packageId = "chrono"; + features = [ "serde" ]; + } + { + name = "home"; + packageId = "home"; + } + { + name = "hyper"; + packageId = "hyper 0.14.28"; + features = [ "client" "runtime" "http2" ]; + } + { + name = "hyper-rustls"; + packageId = "hyper-rustls"; + usesDefaultFeatures = false; + features = [ "tokio-runtime" "http1" "http2" ]; + } + { + name = "ring"; + packageId = "ring"; + } + { + name = "rustls"; + packageId = "rustls 0.21.10"; + } + { + name = "rustls-pemfile"; + packageId = "rustls-pemfile 1.0.4"; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" "rc" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "thiserror"; + packageId = "thiserror"; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "fs" "sync" ]; + } + { + name = "tracing"; + packageId = "tracing"; + } + { + name = "tracing-futures"; + packageId = "tracing-futures"; + } + { + name = "url"; + packageId = "url"; + } + { + name = "which"; + packageId = "which 5.0.0"; + } + ]; + devDependencies = [ + { + name = "tokio"; + packageId = "tokio"; + features = [ "macros" "parking_lot" "rt-multi-thread" ]; + } + ]; + features = { + "default" = [ "hyper-rustls/rustls-native-certs" ]; + "webpki-roots" = [ "hyper-rustls/webpki-roots" ]; + }; + resolvedDefaultFeatures = [ "default" ]; + }; "genawaiter" = rec { crateName = "genawaiter"; version = "0.99.1"; @@ -3802,6 +4706,21 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; + "hex" = rec { + crateName = "hex"; + version = "0.4.3"; + edition = "2018"; + sha256 = "0w1a4davm1lgzpamwnba907aysmlrnygbqmfis2mqjx5m552a93z"; + authors = [ + "KokaKiwi <kokakiwi@kokakiwi.net>" + ]; + features = { + "default" = [ "std" ]; + "serde" = [ "dep:serde" ]; + "std" = [ "alloc" ]; + }; + resolvedDefaultFeatures = [ "alloc" ]; + }; "hex-literal" = rec { crateName = "hex-literal"; version = "0.4.1"; @@ -4232,6 +5151,11 @@ rec { usesDefaultFeatures = false; } { + name = "rustls-native-certs"; + packageId = "rustls-native-certs 0.6.3"; + optional = true; + } + { name = "tokio"; packageId = "tokio"; } @@ -4273,6 +5197,7 @@ rec { "webpki-roots" = [ "dep:webpki-roots" ]; "webpki-tokio" = [ "tokio-runtime" "webpki-roots" ]; }; + resolvedDefaultFeatures = [ "http1" "http2" "rustls-native-certs" "tokio-runtime" ]; }; "hyper-timeout" = rec { crateName = "hyper-timeout"; @@ -4452,6 +5377,16 @@ rec { ]; }; + "ident_case" = rec { + crateName = "ident_case"; + version = "1.0.1"; + edition = "2015"; + sha256 = "0fac21q6pwns8gh1hz3nbq15j8fi441ncl6w4vlnd1cmc55kiq5r"; + authors = [ + "Ted Driggs <ted.driggs@outlook.com>" + ]; + + }; "idna" = rec { crateName = "idna"; version = "0.5.0"; @@ -4579,6 +5514,12 @@ rec { usesDefaultFeatures = false; features = [ "raw" ]; } + { + name = "serde"; + packageId = "serde"; + optional = true; + usesDefaultFeatures = false; + } ]; buildDependencies = [ { @@ -4594,6 +5535,7 @@ rec { "serde" = [ "dep:serde" ]; "serde-1" = [ "serde" ]; }; + resolvedDefaultFeatures = [ "serde" "serde-1" "std" ]; }; "indexmap 2.1.0" = rec { crateName = "indexmap"; @@ -4612,6 +5554,12 @@ rec { usesDefaultFeatures = false; features = [ "raw" ]; } + { + name = "serde"; + packageId = "serde"; + optional = true; + usesDefaultFeatures = false; + } ]; features = { "arbitrary" = [ "dep:arbitrary" ]; @@ -4621,7 +5569,7 @@ rec { "rustc-rayon" = [ "dep:rustc-rayon" ]; "serde" = [ "dep:serde" ]; }; - resolvedDefaultFeatures = [ "default" "std" ]; + resolvedDefaultFeatures = [ "default" "serde" "std" ]; }; "instant" = rec { crateName = "instant"; @@ -4645,6 +5593,16 @@ rec { "web-sys" = [ "dep:web-sys" ]; }; }; + "inventory" = rec { + crateName = "inventory"; + version = "0.3.15"; + edition = "2021"; + sha256 = "0rspmi9qxz9hkajg4dx5hhwmcd3n3qw107hl3050hrs1izbd6n7r"; + authors = [ + "David Tolnay <dtolnay@gmail.com>" + ]; + + }; "ipnet" = rec { crateName = "ipnet"; version = "2.9.0"; @@ -5124,7 +6082,7 @@ rec { "default" = [ "std" "general" "errno" ]; "rustc-dep-of-std" = [ "core" "compiler_builtins" "no_std" ]; }; - resolvedDefaultFeatures = [ "elf" "errno" "general" "ioctl" "no_std" "std" ]; + resolvedDefaultFeatures = [ "elf" "errno" "general" "if_ether" "ioctl" "net" "netlink" "no_std" "prctl" "std" "xdp" ]; }; "litrs" = rec { crateName = "litrs"; @@ -5752,12 +6710,6 @@ rec { packageId = "enum-primitive-derive"; } { - name = "futures-util"; - packageId = "futures-util"; - optional = true; - features = [ "io" ]; - } - { name = "glob"; packageId = "glob"; } @@ -5770,6 +6722,11 @@ rec { packageId = "num-traits"; } { + name = "pin-project-lite"; + packageId = "pin-project-lite"; + optional = true; + } + { name = "serde"; packageId = "serde"; features = [ "derive" ]; @@ -5818,16 +6775,12 @@ rec { packageId = "pretty_assertions"; } { - name = "serde_json"; - packageId = "serde_json"; - } - { - name = "test-case"; - packageId = "test-case"; + name = "rstest"; + packageId = "rstest"; } { - name = "test-generator"; - packageId = "test-generator"; + name = "serde_json"; + packageId = "serde_json"; } { name = "tokio-test"; @@ -5839,11 +6792,13 @@ rec { } ]; features = { - "async" = [ "futures-util" "tokio" ]; - "futures-util" = [ "dep:futures-util" ]; + "async" = [ "tokio" ]; + "default" = [ "async" "wire" ]; + "pin-project-lite" = [ "dep:pin-project-lite" ]; "tokio" = [ "dep:tokio" ]; + "wire" = [ "tokio" "pin-project-lite" ]; }; - resolvedDefaultFeatures = [ "async" "futures-util" "tokio" ]; + resolvedDefaultFeatures = [ "async" "default" "pin-project-lite" "tokio" "wire" ]; }; "nom" = rec { crateName = "nom"; @@ -5918,6 +6873,16 @@ rec { "serde" = [ "dep:serde" ]; }; }; + "num-conv" = rec { + crateName = "num-conv"; + version = "0.1.0"; + edition = "2021"; + sha256 = "1ndiyg82q73783jq18isi71a7mjh56wxrk52rlvyx0mi5z9ibmai"; + authors = [ + "Jacob Pratt <jacob@jhpratt.dev>" + ]; + + }; "num-traits" = rec { crateName = "num-traits"; version = "0.2.18"; @@ -6577,6 +7542,19 @@ rec { ]; }; + "parking" = rec { + crateName = "parking"; + version = "2.2.0"; + edition = "2018"; + sha256 = "1blwbkq6im1hfxp5wlbr475mw98rsyc0bbr2d5n16m38z253p0dv"; + authors = [ + "Stjepan Glavina <stjepang@gmail.com>" + "The Rust Project Developers" + ]; + features = { + "loom" = [ "dep:loom" ]; + }; + }; "parking_lot 0.11.2" = rec { crateName = "parking_lot"; version = "0.11.2"; @@ -6775,6 +7753,7 @@ rec { "serde_derive" = [ "dep:serde_derive" ]; "unstable" = [ "generate" ]; }; + resolvedDefaultFeatures = [ "default" "graphmap" "matrix_graph" "stable_graph" ]; }; "pin-project" = rec { crateName = "pin-project"; @@ -6798,11 +7777,11 @@ rec { dependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "syn"; @@ -6829,6 +7808,41 @@ rec { ]; }; + "piper" = rec { + crateName = "piper"; + version = "0.2.1"; + edition = "2018"; + sha256 = "1m45fkdq7q5l9mv3b0ra10qwm0kb67rjp2q8y91958gbqjqk33b6"; + authors = [ + "Stjepan Glavina <stjepang@gmail.com>" + "John Nunley <dev@notgull.net>" + ]; + dependencies = [ + { + name = "atomic-waker"; + packageId = "atomic-waker"; + } + { + name = "fastrand"; + packageId = "fastrand"; + usesDefaultFeatures = false; + } + { + name = "futures-io"; + packageId = "futures-io"; + optional = true; + } + ]; + features = { + "default" = [ "std" ]; + "futures-io" = [ "dep:futures-io" ]; + "portable-atomic" = [ "atomic-waker/portable-atomic" "portable_atomic_crate" "portable-atomic-util" ]; + "portable-atomic-util" = [ "dep:portable-atomic-util" ]; + "portable_atomic_crate" = [ "dep:portable_atomic_crate" ]; + "std" = [ "fastrand/std" "futures-io" ]; + }; + resolvedDefaultFeatures = [ "default" "futures-io" "std" ]; + }; "pkcs8" = rec { crateName = "pkcs8"; version = "0.10.2"; @@ -6977,6 +7991,65 @@ rec { "image" = [ "dep:image" ]; }; }; + "polling" = rec { + crateName = "polling"; + version = "3.4.0"; + edition = "2021"; + sha256 = "052am20b5r03nwhpnjw86rv3dwsdabvb07anv3fqxfbs65r4w19h"; + authors = [ + "Stjepan Glavina <stjepang@gmail.com>" + "John Nunley <dev@notgull.net>" + ]; + dependencies = [ + { + name = "cfg-if"; + packageId = "cfg-if"; + } + { + name = "concurrent-queue"; + packageId = "concurrent-queue"; + target = { target, features }: (target."windows" or false); + } + { + name = "pin-project-lite"; + packageId = "pin-project-lite"; + target = { target, features }: (target."windows" or false); + } + { + name = "rustix"; + packageId = "rustix"; + usesDefaultFeatures = false; + target = { target, features }: ((target."unix" or false) || ("fuchsia" == target."os" or null) || ("vxworks" == target."os" or null)); + features = [ "event" "fs" "pipe" "process" "std" "time" ]; + } + { + name = "tracing"; + packageId = "tracing"; + usesDefaultFeatures = false; + } + { + name = "windows-sys"; + packageId = "windows-sys 0.52.0"; + target = { target, features }: (target."windows" or false); + features = [ "Wdk_Foundation" "Wdk_Storage_FileSystem" "Win32_Foundation" "Win32_Networking_WinSock" "Win32_Security" "Win32_Storage_FileSystem" "Win32_System_IO" "Win32_System_LibraryLoader" "Win32_System_Threading" "Win32_System_WindowsProgramming" ]; + } + ]; + + }; + "powerfmt" = rec { + crateName = "powerfmt"; + version = "0.2.0"; + edition = "2021"; + sha256 = "14ckj2xdpkhv3h6l5sdmb9f1d57z8hbfpdldjc2vl5givq2y77j3"; + authors = [ + "Jacob Pratt <jacob@jhpratt.dev>" + ]; + features = { + "default" = [ "std" "macros" ]; + "macros" = [ "dep:powerfmt-macros" ]; + "std" = [ "alloc" ]; + }; + }; "ppv-lite86" = rec { crateName = "ppv-lite86"; version = "0.2.17"; @@ -7027,7 +8100,7 @@ rec { dependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; usesDefaultFeatures = false; } { @@ -7040,7 +8113,7 @@ rec { devDependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; usesDefaultFeatures = false; } { @@ -7054,26 +8127,7 @@ rec { "verbatim" = [ "syn/parsing" ]; }; }; - "proc-macro2 0.4.30" = rec { - crateName = "proc-macro2"; - version = "0.4.30"; - edition = "2015"; - sha256 = "0nd71fl24sys066jrha6j7i34nfkjv44yzw8yww9742wmc8j0gfg"; - authors = [ - "Alex Crichton <alex@alexcrichton.com>" - ]; - dependencies = [ - { - name = "unicode-xid"; - packageId = "unicode-xid"; - } - ]; - features = { - "default" = [ "proc-macro" ]; - }; - resolvedDefaultFeatures = [ "default" "proc-macro" ]; - }; - "proc-macro2 1.0.76" = rec { + "proc-macro2" = rec { crateName = "proc-macro2"; version = "1.0.76"; edition = "2021"; @@ -7295,6 +8349,17 @@ rec { usesDefaultFeatures = false; } { + name = "pulldown-cmark"; + packageId = "pulldown-cmark"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "pulldown-cmark-to-cmark"; + packageId = "pulldown-cmark-to-cmark"; + optional = true; + } + { name = "regex"; packageId = "regex"; usesDefaultFeatures = false; @@ -7312,7 +8377,7 @@ rec { } { name = "which"; - packageId = "which"; + packageId = "which 4.4.2"; } ]; features = { @@ -7324,7 +8389,7 @@ rec { "pulldown-cmark-to-cmark" = [ "dep:pulldown-cmark-to-cmark" ]; "syn" = [ "dep:syn" ]; }; - resolvedDefaultFeatures = [ "default" "format" "prettyplease" "syn" ]; + resolvedDefaultFeatures = [ "cleanup-markdown" "default" "format" "prettyplease" "pulldown-cmark" "pulldown-cmark-to-cmark" "syn" ]; }; "prost-derive 0.11.9" = rec { crateName = "prost-derive"; @@ -7350,11 +8415,11 @@ rec { } { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "syn"; @@ -7388,11 +8453,11 @@ rec { } { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "syn"; @@ -7426,6 +8491,197 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; + "prost-wkt" = rec { + crateName = "prost-wkt"; + version = "0.5.0"; + edition = "2021"; + sha256 = "0h3c0jyfpg7f3s9a3mx6xcif28j3ir5c5qmps88bknpiy31zk3jd"; + authors = [ + "fdeantoni <fdeantoni@gmail.com>" + ]; + dependencies = [ + { + name = "chrono"; + packageId = "chrono"; + usesDefaultFeatures = false; + features = [ "serde" ]; + } + { + name = "inventory"; + packageId = "inventory"; + } + { + name = "prost"; + packageId = "prost 0.12.3"; + } + { + name = "serde"; + packageId = "serde"; + } + { + name = "serde_derive"; + packageId = "serde_derive"; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "typetag"; + packageId = "typetag"; + } + ]; + + }; + "prost-wkt-build" = rec { + crateName = "prost-wkt-build"; + version = "0.5.0"; + edition = "2021"; + sha256 = "0883g26vrhx07kv0dq85559pj95zxs10lx042pp4za2clplwlcav"; + authors = [ + "fdeantoni <fdeantoni@gmail.com>" + ]; + dependencies = [ + { + name = "heck"; + packageId = "heck"; + } + { + name = "prost"; + packageId = "prost 0.12.3"; + } + { + name = "prost-build"; + packageId = "prost-build"; + } + { + name = "prost-types"; + packageId = "prost-types"; + } + { + name = "quote"; + packageId = "quote"; + } + ]; + + }; + "prost-wkt-types" = rec { + crateName = "prost-wkt-types"; + version = "0.5.0"; + edition = "2021"; + sha256 = "1vipmgvqqzr3hn9z5v85mx9zznzjwyfpjy8xzg2v94a0f2lf8ns3"; + authors = [ + "fdeantoni <fdeantoni@gmail.com>" + ]; + dependencies = [ + { + name = "chrono"; + packageId = "chrono"; + usesDefaultFeatures = false; + features = [ "serde" ]; + } + { + name = "prost"; + packageId = "prost 0.12.3"; + } + { + name = "prost-wkt"; + packageId = "prost-wkt"; + } + { + name = "serde"; + packageId = "serde"; + } + { + name = "serde_derive"; + packageId = "serde_derive"; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + ]; + buildDependencies = [ + { + name = "prost"; + packageId = "prost 0.12.3"; + } + { + name = "prost-build"; + packageId = "prost-build"; + } + { + name = "prost-types"; + packageId = "prost-types"; + } + { + name = "prost-wkt-build"; + packageId = "prost-wkt-build"; + } + { + name = "regex"; + packageId = "regex"; + } + ]; + features = { + "default" = [ "std" ]; + "protobuf-src" = [ "dep:protobuf-src" ]; + "protox" = [ "dep:protox" ]; + "vendored-protoc" = [ "protobuf-src" ]; + "vendored-protox" = [ "protox" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; + "pulldown-cmark" = rec { + crateName = "pulldown-cmark"; + version = "0.9.6"; + edition = "2021"; + crateBin = [ ]; + sha256 = "0av876a31qvqhy7gzdg134zn4s10smlyi744mz9vrllkf906n82p"; + authors = [ + "Raph Levien <raph.levien@gmail.com>" + "Marcus Klaas de Vries <mail@marcusklaas.nl>" + ]; + dependencies = [ + { + name = "bitflags"; + packageId = "bitflags 2.4.2"; + } + { + name = "memchr"; + packageId = "memchr"; + } + { + name = "unicase"; + packageId = "unicase"; + } + ]; + features = { + "default" = [ "getopts" ]; + "getopts" = [ "dep:getopts" ]; + "serde" = [ "dep:serde" ]; + }; + }; + "pulldown-cmark-to-cmark" = rec { + crateName = "pulldown-cmark-to-cmark"; + version = "10.0.4"; + edition = "2018"; + sha256 = "0gc366cmd5jxal9m95l17rvqsm4dn62lywc8v5gwq8vcjvhyd501"; + authors = [ + "Sebastian Thiel <byronimo@gmail.com>" + "Dylan Owen <dyltotheo@gmail.com>" + "Alessandro Ogier <alessandro.ogier@gmail.com>" + "Zixian Cai <2891235+caizixian@users.noreply.github.com>" + ]; + dependencies = [ + { + name = "pulldown-cmark"; + packageId = "pulldown-cmark"; + usesDefaultFeatures = false; + } + ]; + + }; "quick-error" = rec { crateName = "quick-error"; version = "1.2.3"; @@ -7466,28 +8722,7 @@ rec { }; resolvedDefaultFeatures = [ "default" "overlapped-lists" "serde" "serialize" ]; }; - "quote 0.6.13" = rec { - crateName = "quote"; - version = "0.6.13"; - edition = "2015"; - sha256 = "1qgqq48jymp5h4y082aanf25hrw6bpb678xh3zw993qfhxmkpqkc"; - authors = [ - "David Tolnay <dtolnay@gmail.com>" - ]; - dependencies = [ - { - name = "proc-macro2"; - packageId = "proc-macro2 0.4.30"; - usesDefaultFeatures = false; - } - ]; - features = { - "default" = [ "proc-macro" ]; - "proc-macro" = [ "proc-macro2/proc-macro" ]; - }; - resolvedDefaultFeatures = [ "default" "proc-macro" ]; - }; - "quote 1.0.35" = rec { + "quote" = rec { crateName = "quote"; version = "1.0.35"; edition = "2018"; @@ -7498,7 +8733,7 @@ rec { dependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; usesDefaultFeatures = false; } ]; @@ -8372,9 +9607,9 @@ rec { }; "rstest" = rec { crateName = "rstest"; - version = "0.18.2"; + version = "0.19.0"; edition = "2021"; - sha256 = "1681ncnlzhc8894idm3pqf40nndn4k4kcp0kpv29n68a7hpspvlp"; + sha256 = "0c43nsxpm1b74jxc73xwg94is6bwqvfzkrr1xbqyx7j7l791clwx"; authors = [ "Michele d'Amico <michele.damico@gmail.com>" ]; @@ -8409,9 +9644,9 @@ rec { }; "rstest_macros" = rec { crateName = "rstest_macros"; - version = "0.18.2"; + version = "0.19.0"; edition = "2021"; - sha256 = "01g6rg60snmscipc9xiili7nsn0v25sv64713gp99y2jg0jgha6l"; + sha256 = "09ackagv8kc2v4xy0s7blyg4agij9bz9pbb31l5h4rqzrirdza84"; procMacro = true; authors = [ "Michele d'Amico <michele.damico@gmail.com>" @@ -8427,11 +9662,11 @@ rec { } { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "regex"; @@ -8474,7 +9709,7 @@ rec { dependencies = [ { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "rand"; @@ -8645,7 +9880,7 @@ rec { "thread" = [ "linux-raw-sys/prctl" ]; "use-libc" = [ "libc_errno" "libc" ]; }; - resolvedDefaultFeatures = [ "alloc" "default" "fs" "std" "termios" "use-libc-auxv" ]; + resolvedDefaultFeatures = [ "alloc" "default" "event" "fs" "net" "pipe" "process" "std" "termios" "time" "use-libc-auxv" ]; }; "rustls 0.21.10" = rec { crateName = "rustls"; @@ -9229,9 +10464,9 @@ rec { }; "serde" = rec { crateName = "serde"; - version = "1.0.195"; + version = "1.0.197"; edition = "2018"; - sha256 = "00kbc86kgaihpza0zdglcd2qq5468yg0dvvdmkli2y660bs1s9k3"; + sha256 = "1qjcxqd3p4yh5cmmax9q4ics1zy34j5ij32cvjj5dc5rw5rwic9z"; authors = [ "Erick Tryzelaar <erick.tryzelaar@gmail.com>" "David Tolnay <dtolnay@gmail.com>" @@ -9263,9 +10498,9 @@ rec { }; "serde_derive" = rec { crateName = "serde_derive"; - version = "1.0.195"; + version = "1.0.197"; edition = "2015"; - sha256 = "0b7ag1qm9q3fgwlmyk2ap5gjbqa9vyf2wfmj4xish6yq0f38zzj6"; + sha256 = "02v1x0sdv8qy06lpr6by4ar1n3jz3hmab15cgimpzhgd895v7c3y"; procMacro = true; authors = [ "Erick Tryzelaar <erick.tryzelaar@gmail.com>" @@ -9274,15 +10509,21 @@ rec { dependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; + usesDefaultFeatures = false; + features = [ "proc-macro" ]; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; + usesDefaultFeatures = false; + features = [ "proc-macro" ]; } { name = "syn"; packageId = "syn 2.0.48"; + usesDefaultFeatures = false; + features = [ "clone-impls" "derive" "parsing" "printing" "proc-macro" ]; } ]; features = { }; @@ -9326,7 +10567,7 @@ rec { "preserve_order" = [ "indexmap" "std" ]; "std" = [ "serde/std" ]; }; - resolvedDefaultFeatures = [ "default" "raw_value" "std" ]; + resolvedDefaultFeatures = [ "alloc" "default" "raw_value" "std" ]; }; "serde_path_to_error" = rec { crateName = "serde_path_to_error"; @@ -9348,6 +10589,44 @@ rec { ]; }; + "serde_qs" = rec { + crateName = "serde_qs"; + version = "0.12.0"; + edition = "2018"; + sha256 = "031kgpxbqkkxnql0k7sd80lyp98x7jc92311chrkc7k5d1as6c84"; + authors = [ + "Sam Scott <sam@osohq.com>" + ]; + dependencies = [ + { + name = "percent-encoding"; + packageId = "percent-encoding"; + } + { + name = "serde"; + packageId = "serde"; + } + { + name = "thiserror"; + packageId = "thiserror"; + } + ]; + features = { + "actix-web2" = [ "dep:actix-web2" ]; + "actix-web3" = [ "dep:actix-web3" ]; + "actix-web4" = [ "dep:actix-web4" ]; + "actix2" = [ "actix-web2" "futures" ]; + "actix3" = [ "actix-web3" "futures" ]; + "actix4" = [ "actix-web4" "futures" ]; + "axum" = [ "axum-framework" "futures" ]; + "axum-framework" = [ "dep:axum-framework" ]; + "futures" = [ "dep:futures" ]; + "tracing" = [ "dep:tracing" ]; + "warp" = [ "futures" "tracing" "warp-framework" ]; + "warp-framework" = [ "dep:warp-framework" ]; + }; + resolvedDefaultFeatures = [ "default" ]; + }; "serde_spanned" = rec { crateName = "serde_spanned"; version = "0.6.5"; @@ -9399,6 +10678,143 @@ rec { ]; }; + "serde_with" = rec { + crateName = "serde_with"; + version = "3.7.0"; + edition = "2021"; + sha256 = "16jn72cij27fxjafcsma1z5p587xkk8wqhp2yv98zy5vc7iv107f"; + authors = [ + "Jonas Bushart" + "Marcin Kaลบmierczak" + ]; + dependencies = [ + { + name = "base64"; + packageId = "base64"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "chrono"; + packageId = "chrono"; + rename = "chrono_0_4"; + optional = true; + usesDefaultFeatures = false; + features = [ "serde" ]; + } + { + name = "hex"; + packageId = "hex"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "indexmap"; + packageId = "indexmap 1.9.3"; + rename = "indexmap_1"; + optional = true; + usesDefaultFeatures = false; + features = [ "serde-1" ]; + } + { + name = "indexmap"; + packageId = "indexmap 2.1.0"; + rename = "indexmap_2"; + optional = true; + usesDefaultFeatures = false; + features = [ "serde" ]; + } + { + name = "serde"; + packageId = "serde"; + usesDefaultFeatures = false; + } + { + name = "serde_derive"; + packageId = "serde_derive"; + } + { + name = "serde_json"; + packageId = "serde_json"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "serde_with_macros"; + packageId = "serde_with_macros"; + optional = true; + } + { + name = "time"; + packageId = "time"; + rename = "time_0_3"; + optional = true; + usesDefaultFeatures = false; + } + ]; + devDependencies = [ + { + name = "serde"; + packageId = "serde"; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + features = [ "preserve_order" ]; + } + ]; + features = { + "alloc" = [ "serde/alloc" "base64?/alloc" "chrono_0_4?/alloc" "hex?/alloc" "serde_json?/alloc" "time_0_3?/alloc" ]; + "base64" = [ "dep:base64" "alloc" ]; + "chrono" = [ "chrono_0_4" ]; + "chrono_0_4" = [ "dep:chrono_0_4" ]; + "default" = [ "std" "macros" ]; + "guide" = [ "dep:doc-comment" "dep:document-features" "macros" "std" ]; + "hashbrown_0_14" = [ "dep:hashbrown_0_14" "alloc" ]; + "hex" = [ "dep:hex" "alloc" ]; + "indexmap" = [ "indexmap_1" ]; + "indexmap_1" = [ "dep:indexmap_1" "alloc" ]; + "indexmap_2" = [ "dep:indexmap_2" "alloc" ]; + "json" = [ "dep:serde_json" "alloc" ]; + "macros" = [ "dep:serde_with_macros" ]; + "schemars_0_8" = [ "dep:schemars_0_8" "std" "serde_with_macros?/schemars_0_8" ]; + "std" = [ "alloc" "serde/std" "chrono_0_4?/clock" "chrono_0_4?/std" "indexmap_1?/std" "indexmap_2?/std" "time_0_3?/serde-well-known" "time_0_3?/std" ]; + "time_0_3" = [ "dep:time_0_3" ]; + }; + resolvedDefaultFeatures = [ "alloc" "base64" "default" "macros" "std" ]; + }; + "serde_with_macros" = rec { + crateName = "serde_with_macros"; + version = "3.7.0"; + edition = "2021"; + sha256 = "0mbnika5bw1mvgnl50rs7wfzj7dwxzgwqxnq6656694j38bdqqb5"; + procMacro = true; + authors = [ + "Jonas Bushart" + ]; + dependencies = [ + { + name = "darling"; + packageId = "darling"; + } + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "syn"; + packageId = "syn 2.0.48"; + features = [ "extra-traits" "full" "parsing" ]; + } + ]; + features = { }; + }; "sha1" = rec { crateName = "sha1"; version = "0.10.6"; @@ -9720,11 +11136,11 @@ rec { } { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "syn"; @@ -9860,11 +11276,11 @@ rec { dependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "structmeta-derive"; @@ -9896,11 +11312,11 @@ rec { dependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "syn"; @@ -9929,39 +11345,6 @@ rec { "default" = [ "std" "i128" ]; }; }; - "syn 0.15.44" = rec { - crateName = "syn"; - version = "0.15.44"; - edition = "2015"; - sha256 = "1id5g6x6zihv3j7hwrw3m1jp636bg8dpi671r7zy3jvpkavb794w"; - authors = [ - "David Tolnay <dtolnay@gmail.com>" - ]; - dependencies = [ - { - name = "proc-macro2"; - packageId = "proc-macro2 0.4.30"; - usesDefaultFeatures = false; - } - { - name = "quote"; - packageId = "quote 0.6.13"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "unicode-xid"; - packageId = "unicode-xid"; - } - ]; - features = { - "default" = [ "derive" "parsing" "printing" "clone-impls" "proc-macro" ]; - "printing" = [ "quote" ]; - "proc-macro" = [ "proc-macro2/proc-macro" "quote/proc-macro" ]; - "quote" = [ "dep:quote" ]; - }; - resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "full" "parsing" "printing" "proc-macro" "quote" ]; - }; "syn 1.0.109" = rec { crateName = "syn"; version = "1.0.109"; @@ -9973,12 +11356,12 @@ rec { dependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; usesDefaultFeatures = false; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; optional = true; usesDefaultFeatures = false; } @@ -10007,12 +11390,12 @@ rec { dependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; usesDefaultFeatures = false; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; optional = true; usesDefaultFeatures = false; } @@ -10161,129 +11544,6 @@ rec { ]; }; - "test-case" = rec { - crateName = "test-case"; - version = "3.3.1"; - edition = "2021"; - sha256 = "1a380yzm6787737cw7s09jqmkn9035hghahradl2ikdg2gfm09gb"; - authors = [ - "Marcin Sas-Szymanski <marcin.sas-szymanski@anixe.pl>" - "Wojciech Polak <frondeus@gmail.com>" - "ลukasz Biel <lukasz.p.biel@gmail.com>" - ]; - dependencies = [ - { - name = "test-case-macros"; - packageId = "test-case-macros"; - usesDefaultFeatures = false; - } - ]; - features = { - "regex" = [ "dep:regex" ]; - "with-regex" = [ "regex" "test-case-macros/with-regex" ]; - }; - }; - "test-case-core" = rec { - crateName = "test-case-core"; - version = "3.3.1"; - edition = "2021"; - sha256 = "0krqi0gbi1yyycigyjlak63r8h1n0vms7mg3kckqwlfd87c7zjxd"; - authors = [ - "Marcin Sas-Szymanski <marcin.sas-szymanski@anixe.pl>" - "Wojciech Polak <frondeus@gmail.com>" - "ลukasz Biel <lukasz.p.biel@gmail.com>" - ]; - dependencies = [ - { - name = "cfg-if"; - packageId = "cfg-if"; - } - { - name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; - } - { - name = "quote"; - packageId = "quote 1.0.35"; - } - { - name = "syn"; - packageId = "syn 2.0.48"; - features = [ "full" "extra-traits" ]; - } - ]; - features = { }; - }; - "test-case-macros" = rec { - crateName = "test-case-macros"; - version = "3.3.1"; - edition = "2021"; - sha256 = "1yvgky3qax73bic6m368q04xc955p4a91mddd6b5fk7d04mfg2aw"; - procMacro = true; - authors = [ - "Marcin Sas-Szymanski <marcin.sas-szymanski@anixe.pl>" - "Wojciech Polak <frondeus@gmail.com>" - "ลukasz Biel <lukasz.p.biel@gmail.com>" - ]; - dependencies = [ - { - name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; - } - { - name = "quote"; - packageId = "quote 1.0.35"; - } - { - name = "syn"; - packageId = "syn 2.0.48"; - features = [ "full" "extra-traits" "parsing" ]; - } - { - name = "test-case-core"; - packageId = "test-case-core"; - usesDefaultFeatures = false; - } - ]; - features = { - "with-regex" = [ "test-case-core/with-regex" ]; - }; - }; - "test-generator" = rec { - crateName = "test-generator"; - version = "0.3.0"; - edition = "2018"; - workspace_member = null; - src = pkgs.fetchgit { - url = "https://github.com/JamesGuthrie/test-generator.git"; - rev = "82e799979980962aec1aa324ec6e0e4cad781f41"; - sha256 = "08brp3qqa55hijc7xby3lam2cc84hvx1zzfqv6lj7smlczh8k32y"; - }; - procMacro = true; - authors = [ - "Frank Rehberger <frehberg@gmail.com>" - ]; - dependencies = [ - { - name = "glob"; - packageId = "glob"; - } - { - name = "proc-macro2"; - packageId = "proc-macro2 0.4.30"; - } - { - name = "quote"; - packageId = "quote 0.6.13"; - } - { - name = "syn"; - packageId = "syn 0.15.44"; - features = [ "full" ]; - } - ]; - - }; "test-strategy" = rec { crateName = "test-strategy"; version = "0.2.1"; @@ -10296,11 +11556,11 @@ rec { dependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "structmeta"; @@ -10355,11 +11615,11 @@ rec { dependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "syn"; @@ -10388,6 +11648,120 @@ rec { ]; features = { }; }; + "time" = rec { + crateName = "time"; + version = "0.3.34"; + edition = "2021"; + sha256 = "0jc7wgprzqjhzd0nqkbmdlnjwyddnswmjw86ni2vq55v45jqn968"; + authors = [ + "Jacob Pratt <open-source@jhpratt.dev>" + "Time contributors" + ]; + dependencies = [ + { + name = "deranged"; + packageId = "deranged"; + usesDefaultFeatures = false; + features = [ "powerfmt" ]; + } + { + name = "itoa"; + packageId = "itoa"; + optional = true; + } + { + name = "num-conv"; + packageId = "num-conv"; + } + { + name = "powerfmt"; + packageId = "powerfmt"; + usesDefaultFeatures = false; + } + { + name = "serde"; + packageId = "serde"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "time-core"; + packageId = "time-core"; + } + { + name = "time-macros"; + packageId = "time-macros"; + optional = true; + } + ]; + devDependencies = [ + { + name = "num-conv"; + packageId = "num-conv"; + } + { + name = "serde"; + packageId = "serde"; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + { + name = "time-macros"; + packageId = "time-macros"; + } + ]; + features = { + "alloc" = [ "serde?/alloc" ]; + "default" = [ "std" ]; + "formatting" = [ "dep:itoa" "std" "time-macros?/formatting" ]; + "large-dates" = [ "time-macros?/large-dates" ]; + "local-offset" = [ "std" "dep:libc" "dep:num_threads" ]; + "macros" = [ "dep:time-macros" ]; + "parsing" = [ "time-macros?/parsing" ]; + "quickcheck" = [ "dep:quickcheck" "alloc" "deranged/quickcheck" ]; + "rand" = [ "dep:rand" "deranged/rand" ]; + "serde" = [ "dep:serde" "time-macros?/serde" "deranged/serde" ]; + "serde-human-readable" = [ "serde" "formatting" "parsing" ]; + "serde-well-known" = [ "serde" "formatting" "parsing" ]; + "std" = [ "alloc" "deranged/std" ]; + "wasm-bindgen" = [ "dep:js-sys" ]; + }; + resolvedDefaultFeatures = [ "alloc" "formatting" "parsing" "serde" "serde-well-known" "std" ]; + }; + "time-core" = rec { + crateName = "time-core"; + version = "0.1.2"; + edition = "2021"; + sha256 = "1wx3qizcihw6z151hywfzzyd1y5dl804ydyxci6qm07vbakpr4pg"; + authors = [ + "Jacob Pratt <open-source@jhpratt.dev>" + "Time contributors" + ]; + + }; + "time-macros" = rec { + crateName = "time-macros"; + version = "0.2.17"; + edition = "2021"; + sha256 = "0x3pahhk2751c6kqqq9dk6lz0gydbnxr44q01wpjlrz687ps78vv"; + procMacro = true; + authors = [ + "Jacob Pratt <open-source@jhpratt.dev>" + "Time contributors" + ]; + dependencies = [ + { + name = "num-conv"; + packageId = "num-conv"; + } + { + name = "time-core"; + packageId = "time-core"; + } + ]; + features = { }; + resolvedDefaultFeatures = [ "formatting" "parsing" "serde" ]; + }; "tinytemplate" = rec { crateName = "tinytemplate"; version = "1.2.1"; @@ -10680,11 +12054,11 @@ rec { dependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "syn"; @@ -11404,7 +12778,7 @@ rec { } { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "prost-build"; @@ -11413,7 +12787,7 @@ rec { } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "syn"; @@ -11426,7 +12800,7 @@ rec { "prost" = [ "prost-build" ]; "prost-build" = [ "dep:prost-build" ]; }; - resolvedDefaultFeatures = [ "default" "prost" "prost-build" "transport" ]; + resolvedDefaultFeatures = [ "cleanup-markdown" "default" "prost" "prost-build" "transport" ]; }; "tonic-reflection" = rec { crateName = "tonic-reflection"; @@ -11683,11 +13057,11 @@ rec { dependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "syn"; @@ -11728,6 +13102,42 @@ rec { }; resolvedDefaultFeatures = [ "default" "once_cell" "std" "valuable" ]; }; + "tracing-futures" = rec { + crateName = "tracing-futures"; + version = "0.2.5"; + edition = "2018"; + sha256 = "1wimg0iwa2ldq7xv98lvivvf3q9ykfminig8r1bs0ig22np9bl4p"; + authors = [ + "Eliza Weisman <eliza@buoyant.io>" + "Tokio Contributors <team@tokio.rs>" + ]; + dependencies = [ + { + name = "pin-project"; + packageId = "pin-project"; + optional = true; + } + { + name = "tracing"; + packageId = "tracing"; + usesDefaultFeatures = false; + } + ]; + features = { + "default" = [ "std-future" "std" ]; + "futures" = [ "dep:futures" ]; + "futures-01" = [ "futures_01" "std" ]; + "futures-03" = [ "std-future" "futures" "futures-task" "std" ]; + "futures-task" = [ "dep:futures-task" ]; + "futures_01" = [ "dep:futures_01" ]; + "pin-project" = [ "dep:pin-project" ]; + "std" = [ "tracing/std" ]; + "std-future" = [ "pin-project" ]; + "tokio" = [ "dep:tokio" ]; + "tokio-executor" = [ "dep:tokio-executor" ]; + }; + resolvedDefaultFeatures = [ "default" "pin-project" "std" "std-future" ]; + }; "tracing-log" = rec { crateName = "tracing-log"; version = "0.2.0"; @@ -12108,8 +13518,8 @@ rec { ]; devDependencies = [ { - name = "test-case"; - packageId = "test-case"; + name = "rstest"; + packageId = "rstest"; } ]; features = { @@ -12137,6 +13547,11 @@ rec { packageId = "async-tempfile"; } { + name = "bigtable_rs"; + packageId = "bigtable_rs"; + optional = true; + } + { name = "blake3"; packageId = "blake3"; features = [ "rayon" "std" "traits-preview" ]; @@ -12190,6 +13605,10 @@ rec { packageId = "parking_lot 0.12.1"; } { + name = "petgraph"; + packageId = "petgraph"; + } + { name = "pin-project-lite"; packageId = "pin-project-lite"; } @@ -12198,6 +13617,19 @@ rec { packageId = "prost 0.12.3"; } { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde_qs"; + packageId = "serde_qs"; + } + { + name = "serde_with"; + packageId = "serde_with"; + } + { name = "sled"; packageId = "sled"; } @@ -12216,6 +13648,10 @@ rec { features = [ "fs" "net" ]; } { + name = "tokio-tar"; + packageId = "tokio-tar"; + } + { name = "tokio-util"; packageId = "tokio-util"; features = [ "io" "io-util" ]; @@ -12292,6 +13728,10 @@ rec { ]; devDependencies = [ { + name = "async-process"; + packageId = "async-process"; + } + { name = "hex-literal"; packageId = "hex-literal"; } @@ -12308,22 +13748,22 @@ rec { packageId = "tempfile"; } { - name = "test-case"; - packageId = "test-case"; - } - { name = "tokio-retry"; packageId = "tokio-retry"; } + { + name = "xattr"; + packageId = "xattr"; + } ]; features = { - "cloud" = [ "object_store/aws" "object_store/azure" "object_store/gcp" ]; + "cloud" = [ "dep:bigtable_rs" "object_store/aws" "object_store/azure" "object_store/gcp" ]; "fs" = [ "dep:libc" "dep:fuse-backend-rs" ]; "fuse" = [ "fs" ]; "tonic-reflection" = [ "dep:tonic-reflection" ]; "virtiofs" = [ "fs" "dep:vhost" "dep:vhost-user-backend" "dep:virtio-queue" "dep:vm-memory" "dep:vmm-sys-util" "dep:virtio-bindings" "fuse-backend-rs?/vhost-user-fs" "fuse-backend-rs?/virtiofs" ]; }; - resolvedDefaultFeatures = [ "cloud" "default" "fs" "fuse" "tonic-reflection" "virtiofs" ]; + resolvedDefaultFeatures = [ "cloud" "default" "fs" "fuse" "integration" "tonic-reflection" "virtiofs" ]; }; "tvix-cli" = rec { crateName = "tvix-cli"; @@ -12408,12 +13848,6 @@ rec { packageId = "wu-manber"; } ]; - devDependencies = [ - { - name = "test-case"; - packageId = "test-case"; - } - ]; }; "tvix-eval" = rec { @@ -12599,11 +14033,11 @@ rec { dependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "syn"; @@ -12636,10 +14070,6 @@ rec { features = [ "tokio" "gzip" "bzip2" "xz" ]; } { - name = "async-recursion"; - packageId = "async-recursion"; - } - { name = "bstr"; packageId = "bstr"; } @@ -12660,6 +14090,10 @@ rec { packageId = "magic"; } { + name = "md-5"; + packageId = "md-5"; + } + { name = "nix-compat"; packageId = "nix-compat"; } @@ -12682,6 +14116,10 @@ rec { packageId = "serde_json"; } { + name = "sha1"; + packageId = "sha1"; + } + { name = "sha2"; packageId = "sha2"; } @@ -12725,6 +14163,10 @@ rec { usesDefaultFeatures = false; } { + name = "url"; + packageId = "url"; + } + { name = "walkdir"; packageId = "walkdir"; } @@ -12764,10 +14206,6 @@ rec { name = "tempfile"; packageId = "tempfile"; } - { - name = "test-case"; - packageId = "test-case"; - } ]; features = { "default" = [ "nix_tests" ]; @@ -12825,14 +14263,20 @@ rec { packageId = "anyhow"; } { - name = "async-recursion"; - packageId = "async-recursion"; + name = "async-compression"; + packageId = "async-compression"; + features = [ "tokio" "bzip2" "gzip" "xz" "zstd" ]; } { name = "async-stream"; packageId = "async-stream"; } { + name = "bigtable_rs"; + packageId = "bigtable_rs"; + optional = true; + } + { name = "blake3"; packageId = "blake3"; features = [ "rayon" "std" ]; @@ -12902,6 +14346,23 @@ rec { features = [ "rustls-tls-native-roots" "stream" ]; } { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "serde_qs"; + packageId = "serde_qs"; + } + { + name = "serde_with"; + packageId = "serde_with"; + } + { name = "sha2"; packageId = "sha2"; } @@ -12972,10 +14433,6 @@ rec { name = "walkdir"; packageId = "walkdir"; } - { - name = "xz2"; - packageId = "xz2"; - } ]; buildDependencies = [ { @@ -12989,6 +14446,10 @@ rec { ]; devDependencies = [ { + name = "async-process"; + packageId = "async-process"; + } + { name = "rstest"; packageId = "rstest"; } @@ -13001,23 +14462,19 @@ rec { packageId = "tempfile"; } { - name = "test-case"; - packageId = "test-case"; - } - { name = "tokio-retry"; packageId = "tokio-retry"; } ]; features = { - "cloud" = [ "tvix-castore/cloud" ]; + "cloud" = [ "dep:bigtable_rs" "tvix-castore/cloud" ]; "default" = [ "cloud" "fuse" "otlp" "tonic-reflection" ]; "fuse" = [ "tvix-castore/fuse" ]; "otlp" = [ "dep:opentelemetry" "dep:opentelemetry-otlp" "dep:opentelemetry_sdk" ]; "tonic-reflection" = [ "dep:tonic-reflection" "tvix-castore/tonic-reflection" ]; "virtiofs" = [ "tvix-castore/virtiofs" ]; }; - resolvedDefaultFeatures = [ "cloud" "default" "fuse" "otlp" "tonic-reflection" "virtiofs" ]; + resolvedDefaultFeatures = [ "cloud" "default" "fuse" "integration" "otlp" "tonic-reflection" "virtiofs" ]; }; "typenum" = rec { crateName = "typenum"; @@ -13034,6 +14491,70 @@ rec { "scale_info" = [ "scale-info/derive" ]; }; }; + "typetag" = rec { + crateName = "typetag"; + version = "0.2.16"; + edition = "2021"; + sha256 = "1bwswa9ah2sc6fmlfw2pim73rr1n6vhfwmidrsga8cn09r0ih7b6"; + authors = [ + "David Tolnay <dtolnay@gmail.com>" + ]; + dependencies = [ + { + name = "erased-serde"; + packageId = "erased-serde"; + usesDefaultFeatures = false; + features = [ "alloc" ]; + } + { + name = "inventory"; + packageId = "inventory"; + } + { + name = "once_cell"; + packageId = "once_cell"; + usesDefaultFeatures = false; + features = [ "alloc" ]; + } + { + name = "serde"; + packageId = "serde"; + usesDefaultFeatures = false; + features = [ "alloc" "derive" ]; + } + { + name = "typetag-impl"; + packageId = "typetag-impl"; + } + ]; + + }; + "typetag-impl" = rec { + crateName = "typetag-impl"; + version = "0.2.16"; + edition = "2021"; + sha256 = "1cabnvm526bcgh1sh34js5ils0gz4xwlgvwhm992acdr8xzqhwxc"; + procMacro = true; + authors = [ + "David Tolnay <dtolnay@gmail.com>" + ]; + dependencies = [ + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "syn"; + packageId = "syn 2.0.48"; + features = [ "full" ]; + } + ]; + + }; "unarray" = rec { crateName = "unarray"; version = "0.1.4"; @@ -13041,6 +14562,22 @@ rec { sha256 = "154smf048k84prsdgh09nkm2n0w0336v84jd4zikyn6v6jrqbspa"; }; + "unicase" = rec { + crateName = "unicase"; + version = "2.7.0"; + edition = "2015"; + sha256 = "12gd74j79f94k4clxpf06l99wiv4p30wjr0qm04ihqk9zgdd9lpp"; + authors = [ + "Sean McArthur <sean@seanmonstar.com>" + ]; + buildDependencies = [ + { + name = "version_check"; + packageId = "version_check"; + } + ]; + features = { }; + }; "unicode-bidi" = rec { crateName = "unicode-bidi"; version = "0.3.15"; @@ -13119,18 +14656,6 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; - "unicode-xid" = rec { - crateName = "unicode-xid"; - version = "0.1.0"; - edition = "2015"; - sha256 = "1z57lqh4s18rr4x0j4fw4fmp9hf9346h0kmdgqsqx0fhjr3k0wpw"; - authors = [ - "erick.tryzelaar <erick.tryzelaar@gmail.com>" - "kwantam <kwantam@gmail.com>" - ]; - features = { }; - resolvedDefaultFeatures = [ "default" ]; - }; "untrusted" = rec { crateName = "untrusted"; version = "0.9.0"; @@ -13598,11 +15123,11 @@ rec { } { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "syn"; @@ -13664,7 +15189,7 @@ rec { dependencies = [ { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "wasm-bindgen-macro-support"; @@ -13688,11 +15213,11 @@ rec { dependencies = [ { name = "proc-macro2"; - packageId = "proc-macro2 1.0.76"; + packageId = "proc-macro2"; } { name = "quote"; - packageId = "quote 1.0.35"; + packageId = "quote"; } { name = "syn"; @@ -14256,7 +15781,7 @@ rec { ]; }; - "which" = rec { + "which 4.4.2" = rec { crateName = "which"; version = "4.4.2"; edition = "2021"; @@ -14290,6 +15815,46 @@ rec { "regex" = [ "dep:regex" ]; }; }; + "which 5.0.0" = rec { + crateName = "which"; + version = "5.0.0"; + edition = "2021"; + sha256 = "053fpbczryyn8lcbpkvwl8v2rzld0pr30r5lh1cxv87kjs2ymwwv"; + authors = [ + "Harry Fei <tiziyuanfang@gmail.com>" + ]; + dependencies = [ + { + name = "either"; + packageId = "either"; + } + { + name = "home"; + packageId = "home"; + target = { target, features }: ((target."windows" or false) || (target."unix" or false) || ("redox" == target."os" or null)); + } + { + name = "once_cell"; + packageId = "once_cell"; + target = { target, features }: (target."windows" or false); + } + { + name = "rustix"; + packageId = "rustix"; + usesDefaultFeatures = false; + features = [ "fs" "std" ]; + } + { + name = "windows-sys"; + packageId = "windows-sys 0.48.0"; + target = { target, features }: (target."windows" or false); + features = [ "Win32_Storage_FileSystem" "Win32_Foundation" ]; + } + ]; + features = { + "regex" = [ "dep:regex" ]; + }; + }; "winapi" = rec { crateName = "winapi"; version = "0.3.9"; @@ -14908,7 +16473,7 @@ rec { "Win32_Web" = [ "Win32" ]; "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; }; - resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_NetworkManagement" "Win32_NetworkManagement_IpHelper" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Com" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_Memory" "Win32_System_Threading" "Win32_UI" "Win32_UI_Shell" "default" ]; + resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Win32" "Win32_Foundation" "Win32_NetworkManagement" "Win32_NetworkManagement_IpHelper" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Com" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_Threading" "Win32_System_WindowsProgramming" "Win32_UI" "Win32_UI_Shell" "default" ]; }; "windows-targets 0.48.5" = rec { crateName = "windows-targets"; @@ -15505,8 +17070,9 @@ rec { # because we compiled those test binaries in the former and not the latter. # So all paths will expect source tree to be there and not in the build top directly. # For example: $NIX_BUILD_TOP := /build in general, if you ask yourself. - # TODO(raitobezarius): I believe there could be more edge cases if `crate.sourceRoot` - # do exist but it's very hard to reason about them, so let's wait until the first bug report. + # NOTE: There could be edge cases if `crate.sourceRoot` does exist but + # it's very hard to reason about them. + # Open a bug if you run into this! mkdir -p source/ cd source/ diff --git a/tvix/boot/default.nix b/tvix/boot/default.nix index 22a696060c..85995ffbf2 100644 --- a/tvix/boot/default.nix +++ b/tvix/boot/default.nix @@ -31,14 +31,14 @@ rec { }); # A build framework for minimal initrds - uroot = pkgs.buildGoModule { + uroot = pkgs.buildGoModule rec { pname = "u-root"; - version = "unstable-2023-09-20"; + version = "0.14.0"; src = pkgs.fetchFromGitHub { owner = "u-root"; repo = "u-root"; - rev = "72921548ce2e88c4c5b62e83c717cbd834b58067"; - hash = "sha256-fEoUGqh6ZXprtSpJ55MeuSFe7L5A/rkIIVLCwxbPHzE="; + rev = "v${version}"; + hash = "sha256-8zA3pHf45MdUcq/MA/mf0KCTxB1viHieU/oigYwIPgo="; }; vendorHash = null; @@ -67,7 +67,7 @@ rec { # Start a `tvix-store` virtiofs daemon from $PATH, then a cloud-hypervisor # pointed to it. # Supports the following env vars (and defaults) - # CH_NUM_CPUS=1 + # CH_NUM_CPUS=2 # CH_MEM_SIZE=512M # CH_CMDLINE="" runVM = pkgs.writers.writeBashBin "run-tvix-vm" '' @@ -89,7 +89,7 @@ rec { # Wait for the socket to exist. until [ -e $tempdir/tvix.sock ]; do sleep 0.1; done - CH_NUM_CPUS="''${CH_NUM_CPUS:-1}" + CH_NUM_CPUS="''${CH_NUM_CPUS:-2}" CH_MEM_SIZE="''${CH_MEM_SIZE:-512M}" CH_CMDLINE="''${CH_CMDLINE:-}" @@ -102,7 +102,7 @@ rec { --kernel ${kernel.dev}/vmlinux \ --initramfs ${initrd} \ --cmdline "console=ttyS0 $CH_CMDLINE" \ - --fs tag=tvix,socket=$tempdir/tvix.sock,num_queues=1,queue_size=512 + --fs tag=tvix,socket=$tempdir/tvix.sock,num_queues=''${CH_NUM_CPU},queue_size=512 ''; meta.ci.targets = [ diff --git a/tvix/boot/tests/default.nix b/tvix/boot/tests/default.nix index 108803d7ab..d16dba79f1 100644 --- a/tvix/boot/tests/default.nix +++ b/tvix/boot/tests/default.nix @@ -32,14 +32,12 @@ let pkgs.stdenv.mkDerivation { name = "run-vm"; + __structuredAttrs = true; + exportReferencesGraph.closure = [ path ]; + nativeBuildInputs = [ depot.tvix.store depot.tvix.boot.runVM - ] ++ lib.optionals isClosure [ - depot.tvix.nar-bridge - pkgs.curl - pkgs.parallel - pkgs.xz.bin ]; buildCommand = '' touch $out @@ -67,42 +65,43 @@ let echo "imported to $outpath" '' + lib.optionalString (isClosure) '' - echo "Starting nar-bridgeโฆ" - nar-bridge-http \ - --otlp=false \ - --store-addr=unix://$PWD/tvix-store.sock \ - --listen-addr=$PWD/nar-bridge.sock & - - # Wait for the socket to be created. - while [ ! -e $PWD/nar-bridge.sock ]; do sleep 1; done - - # Upload. We can't use nix copy --to http://โฆ, as it wants access to the nix db. - # However, we can use mkBinaryCache to assemble .narinfo and .nar.xz to upload, - # and then drive a HTTP client ourselves. - to_upload=${pkgs.mkBinaryCache { rootPaths = [path];}} - - # Upload all NAR files (with some parallelism). - # As mkBinaryCache produces them xz-compressed, unpack them on the fly. - # nar-bridge doesn't care about the path we upload *to*, but a - # subsequent .narinfo upload need to refer to its contents (by narhash). - echo -e "Uploading NARsโฆ " - ls -d $to_upload/nar/*.nar.xz | parallel 'xz -d < {} | curl -s -T - --unix-socket $PWD/nar-bridge.sock http://localhost:9000/nar/$(basename {} | cut -d "." -f 1).nar' - echo "Done." - - # Upload all NARInfo files. - # FUTUREWORK: This doesn't upload them in order, and currently relies - # on PathInfoService not doing any checking. - # In the future, we might want to make this behaviour configurable, - # and disable checking here, to keep the logic simple. - ls -d $to_upload/*.narinfo | parallel 'curl -s -T - --unix-socket $PWD/nar-bridge.sock http://localhost:9000/$(basename {}) < {}' + echo "Copying closure ${path}โฆ" + # This picks up the `closure` key in `$NIX_ATTRS_JSON_FILE` automatically. + tvix-store --otlp=false copy '' + '' # Invoke a VM using tvix as the backing store, ensure the outpath appears in its listing. + echo "Starting VMโฆ" CH_CMDLINE="${vmCmdline}" run-tvix-vm 2>&1 | tee output.txt grep "${assertVMOutput}" output.txt ''; requiredSystemFeatures = [ "kvm" ]; }; + + systemFor = sys: (depot.ops.nixos.nixosFor sys).system; + + testSystem = systemFor ({ modulesPath, pkgs, ... }: { + # Set some options necessary to evaluate. + boot.loader.systemd-boot.enable = true; + # TODO: figure out how to disable this without causing eval to fail + fileSystems."/" = { + device = "/dev/root"; + fsType = "tmpfs"; + }; + + services.getty.helpLine = "Onwards and upwards."; + systemd.services.do-shutdown = { + after = [ "getty.target" ]; + description = "Shut down again"; + wantedBy = [ "multi-user.target" ]; + serviceConfig.Type = "oneshot"; + script = "/run/current-system/sw/bin/systemctl poweroff --when=+10s"; + }; + + # Don't warn about stateVersion. + system.stateVersion = "24.05"; + }); + in depot.nix.readTree.drvTargets { @@ -128,4 +127,12 @@ depot.nix.readTree.drvTargets path = depot.tvix.store; isClosure = true; }); + + closure-nixos = (mkBootTest { + blobServiceAddr = "objectstore+file://$PWD/blobs"; + path = testSystem; + isClosure = true; + vmCmdline = "init=${testSystem}/init panic=-1"; # reboot immediately on panic + assertVMOutput = "Onwards and upwards."; + }); } diff --git a/tvix/build/Cargo.toml b/tvix/build/Cargo.toml index bda2d136c7..626fd35d77 100644 --- a/tvix/build/Cargo.toml +++ b/tvix/build/Cargo.toml @@ -30,4 +30,4 @@ default = [] tonic-reflection = ["dep:tonic-reflection"] [dev-dependencies] -test-case = "3.3.1" +rstest = "0.19.0" diff --git a/tvix/build/src/buildservice/from_addr.rs b/tvix/build/src/buildservice/from_addr.rs index f5c4e6a490..cc5403edef 100644 --- a/tvix/build/src/buildservice/from_addr.rs +++ b/tvix/build/src/buildservice/from_addr.rs @@ -50,38 +50,41 @@ mod tests { use std::sync::Arc; use super::from_addr; - use test_case::test_case; + use rstest::rstest; use tvix_castore::{ blobservice::{BlobService, MemoryBlobService}, directoryservice::{DirectoryService, MemoryDirectoryService}, }; + #[rstest] /// This uses an unsupported scheme. - #[test_case("http://foo.example/test", false; "unsupported scheme")] + #[case::unsupported_scheme("http://foo.example/test", false)] /// This configures dummy - #[test_case("dummy://", true; "valid dummy")] + #[case::valid_dummy("dummy://", true)] /// Correct scheme to connect to a unix socket. - #[test_case("grpc+unix:///path/to/somewhere", true; "grpc valid unix socket")] + #[case::grpc_valid_unix_socket("grpc+unix:///path/to/somewhere", true)] /// Correct scheme for unix socket, but setting a host too, which is invalid. - #[test_case("grpc+unix://host.example/path/to/somewhere", false; "grpc invalid unix socket and host")] + #[case::grpc_invalid_unix_socket_and_host("grpc+unix://host.example/path/to/somewhere", false)] /// Correct scheme to connect to localhost, with port 12345 - #[test_case("grpc+http://[::1]:12345", true; "grpc valid IPv6 localhost port 12345")] + #[case::grpc_valid_ipv6_localhost_port_12345("grpc+http://[::1]:12345", true)] /// Correct scheme to connect to localhost over http, without specifying a port. - #[test_case("grpc+http://localhost", true; "grpc valid http host without port")] + #[case::grpc_valid_http_host_without_port("grpc+http://localhost", true)] /// Correct scheme to connect to localhost over http, without specifying a port. - #[test_case("grpc+https://localhost", true; "grpc valid https host without port")] + #[case::grpc_valid_https_host_without_port("grpc+https://localhost", true)] /// Correct scheme to connect to localhost over http, but with additional path, which is invalid. - #[test_case("grpc+http://localhost/some-path", false; "grpc valid invalid host and path")] + #[case::grpc_invalid_host_and_path("grpc+http://localhost/some-path", false)] #[tokio::test] - async fn test_from_addr(uri_str: &str, is_ok: bool) { + async fn test_from_addr(#[case] uri_str: &str, #[case] exp_succeed: bool) { let blob_service: Arc<dyn BlobService> = Arc::from(MemoryBlobService::default()); let directory_service: Arc<dyn DirectoryService> = Arc::from(MemoryDirectoryService::default()); - assert_eq!( - from_addr(uri_str, blob_service, directory_service) - .await - .is_ok(), - is_ok - ) + + let resp = from_addr(uri_str, blob_service, directory_service).await; + + if exp_succeed { + resp.expect("should succeed"); + } else { + assert!(resp.is_err(), "should fail"); + } } } diff --git a/tvix/build/src/proto/mod.rs b/tvix/build/src/proto/mod.rs index c7831795c3..e359b5b5b7 100644 --- a/tvix/build/src/proto/mod.rs +++ b/tvix/build/src/proto/mod.rs @@ -236,28 +236,27 @@ impl build_request::BuildConstraints { #[cfg(test)] mod tests { - use test_case::test_case; - - use crate::proto::is_clean_relative_path; - - use super::is_clean_path; - - #[test_case("foo/bar/", false; "fail trailing slash")] - #[test_case("foo/../bar", false; "fail dotdot")] - #[test_case("foo/./bar", false; "fail singledot")] - #[test_case("foo//bar", false; "fail unnecessary slashes")] - #[test_case("//foo/bar", false; "fail absolute unnecessary slashes")] - #[test_case("", true; "ok empty")] - #[test_case("foo/bar", true; "ok relative")] - #[test_case("/", true; "ok absolute")] - #[test_case("/foo/bar", true; "ok absolute2")] - fn test_is_clean_path(s: &str, expected: bool) { + use super::{is_clean_path, is_clean_relative_path}; + use rstest::rstest; + + #[rstest] + #[case::fail_trailing_slash("foo/bar/", false)] + #[case::fail_dotdot("foo/../bar", false)] + #[case::fail_singledot("foo/./bar", false)] + #[case::fail_unnecessary_slashes("foo//bar", false)] + #[case::fail_absolute_unnecessary_slashes("//foo/bar", false)] + #[case::ok_empty("", true)] + #[case::ok_relative("foo/bar", true)] + #[case::ok_absolute("/", true)] + #[case::ok_absolute2("/foo/bar", true)] + fn test_is_clean_path(#[case] s: &str, #[case] expected: bool) { assert_eq!(is_clean_path(s), expected); } - #[test_case("/", false; "fail absolute")] - #[test_case("foo/bar", true; "ok relative")] - fn test_is_clean_relative_path(s: &str, expected: bool) { + #[rstest] + #[case::fail_absolute("/", false)] + #[case::ok_relative("foo/bar", true)] + fn test_is_clean_relative_path(#[case] s: &str, #[case] expected: bool) { assert_eq!(is_clean_relative_path(s), expected); } diff --git a/tvix/buildkite.yml b/tvix/buildkite.yml index 4001ba9b9c..2a24ca4264 100644 --- a/tvix/buildkite.yml +++ b/tvix/buildkite.yml @@ -7,3 +7,4 @@ steps: - label: ":crab: cargo build" command: | nix-shell --run "cargo build && cargo test" + timeout_in_minutes: 10 diff --git a/tvix/castore/Cargo.toml b/tvix/castore/Cargo.toml index 274eacdb58..2797ef08f2 100644 --- a/tvix/castore/Cargo.toml +++ b/tvix/castore/Cargo.toml @@ -22,6 +22,7 @@ sled = { version = "0.34.7" } thiserror = "1.0.38" tokio-stream = { version = "0.1.14", features = ["fs", "net"] } tokio-util = { version = "0.7.9", features = ["io", "io-util"] } +tokio-tar = "0.3.1" tokio = { version = "1.32.0", features = ["fs", "macros", "net", "rt", "rt-multi-thread", "signal"] } tonic = "0.11.0" tower = "0.4.13" @@ -29,6 +30,16 @@ tracing = "0.1.37" url = "2.4.0" walkdir = "2.4.0" zstd = "0.13.0" +serde = { version = "1.0.197", features = [ "derive" ] } +serde_with = "3.7.0" +serde_qs = "0.12.0" +petgraph = "0.6.4" + +[dependencies.bigtable_rs] +optional = true +# https://github.com/liufuyang/bigtable_rs/pull/72 +git = "https://github.com/flokli/bigtable_rs" +rev = "0af404741dfc40eb9fa99cf4d4140a09c5c20df7" [dependencies.fuse-backend-rs] optional = true @@ -71,16 +82,18 @@ prost-build = "0.12.1" tonic-build = "0.11.0" [dev-dependencies] -rstest = "0.18.2" -test-case = "3.3.1" +async-process = "2.1.0" +rstest = "0.19.0" tempfile = "3.3.0" tokio-retry = "0.3.0" hex-literal = "0.4.1" rstest_reuse = "0.6.0" +xattr = "1.3.1" [features] default = [] cloud = [ + "dep:bigtable_rs", "object_store/aws", "object_store/azure", "object_store/gcp", @@ -99,3 +112,7 @@ virtiofs = [ ] fuse = ["fs"] tonic-reflection = ["dep:tonic-reflection"] +# Whether to run the integration tests. +# Requires the following packages in $PATH: +# cbtemulator, google-cloud-bigtable-tool +integration = [] diff --git a/tvix/castore/default.nix b/tvix/castore/default.nix index efa4da59c6..641d883760 100644 --- a/tvix/castore/default.nix +++ b/tvix/castore/default.nix @@ -1,8 +1,23 @@ { depot, pkgs, ... }: -depot.tvix.crates.workspaceMembers.tvix-castore.build.override { +(depot.tvix.crates.workspaceMembers.tvix-castore.build.override { runTests = true; testPreRun = '' export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt; ''; -} + + # enable some optional features. + features = [ "default" "cloud" ]; +}).overrideAttrs (_: { + meta.ci.targets = [ "integration-tests" ]; + passthru.integration-tests = depot.tvix.crates.workspaceMembers.tvix-castore.build.override { + runTests = true; + testPreRun = '' + export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt; + export PATH="$PATH:${pkgs.lib.makeBinPath [pkgs.cbtemulator pkgs.google-cloud-bigtable-tool]}" + ''; + + # enable some optional features. + features = [ "default" "cloud" "integration" ]; + }; +}) diff --git a/tvix/castore/src/blobservice/chunked_reader.rs b/tvix/castore/src/blobservice/chunked_reader.rs index e2a9f614c4..6e8355874b 100644 --- a/tvix/castore/src/blobservice/chunked_reader.rs +++ b/tvix/castore/src/blobservice/chunked_reader.rs @@ -1,12 +1,12 @@ -use futures::TryStreamExt; +use futures::{ready, TryStreamExt}; use pin_project_lite::pin_project; use tokio::io::{AsyncRead, AsyncSeekExt}; use tokio_stream::StreamExt; use tokio_util::io::{ReaderStream, StreamReader}; -use tracing::warn; +use tracing::{instrument, trace, warn}; use crate::B3Digest; -use std::{cmp::Ordering, pin::Pin, task::Poll}; +use std::{cmp::Ordering, pin::Pin}; use super::{BlobReader, BlobService}; @@ -60,15 +60,12 @@ where let filled_before = buf.filled().len(); let this = self.project(); - match this.r.poll_read(cx, buf) { - Poll::Ready(a) => { - let bytes_read = buf.filled().len() - filled_before; - *this.pos += bytes_read as u64; - Poll::Ready(a) - } - Poll::Pending => Poll::Pending, - } + ready!(this.r.poll_read(cx, buf))?; + let bytes_read = buf.filled().len() - filled_before; + *this.pos += bytes_read as u64; + + Ok(()).into() } } @@ -76,14 +73,12 @@ impl<BS> tokio::io::AsyncSeek for ChunkedReader<BS> where BS: AsRef<dyn BlobService> + Clone + Send + 'static, { + #[instrument(skip(self), err(Debug))] fn start_seek(self: Pin<&mut Self>, position: std::io::SeekFrom) -> std::io::Result<()> { let total_len = self.chunked_blob.blob_length(); - let current_pos = self.pos; - let this = self.project(); - let pos: &mut u64 = this.pos; - let mut r: Pin<&mut Box<dyn AsyncRead + Send + Unpin>> = this.r; + let mut this = self.project(); - let new_position: u64 = match position { + let absolute_offset: u64 = match position { std::io::SeekFrom::Start(from_start) => from_start, std::io::SeekFrom::End(from_end) => { // note from_end is i64, not u64, so this is usually negative. @@ -96,7 +91,7 @@ where } std::io::SeekFrom::Current(from_current) => { // note from_end is i64, not u64, so this can be positive or negative. - current_pos + (*this.pos) .checked_add_signed(from_current) .ok_or_else(|| { std::io::Error::new( @@ -107,17 +102,23 @@ where } }; - // ensure the new position still is inside the file. - if new_position > total_len { - Err(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - "seeked beyond EOF", - ))? - } + // check if the position actually did change. + if absolute_offset != *this.pos { + // ensure the new position still is inside the file. + if absolute_offset > total_len { + Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "seeked beyond EOF", + ))? + } - // Update the position and the internal reader. - *pos = new_position; - *r = this.chunked_blob.reader_skipped_offset(new_position); + // Update the position and the internal reader. + *this.pos = absolute_offset; + + // FUTUREWORK: if we can seek forward, avoid re-assembling. + // At least if it's still in the same chunk? + *this.r = this.chunked_blob.reader_skipped_offset(absolute_offset); + } Ok(()) } @@ -176,6 +177,7 @@ where /// For a given position pos, return the chunk containing the data. /// In case this would range outside the blob, None is returned. + #[instrument(level = "trace", skip(self), ret)] fn get_chunk_idx_for_position(&self, pos: u64) -> Option<usize> { // FUTUREWORK: benchmark when to use linear search, binary_search and BTreeSet self.chunks @@ -197,6 +199,7 @@ where /// From the first relevant chunk, the irrelevant bytes are skipped too. /// The returned boxed thing does not implement AsyncSeek on its own, but /// ChunkedReader does. + #[instrument(level = "trace", skip(self))] fn reader_skipped_offset(&self, offset: u64) -> Box<dyn tokio::io::AsyncRead + Send + Unpin> { if offset == self.blob_length() { return Box::new(std::io::Cursor::new(vec![])); @@ -211,11 +214,12 @@ where let blob_service = self.blob_service.clone(); let chunks: Vec<_> = self.chunks[start_chunk_idx..].to_vec(); - let readers_stream = tokio_stream::iter(chunks).map( - move |(_chunk_start_offset, _chunk_size, chunk_digest)| { + let readers_stream = tokio_stream::iter(chunks.into_iter().enumerate()).map( + move |(nth_chunk, (_chunk_start_offset, chunk_size, chunk_digest))| { let chunk_digest = chunk_digest.to_owned(); let blob_service = blob_service.clone(); async move { + trace!(chunk_size=%chunk_size, chunk_digest=%chunk_digest, "open_read on chunk in stream"); let mut blob_reader = blob_service .as_ref() .open_read(&chunk_digest.to_owned()) @@ -225,7 +229,8 @@ where std::io::Error::new(std::io::ErrorKind::NotFound, "chunk not found") })?; - if skip_first_chunk_bytes > 0 { + // iff this is the first chunk in the stream, skip by skip_first_chunk_bytes + if nth_chunk == 0 && skip_first_chunk_bytes > 0 { blob_reader .seek(std::io::SeekFrom::Start(skip_first_chunk_bytes as u64)) .await?; diff --git a/tvix/castore/src/blobservice/from_addr.rs b/tvix/castore/src/blobservice/from_addr.rs index b7ffc48e23..3e3f943e59 100644 --- a/tvix/castore/src/blobservice/from_addr.rs +++ b/tvix/castore/src/blobservice/from_addr.rs @@ -84,68 +84,65 @@ pub async fn from_addr(uri: &str) -> Result<Box<dyn BlobService>, crate::Error> mod tests { use super::from_addr; use lazy_static::lazy_static; + use rstest::rstest; use tempfile::TempDir; - use test_case::test_case; lazy_static! { static ref TMPDIR_SLED_1: TempDir = TempDir::new().unwrap(); static ref TMPDIR_SLED_2: TempDir = TempDir::new().unwrap(); } + #[rstest] /// This uses an unsupported scheme. - #[test_case("http://foo.example/test", false; "unsupported scheme")] + #[case::unsupported_scheme("http://foo.example/test", false)] /// This configures sled in temporary mode. - #[test_case("sled://", true; "sled valid temporary")] + #[case::sled_temporary("sled://", true)] /// This configures sled with /, which should fail. - #[test_case("sled:///", false; "sled invalid root")] + #[case::sled_invalid_root("sled:///", false)] /// This configures sled with a host, not path, which should fail. - #[test_case("sled://foo.example", false; "sled invalid host")] + #[case::sled_invalid_host("sled://foo.example", false)] /// This configures sled with a valid path path, which should succeed. - #[test_case(&format!("sled://{}", &TMPDIR_SLED_1.path().to_str().unwrap()), true; "sled valid path")] + #[case::sled_valid_path(&format!("sled://{}", &TMPDIR_SLED_1.path().to_str().unwrap()), true)] /// This configures sled with a host, and a valid path path, which should fail. - #[test_case(&format!("sled://foo.example{}", &TMPDIR_SLED_2.path().to_str().unwrap()), false; "sled invalid host with valid path")] + #[case::sled_invalid_host_with_valid_path(&format!("sled://foo.example{}", &TMPDIR_SLED_2.path().to_str().unwrap()), false)] /// This correctly sets the scheme, and doesn't set a path. - #[test_case("memory://", true; "memory valid")] + #[case::memory_valid("memory://", true)] /// This sets a memory url host to `foo` - #[test_case("memory://foo", false; "memory invalid host")] + #[case::memory_invalid_host("memory://foo", false)] /// This sets a memory url path to "/", which is invalid. - #[test_case("memory:///", false; "memory invalid root path")] + #[case::memory_invalid_root_path("memory:///", false)] /// This sets a memory url path to "/foo", which is invalid. - #[test_case("memory:///foo", false; "memory invalid root path foo")] + #[case::memory_invalid_root_path_foo("memory:///foo", false)] /// Correct scheme to connect to a unix socket. - #[test_case("grpc+unix:///path/to/somewhere", true; "grpc valid unix socket")] + #[case::grpc_valid_unix_socket("grpc+unix:///path/to/somewhere", true)] /// Correct scheme for unix socket, but setting a host too, which is invalid. - #[test_case("grpc+unix://host.example/path/to/somewhere", false; "grpc invalid unix socket and host")] + #[case::grpc_invalid_unix_socket_and_host("grpc+unix://host.example/path/to/somewhere", false)] /// Correct scheme to connect to localhost, with port 12345 - #[test_case("grpc+http://[::1]:12345", true; "grpc valid IPv6 localhost port 12345")] + #[case::grpc_valid_ipv6_localhost_port_12345("grpc+http://[::1]:12345", true)] /// Correct scheme to connect to localhost over http, without specifying a port. - #[test_case("grpc+http://localhost", true; "grpc valid http host without port")] + #[case::grpc_valid_http_host_without_port("grpc+http://localhost", true)] /// Correct scheme to connect to localhost over http, without specifying a port. - #[test_case("grpc+https://localhost", true; "grpc valid https host without port")] + #[case::grpc_valid_https_host_without_port("grpc+https://localhost", true)] /// Correct scheme to connect to localhost over http, but with additional path, which is invalid. - #[test_case("grpc+http://localhost/some-path", false; "grpc invalid has path")] + #[case::grpc_invalid_has_path("grpc+http://localhost/some-path", false)] /// An example for object store (InMemory) - #[test_case("objectstore+memory:///", true; "objectstore valid memory url")] + #[case::objectstore_valid_memory("objectstore+memory:///", true)] /// An example for object store (LocalFileSystem) - #[test_case("objectstore+file:///foo/bar", true; "objectstore valid file url")] + #[case::objectstore_valid_file("objectstore+file:///foo/bar", true)] // An example for object store (HTTP / WebDAV) - #[test_case("objectstore+https://localhost:8080/some-path", true; "objectstore valid http url")] - #[tokio::test] - async fn test_from_addr_tokio(uri_str: &str, exp_succeed: bool) { - if exp_succeed { - from_addr(uri_str).await.expect("should succeed"); - } else { - assert!(from_addr(uri_str).await.is_err(), "should fail"); - } - } - - #[cfg(feature = "cloud")] + #[case::objectstore_valid_http_url("objectstore+https://localhost:8080/some-path", true)] /// An example for object store (S3) - #[test_case("objectstore+s3://bucket/path", true; "objectstore valid s3 url")] + #[cfg_attr( + feature = "cloud", + case::objectstore_valid_s3_url("objectstore+s3://bucket/path", true) + )] /// An example for object store (GCS) - #[test_case("objectstore+gs://bucket/path", true; "objectstore valid gcs url")] + #[cfg_attr( + feature = "cloud", + case::objectstore_valid_gcs_url("objectstore+gs://bucket/path", true) + )] #[tokio::test] - async fn test_from_addr_tokio_cloud(uri_str: &str, exp_succeed: bool) { + async fn test_from_addr_tokio(#[case] uri_str: &str, #[case] exp_succeed: bool) { if exp_succeed { from_addr(uri_str).await.expect("should succeed"); } else { diff --git a/tvix/castore/src/blobservice/grpc.rs b/tvix/castore/src/blobservice/grpc.rs index 9230abaf34..5663cd3838 100644 --- a/tvix/castore/src/blobservice/grpc.rs +++ b/tvix/castore/src/blobservice/grpc.rs @@ -1,10 +1,15 @@ -use super::{naive_seeker::NaiveSeeker, BlobReader, BlobService, BlobWriter, ChunkedReader}; +use super::{BlobReader, BlobService, BlobWriter, ChunkedReader}; use crate::{ proto::{self, stat_blob_response::ChunkMeta}, B3Digest, }; use futures::sink::SinkExt; -use std::{io, pin::pin, sync::Arc, task::Poll}; +use std::{ + io::{self, Cursor}, + pin::pin, + sync::Arc, + task::Poll, +}; use tokio::io::AsyncWriteExt; use tokio::task::JoinHandle; use tokio_stream::{wrappers::ReceiverStream, StreamExt}; @@ -13,7 +18,7 @@ use tokio_util::{ sync::PollSender, }; use tonic::{async_trait, transport::Channel, Code, Status}; -use tracing::{instrument, warn}; +use tracing::instrument; /// Connects to a (remote) tvix-store BlobService over gRPC. #[derive(Clone)] @@ -55,11 +60,10 @@ impl BlobService for GRPCBlobService { #[instrument(skip(self, digest), fields(blob.digest=%digest), err)] async fn open_read(&self, digest: &B3Digest) -> io::Result<Option<Box<dyn BlobReader>>> { // First try to get a list of chunks. In case there's only one chunk returned, - // or the backend does not support chunking, return a NaiveSeeker. - // Otherwise use a ChunkedReader. - // TODO: we should check if we want to replace NaiveSeeker with a simple - // Cursor on the raw `Vec<u8>`, as seeking backwards is something that - // clients generally do. + // buffer its data into a Vec, otherwise use a ChunkedReader. + // We previously used NaiveSeeker here, but userland likes to seek backwards too often, + // and without store composition this will get very noisy. + // FUTUREWORK: use CombinedBlobService and store composition. match self.chunks(digest).await { Ok(None) => Ok(None), Ok(Some(chunks)) => { @@ -82,9 +86,13 @@ impl BlobService for GRPCBlobService { }); // Use StreamReader::new to convert to an AsyncRead. - let data_reader = tokio_util::io::StreamReader::new(data_stream); + let mut data_reader = tokio_util::io::StreamReader::new(data_stream); + + let mut buf = Vec::new(); + // TODO: only do this up to a certain limit. + tokio::io::copy(&mut data_reader, &mut buf).await?; - Ok(Some(Box::new(NaiveSeeker::new(data_reader)))) + Ok(Some(Box::new(Cursor::new(buf)))) } Err(e) if e.code() == Code::NotFound => Ok(None), Err(e) => Err(io::Error::new(io::ErrorKind::Other, e)), @@ -161,9 +169,6 @@ impl BlobService for GRPCBlobService { resp.validate() .map_err(|e| std::io::Error::new(io::ErrorKind::InvalidData, e))?; - if resp.chunks.is_empty() { - warn!("chunk list is empty"); - } Ok(Some(resp.chunks)) } } diff --git a/tvix/castore/src/blobservice/naive_seeker.rs b/tvix/castore/src/blobservice/naive_seeker.rs index e65a82c7f4..f5a5307150 100644 --- a/tvix/castore/src/blobservice/naive_seeker.rs +++ b/tvix/castore/src/blobservice/naive_seeker.rs @@ -1,9 +1,10 @@ use super::BlobReader; +use futures::ready; use pin_project_lite::pin_project; use std::io; use std::task::Poll; use tokio::io::AsyncRead; -use tracing::{debug, instrument}; +use tracing::{debug, instrument, trace, warn}; pin_project! { /// This implements [tokio::io::AsyncSeek] for and [tokio::io::AsyncRead] by @@ -64,6 +65,9 @@ pin_project! { } } +/// The buffer size used to discard data. +const DISCARD_BUF_SIZE: usize = 4096; + impl<R: tokio::io::AsyncRead> NaiveSeeker<R> { pub fn new(r: R) -> Self { NaiveSeeker { @@ -75,6 +79,7 @@ impl<R: tokio::io::AsyncRead> NaiveSeeker<R> { } impl<R: tokio::io::AsyncRead> tokio::io::AsyncRead for NaiveSeeker<R> { + #[instrument(level = "trace", skip_all)] fn poll_read( self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>, @@ -83,18 +88,16 @@ impl<R: tokio::io::AsyncRead> tokio::io::AsyncRead for NaiveSeeker<R> { // The amount of data read can be determined by the increase // in the length of the slice returned by `ReadBuf::filled`. let filled_before = buf.filled().len(); + let this = self.project(); - let pos: &mut u64 = this.pos; + ready!(this.r.poll_read(cx, buf))?; - match this.r.poll_read(cx, buf) { - Poll::Ready(a) => { - let bytes_read = buf.filled().len() - filled_before; - *pos += bytes_read as u64; + let bytes_read = buf.filled().len() - filled_before; + *this.pos += bytes_read as u64; - Poll::Ready(a) - } - Poll::Pending => Poll::Pending, - } + trace!(bytes_read = bytes_read, new_pos = this.pos, "poll_read"); + + Ok(()).into() } } @@ -106,16 +109,18 @@ impl<R: tokio::io::AsyncRead> tokio::io::AsyncBufRead for NaiveSeeker<R> { self.project().r.poll_fill_buf(cx) } + #[instrument(level = "trace", skip(self))] fn consume(self: std::pin::Pin<&mut Self>, amt: usize) { let this = self.project(); this.r.consume(amt); - let pos: &mut u64 = this.pos; - *pos += amt as u64; + *this.pos += amt as u64; + + trace!(new_pos = this.pos, "consume"); } } impl<R: tokio::io::AsyncRead> tokio::io::AsyncSeek for NaiveSeeker<R> { - #[instrument(skip(self))] + #[instrument(level="trace", skip(self), fields(inner_pos=%self.pos), err(Debug))] fn start_seek( self: std::pin::Pin<&mut Self>, position: std::io::SeekFrom, @@ -150,23 +155,24 @@ impl<R: tokio::io::AsyncRead> tokio::io::AsyncSeek for NaiveSeeker<R> { } }; - debug!(absolute_offset=?absolute_offset, "seek"); - - // we already know absolute_offset is larger than self.pos + // we already know absolute_offset is >= self.pos debug_assert!( absolute_offset >= self.pos, - "absolute_offset {} is larger than self.pos {}", + "absolute_offset {} must be >= self.pos {}", absolute_offset, self.pos ); // calculate bytes to skip - *self.project().bytes_to_skip = absolute_offset - self.pos; + let this = self.project(); + *this.bytes_to_skip = absolute_offset - *this.pos; + + debug!(bytes_to_skip = *this.bytes_to_skip, "seek"); Ok(()) } - #[instrument(skip(self))] + #[instrument(skip_all)] fn poll_complete( mut self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>, @@ -178,44 +184,34 @@ impl<R: tokio::io::AsyncRead> tokio::io::AsyncSeek for NaiveSeeker<R> { // discard some bytes, until pos is where we want it to be. // We create a buffer that we'll discard later on. - let mut buf = [0; 1024]; + let mut discard_buf = [0; DISCARD_BUF_SIZE]; // Loop until we've reached the desired seek position. This is done by issuing repeated - // `poll_read` calls. If the data is not available yet, we will yield back to the executor + // `poll_read` calls. + // If the data is not available yet, we will yield back to the executor // and wait to be polled again. loop { + if self.bytes_to_skip == 0 { + return Poll::Ready(Ok(self.pos)); + } + // calculate the length we want to skip at most, which is either a max // buffer size, or the number of remaining bytes to read, whatever is // smaller. - let bytes_to_skip = std::cmp::min(self.bytes_to_skip as usize, buf.len()); - - let mut read_buf = tokio::io::ReadBuf::new(&mut buf[..bytes_to_skip]); - - match self.as_mut().poll_read(cx, &mut read_buf) { - Poll::Ready(_a) => { - let bytes_read = read_buf.filled().len() as u64; + let bytes_to_skip_now = std::cmp::min(self.bytes_to_skip as usize, discard_buf.len()); + let mut discard_buf = tokio::io::ReadBuf::new(&mut discard_buf[..bytes_to_skip_now]); - if bytes_read == 0 { - return Poll::Ready(Err(io::Error::new( - io::ErrorKind::UnexpectedEof, - format!( - "tried to skip {} bytes, but only was able to skip {} until reaching EOF", - bytes_to_skip, bytes_read - ), - ))); - } + ready!(self.as_mut().poll_read(cx, &mut discard_buf))?; + let bytes_skipped = discard_buf.filled().len(); - // calculate bytes to skip - let bytes_to_skip = self.bytes_to_skip - bytes_read; - - *self.as_mut().project().bytes_to_skip = bytes_to_skip; - - if bytes_to_skip == 0 { - return Poll::Ready(Ok(self.pos)); - } - } - Poll::Pending => return Poll::Pending, - }; + if bytes_skipped == 0 { + return Poll::Ready(Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "got EOF while trying to skip bytes", + ))); + } + // decrement bytes to skip. The poll_read call already updated self.pos. + *self.as_mut().project().bytes_to_skip -= bytes_skipped as u64; } } } @@ -224,16 +220,16 @@ impl<R: tokio::io::AsyncRead + Send + Unpin + 'static> BlobReader for NaiveSeeke #[cfg(test)] mod tests { - use super::NaiveSeeker; + use super::{NaiveSeeker, DISCARD_BUF_SIZE}; use std::io::{Cursor, SeekFrom}; use tokio::io::{AsyncReadExt, AsyncSeekExt}; - /// This seek requires multiple `poll_read` as we use a 1024 bytes internal - /// buffer when doing the seek. + /// This seek requires multiple `poll_read` as we use a multiples of + /// DISCARD_BUF_SIZE when doing the seek. /// This ensures we don't hang indefinitely. #[tokio::test] async fn seek() { - let buf = vec![0u8; 4096]; + let buf = vec![0u8; DISCARD_BUF_SIZE * 4]; let reader = Cursor::new(&buf); let mut seeker = NaiveSeeker::new(reader); seeker.seek(SeekFrom::Start(4000)).await.unwrap(); @@ -241,29 +237,29 @@ mod tests { #[tokio::test] async fn seek_read() { - let mut buf = vec![0u8; 2048]; - buf.extend_from_slice(&[1u8; 2048]); - buf.extend_from_slice(&[2u8; 2048]); + let mut buf = vec![0u8; DISCARD_BUF_SIZE * 2]; + buf.extend_from_slice(&[1u8; DISCARD_BUF_SIZE * 2]); + buf.extend_from_slice(&[2u8; DISCARD_BUF_SIZE * 2]); let reader = Cursor::new(&buf); let mut seeker = NaiveSeeker::new(reader); - let mut read_buf = vec![0u8; 1024]; + let mut read_buf = vec![0u8; DISCARD_BUF_SIZE]; seeker.read_exact(&mut read_buf).await.expect("must read"); - assert_eq!(read_buf.as_slice(), &[0u8; 1024]); + assert_eq!(read_buf.as_slice(), &[0u8; DISCARD_BUF_SIZE]); seeker - .seek(SeekFrom::Current(1024)) + .seek(SeekFrom::Current(DISCARD_BUF_SIZE as i64)) .await .expect("must seek"); seeker.read_exact(&mut read_buf).await.expect("must read"); - assert_eq!(read_buf.as_slice(), &[1u8; 1024]); + assert_eq!(read_buf.as_slice(), &[1u8; DISCARD_BUF_SIZE]); seeker - .seek(SeekFrom::Start(2 * 2048)) + .seek(SeekFrom::Start(2 * 2 * DISCARD_BUF_SIZE as u64)) .await .expect("must seek"); seeker.read_exact(&mut read_buf).await.expect("must read"); - assert_eq!(read_buf.as_slice(), &[2u8; 1024]); + assert_eq!(read_buf.as_slice(), &[2u8; DISCARD_BUF_SIZE]); } } diff --git a/tvix/castore/src/blobservice/object_store.rs b/tvix/castore/src/blobservice/object_store.rs index 0dba19ef54..d2d0a288a5 100644 --- a/tvix/castore/src/blobservice/object_store.rs +++ b/tvix/castore/src/blobservice/object_store.rs @@ -14,7 +14,7 @@ use prost::Message; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use tokio_stream::StreamExt; use tonic::async_trait; -use tracing::{debug, info, instrument, trace, Level}; +use tracing::{debug, instrument, trace, Level}; use url::Url; use crate::{ @@ -125,7 +125,14 @@ impl BlobService for ObjectStoreBlobService { match self.object_store.head(&p).await { Ok(_) => Ok(true), - Err(object_store::Error::NotFound { .. }) => Ok(false), + Err(object_store::Error::NotFound { .. }) => { + let p = derive_chunk_path(&self.base_path, digest); + match self.object_store.head(&p).await { + Ok(_) => Ok(true), + Err(object_store::Error::NotFound { .. }) => Ok(false), + Err(e) => Err(e)?, + } + } Err(e) => Err(e)?, } } @@ -142,6 +149,7 @@ impl BlobService for ObjectStoreBlobService { .await { Ok(res) => { + // handle reading blobs that are small enough to fit inside a single chunk: // fetch the entire chunk into memory, decompress, ensure the b3 digest matches, // and return a io::Cursor over that data. // FUTUREWORK: use zstd::bulk to prevent decompression bombs @@ -211,18 +219,51 @@ impl BlobService for ObjectStoreBlobService { #[instrument(skip_all, err, fields(blob.digest=%digest))] async fn chunks(&self, digest: &B3Digest) -> io::Result<Option<Vec<ChunkMeta>>> { - let p = derive_blob_path(&self.base_path, digest); - - match self.object_store.get(&p).await { + match self + .object_store + .get(&derive_blob_path(&self.base_path, digest)) + .await + { Ok(get_result) => { // fetch the data at the blob path let blob_data = get_result.bytes().await?; // parse into StatBlobResponse let stat_blob_response: StatBlobResponse = StatBlobResponse::decode(blob_data)?; + debug!( + chunk.count = stat_blob_response.chunks.len(), + blob.size = stat_blob_response + .chunks + .iter() + .map(|x| x.size) + .sum::<u64>(), + "found more granular chunks" + ); + Ok(Some(stat_blob_response.chunks)) } - Err(object_store::Error::NotFound { .. }) => Ok(None), + Err(object_store::Error::NotFound { .. }) => { + // If there's only a chunk, we must return the empty vec here, rather than None. + match self + .object_store + .head(&derive_chunk_path(&self.base_path, digest)) + .await + { + Ok(_) => { + // present, but no more chunks available + debug!("found a single chunk"); + Ok(Some(vec![])) + } + Err(object_store::Error::NotFound { .. }) => { + // Neither blob nor single chunk found + debug!("not found"); + Ok(None) + } + // error checking for chunk + Err(e) => Err(e.into()), + } + } + // error checking for blob Err(err) => Err(err.into()), } } @@ -318,7 +359,7 @@ async fn upload_chunk( match object_store.head(&chunk_path).await { // chunk already exists, nothing to do Ok(_) => { - info!("chunk already exists"); + debug!("chunk already exists"); } // chunk does not yet exist, compress and upload. @@ -326,7 +367,7 @@ async fn upload_chunk( let chunk_data_compressed = zstd::encode_all(Cursor::new(chunk_data), zstd::DEFAULT_COMPRESSION_LEVEL)?; - info!(chunk.compressed_size=%chunk_data_compressed.len(), "uploading chunk"); + debug!(chunk.compressed_size=%chunk_data_compressed.len(), "uploading chunk"); object_store .as_ref() @@ -471,67 +512,13 @@ where #[cfg(test)] mod test { - use std::{io::Cursor, sync::Arc}; - - use test_case::test_case; - use url::Url; - use super::chunk_and_upload; use crate::{ blobservice::{BlobService, ObjectStoreBlobService}, - fixtures::{BLOB_A, BLOB_A_DIGEST, BLOB_B, BLOB_B_DIGEST, EMPTY_BLOB_DIGEST}, - B3Digest, + fixtures::{BLOB_A, BLOB_A_DIGEST}, }; - - #[test_case(&BLOB_A, &BLOB_A_DIGEST; "blob a")] - #[test_case(&BLOB_B, &BLOB_B_DIGEST; "blob b")] - #[test_case(&[], &EMPTY_BLOB_DIGEST; "empty blob")] - #[tokio::test] - async fn upload_blob(fix_blob_contents: &[u8], fix_blob_digest: &B3Digest) { - let blobsvc = - ObjectStoreBlobService::parse_url(&Url::parse("memory:///").unwrap()).unwrap(); - - // Initially, the blob should not be present - assert!(!blobsvc.has(fix_blob_digest).await.unwrap()); - - // Using the open_read should return a Ok(None). - // As for the empty blob, we're ok with it actually returning data too. - { - let resp = blobsvc - .open_read(fix_blob_digest) - .await - .expect("open should not fail"); - if fix_blob_digest != &*EMPTY_BLOB_DIGEST { - assert!(resp.is_none()); - } - } - - // upload blob - let mut bw = blobsvc.open_write().await; - tokio::io::copy(&mut Cursor::new(fix_blob_contents), &mut bw) - .await - .expect("copy succeeds"); - - let blob_digest = bw.close().await.expect("close succeeds"); - assert_eq!(fix_blob_digest, &blob_digest); - - // blob should be present now - assert!(blobsvc.has(fix_blob_digest).await.unwrap()); - - // reading it should return the same data. - let mut br = blobsvc - .open_read(fix_blob_digest) - .await - .expect("open succeeds") - .expect("is some"); - - let mut buf = Vec::new(); - tokio::io::copy(&mut br, &mut buf) - .await - .expect("copy must succeed"); - - assert_eq!(fix_blob_contents, buf, "read data should match"); - } + use std::{io::Cursor, sync::Arc}; + use url::Url; /// Tests chunk_and_upload directly, bypassing the BlobWriter at open_write(). #[tokio::test] diff --git a/tvix/castore/src/blobservice/tests/mod.rs b/tvix/castore/src/blobservice/tests/mod.rs index b1ae66bac4..30c4e97634 100644 --- a/tvix/castore/src/blobservice/tests/mod.rs +++ b/tvix/castore/src/blobservice/tests/mod.rs @@ -22,8 +22,9 @@ use self::utils::make_grpc_blob_service_client; /// See https://github.com/la10736/rstest/issues/130#issuecomment-968864832 #[template] #[rstest] -#[case::memory(blobservice::from_addr("memory://").await.unwrap())] #[case::grpc(make_grpc_blob_service_client().await)] +#[case::memory(blobservice::from_addr("memory://").await.unwrap())] +#[case::objectstore_memory(blobservice::from_addr("objectstore+memory://").await.unwrap())] #[case::sled(blobservice::from_addr("sled://").await.unwrap())] pub fn blob_services(#[case] blob_service: impl BlobService) {} diff --git a/tvix/castore/src/directoryservice/bigtable.rs b/tvix/castore/src/directoryservice/bigtable.rs new file mode 100644 index 0000000000..0fdb24628f --- /dev/null +++ b/tvix/castore/src/directoryservice/bigtable.rs @@ -0,0 +1,357 @@ +use bigtable_rs::{bigtable, google::bigtable::v2 as bigtable_v2}; +use bytes::Bytes; +use data_encoding::HEXLOWER; +use futures::stream::BoxStream; +use prost::Message; +use serde::{Deserialize, Serialize}; +use serde_with::{serde_as, DurationSeconds}; +use tonic::async_trait; +use tracing::{instrument, trace, warn}; + +use super::{utils::traverse_directory, DirectoryPutter, DirectoryService, SimplePutter}; +use crate::{proto, B3Digest, Error}; + +/// There should not be more than 10 MiB in a single cell. +/// https://cloud.google.com/bigtable/docs/schema-design#cells +const CELL_SIZE_LIMIT: u64 = 10 * 1024 * 1024; + +/// Provides a [DirectoryService] implementation using +/// [Bigtable](https://cloud.google.com/bigtable/docs/) +/// as an underlying K/V store. +/// +/// # Data format +/// We use Bigtable as a plain K/V store. +/// The row key is the digest of the directory, in hexlower. +/// Inside the row, we currently have a single column/cell, again using the +/// hexlower directory digest. +/// Its value is the Directory message, serialized in canonical protobuf. +/// We currently only populate this column. +/// +/// In the future, we might want to introduce "bucketing", essentially storing +/// all directories inserted via `put_multiple_start` in a batched form. +/// This will prevent looking up intermediate Directories, which are not +/// directly at the root, so rely on store composition. +#[derive(Clone)] +pub struct BigtableDirectoryService { + client: bigtable::BigTable, + params: BigtableParameters, + + #[cfg(test)] + #[allow(dead_code)] + /// Holds the temporary directory containing the unix socket, and the + /// spawned emulator process. + emulator: std::sync::Arc<(tempfile::TempDir, async_process::Child)>, +} + +/// Represents configuration of [BigtableDirectoryService]. +/// This currently conflates both connect parameters and data model/client +/// behaviour parameters. +#[serde_as] +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] +pub struct BigtableParameters { + project_id: String, + instance_name: String, + #[serde(default)] + is_read_only: bool, + #[serde(default = "default_channel_size")] + channel_size: usize, + + #[serde_as(as = "Option<DurationSeconds<String>>")] + #[serde(default = "default_timeout")] + timeout: Option<std::time::Duration>, + table_name: String, + family_name: String, + + #[serde(default = "default_app_profile_id")] + app_profile_id: String, +} + +fn default_app_profile_id() -> String { + "default".to_owned() +} + +fn default_channel_size() -> usize { + 4 +} + +fn default_timeout() -> Option<std::time::Duration> { + Some(std::time::Duration::from_secs(4)) +} + +impl BigtableDirectoryService { + #[cfg(not(test))] + pub async fn connect(params: BigtableParameters) -> Result<Self, bigtable::Error> { + let connection = bigtable::BigTableConnection::new( + ¶ms.project_id, + ¶ms.instance_name, + params.is_read_only, + params.channel_size, + params.timeout, + ) + .await?; + + Ok(Self { + client: connection.client(), + params, + }) + } + + #[cfg(test)] + pub async fn connect(params: BigtableParameters) -> Result<Self, bigtable::Error> { + use std::time::Duration; + + use async_process::{Command, Stdio}; + use tempfile::TempDir; + use tokio_retry::{strategy::ExponentialBackoff, Retry}; + + let tmpdir = TempDir::new().unwrap(); + + let socket_path = tmpdir.path().join("cbtemulator.sock"); + + let emulator_process = Command::new("cbtemulator") + .arg("-address") + .arg(socket_path.clone()) + .stderr(Stdio::piped()) + .stdout(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .expect("failed to spawn emulator"); + + Retry::spawn( + ExponentialBackoff::from_millis(20) + .max_delay(Duration::from_secs(1)) + .take(3), + || async { + if socket_path.exists() { + Ok(()) + } else { + Err(()) + } + }, + ) + .await + .expect("failed to wait for socket"); + + // populate the emulator + for cmd in &[ + vec!["createtable", ¶ms.table_name], + vec!["createfamily", ¶ms.table_name, ¶ms.family_name], + ] { + Command::new("cbt") + .args({ + let mut args = vec![ + "-instance", + ¶ms.instance_name, + "-project", + ¶ms.project_id, + ]; + args.extend_from_slice(cmd); + args + }) + .env( + "BIGTABLE_EMULATOR_HOST", + format!("unix://{}", socket_path.to_string_lossy()), + ) + .output() + .await + .expect("failed to run cbt setup command"); + } + + let connection = bigtable_rs::bigtable::BigTableConnection::new_with_emulator( + &format!("unix://{}", socket_path.to_string_lossy()), + ¶ms.project_id, + ¶ms.instance_name, + params.is_read_only, + params.timeout, + )?; + + Ok(Self { + client: connection.client(), + params, + emulator: (tmpdir, emulator_process).into(), + }) + } +} + +/// Derives the row/column key for a given blake3 digest. +/// We use hexlower encoding, also because it can't be misinterpreted as RE2. +fn derive_directory_key(digest: &B3Digest) -> String { + HEXLOWER.encode(digest.as_slice()) +} + +#[async_trait] +impl DirectoryService for BigtableDirectoryService { + #[instrument(skip(self, digest), err, fields(directory.digest = %digest))] + async fn get(&self, digest: &B3Digest) -> Result<Option<proto::Directory>, Error> { + let mut client = self.client.clone(); + let directory_key = derive_directory_key(digest); + + let request = bigtable_v2::ReadRowsRequest { + app_profile_id: self.params.app_profile_id.to_string(), + table_name: client.get_full_table_name(&self.params.table_name), + rows_limit: 1, + rows: Some(bigtable_v2::RowSet { + row_keys: vec![directory_key.clone().into()], + row_ranges: vec![], + }), + // Filter selected family name, and column qualifier matching our digest. + // This is to ensure we don't fail once we start bucketing. + filter: Some(bigtable_v2::RowFilter { + filter: Some(bigtable_v2::row_filter::Filter::Chain( + bigtable_v2::row_filter::Chain { + filters: vec![ + bigtable_v2::RowFilter { + filter: Some( + bigtable_v2::row_filter::Filter::FamilyNameRegexFilter( + self.params.family_name.to_string(), + ), + ), + }, + bigtable_v2::RowFilter { + filter: Some( + bigtable_v2::row_filter::Filter::ColumnQualifierRegexFilter( + directory_key.clone().into(), + ), + ), + }, + ], + }, + )), + }), + ..Default::default() + }; + + let mut response = client + .read_rows(request) + .await + .map_err(|e| Error::StorageError(format!("unable to read rows: {}", e)))?; + + if response.len() != 1 { + if response.len() > 1 { + // This shouldn't happen, we limit number of rows to 1 + return Err(Error::StorageError( + "got more than one row from bigtable".into(), + )); + } + // else, this is simply a "not found". + return Ok(None); + } + + let (row_key, mut row_cells) = response.pop().unwrap(); + if row_key != directory_key.as_bytes() { + // This shouldn't happen, we requested this row key. + return Err(Error::StorageError( + "got wrong row key from bigtable".into(), + )); + } + + let row_cell = row_cells + .pop() + .ok_or_else(|| Error::StorageError("found no cells".into()))?; + + // Ensure there's only one cell (so no more left after the pop()) + // This shouldn't happen, We filter out other cells in our query. + if !row_cells.is_empty() { + return Err(Error::StorageError( + "more than one cell returned from bigtable".into(), + )); + } + + // We also require the qualifier to be correct in the filter above, + // so this shouldn't happen. + if directory_key.as_bytes() != row_cell.qualifier { + return Err(Error::StorageError("unexpected cell qualifier".into())); + } + + // For the data in that cell, ensure the digest matches what's requested, before parsing. + let got_digest = B3Digest::from(blake3::hash(&row_cell.value).as_bytes()); + if got_digest != *digest { + return Err(Error::StorageError(format!( + "invalid digest: {}", + got_digest + ))); + } + + // Try to parse the value into a Directory message. + let directory = proto::Directory::decode(Bytes::from(row_cell.value)) + .map_err(|e| Error::StorageError(format!("unable to decode directory proto: {}", e)))?; + + // validate the Directory. + directory + .validate() + .map_err(|e| Error::StorageError(format!("invalid Directory message: {}", e)))?; + + Ok(Some(directory)) + } + + #[instrument(skip(self, directory), err, fields(directory.digest = %directory.digest()))] + async fn put(&self, directory: proto::Directory) -> Result<B3Digest, Error> { + let directory_digest = directory.digest(); + let mut client = self.client.clone(); + let directory_key = derive_directory_key(&directory_digest); + + // Ensure the directory we're trying to upload passes validation + directory + .validate() + .map_err(|e| Error::InvalidRequest(format!("directory is invalid: {}", e)))?; + + let data = directory.encode_to_vec(); + if data.len() as u64 > CELL_SIZE_LIMIT { + return Err(Error::StorageError( + "Directory exceeds cell limit on Bigtable".into(), + )); + } + + let resp = client + .check_and_mutate_row(bigtable_v2::CheckAndMutateRowRequest { + table_name: client.get_full_table_name(&self.params.table_name), + app_profile_id: self.params.app_profile_id.to_string(), + row_key: directory_key.clone().into(), + predicate_filter: Some(bigtable_v2::RowFilter { + filter: Some(bigtable_v2::row_filter::Filter::ColumnQualifierRegexFilter( + directory_key.clone().into(), + )), + }), + // If the column was already found, do nothing. + true_mutations: vec![], + // Else, do the insert. + false_mutations: vec![ + // https://cloud.google.com/bigtable/docs/writes + bigtable_v2::Mutation { + mutation: Some(bigtable_v2::mutation::Mutation::SetCell( + bigtable_v2::mutation::SetCell { + family_name: self.params.family_name.to_string(), + column_qualifier: directory_key.clone().into(), + timestamp_micros: -1, // use server time to fill timestamp + value: data, + }, + )), + }, + ], + }) + .await + .map_err(|e| Error::StorageError(format!("unable to mutate rows: {}", e)))?; + + if resp.predicate_matched { + trace!("already existed") + } + + Ok(directory_digest) + } + + #[instrument(skip_all, fields(directory.digest = %root_directory_digest))] + fn get_recursive( + &self, + root_directory_digest: &B3Digest, + ) -> BoxStream<Result<proto::Directory, Error>> { + traverse_directory(self.clone(), root_directory_digest) + } + + #[instrument(skip_all)] + fn put_multiple_start(&self) -> Box<(dyn DirectoryPutter + 'static)> + where + Self: Clone, + { + Box::new(SimplePutter::new(self.clone())) + } +} diff --git a/tvix/castore/src/directoryservice/closure_validator.rs b/tvix/castore/src/directoryservice/closure_validator.rs index cc421d4ab5..183928a86f 100644 --- a/tvix/castore/src/directoryservice/closure_validator.rs +++ b/tvix/castore/src/directoryservice/closure_validator.rs @@ -2,7 +2,11 @@ use std::collections::{HashMap, HashSet}; use bstr::ByteSlice; -use tracing::{instrument, trace}; +use petgraph::{ + graph::{DiGraph, NodeIndex}, + visit::Bfs, +}; +use tracing::instrument; use crate::{ proto::{self, Directory}, @@ -13,14 +17,16 @@ use crate::{ /// Directories), and their insertion order. /// /// Directories need to be inserted (via `add`), in an order from the leaves to -/// the root. During insertion, We validate as much as we can at that time: +/// the root (DFS Post-Order). +/// During insertion, We validate as much as we can at that time: /// /// - individual validation of Directory messages /// - validation of insertion order (no upload of not-yet-known Directories) /// - validation of size fields of referred Directories /// -/// Internally it keeps all received Directories (and their sizes) in a HashMap, -/// keyed by digest. +/// Internally it keeps all received Directories in a directed graph, +/// with node weights being the Directories and edges pointing to child +/// directories. /// /// Once all Directories have been inserted, a finalize function can be /// called to get a (deduplicated and) validated list of directories, in @@ -29,23 +35,29 @@ use crate::{ /// there's no disconnected components, and only one root. #[derive(Default)] pub struct ClosureValidator { - directories_and_sizes: HashMap<B3Digest, (Directory, u64)>, + // A directed graph, using Directory as node weight, without edge weights. + // Edges point from parents to children. + graph: DiGraph<Directory, ()>, + + // A lookup table from directory digest to node index. + digest_to_node_ix: HashMap<B3Digest, NodeIndex>, - /// Keeps track of the last-inserted directory digest. Used to start the - /// connectivity check. - last_directory_digest: Option<B3Digest>, + /// Keeps track of the last-inserted directory graph node index. + /// On a correct insert, this will be the root node, from which the DFS post + /// order traversal will start from. + last_directory_ix: Option<NodeIndex>, } impl ClosureValidator { /// Insert a new Directory into the closure. /// Perform individual Directory validation, validation of insertion order /// and size fields. - #[instrument(level = "trace", skip_all, fields(directory.digest=%directory.digest()), err)] + #[instrument(level = "trace", skip_all, fields(directory.digest=%directory.digest(), directory.size=%directory.size()), err)] pub fn add(&mut self, directory: proto::Directory) -> Result<(), Error> { let digest = directory.digest(); - if self.directories_and_sizes.contains_key(&digest) { - trace!("already seen, skipping"); + // If we already saw this node previously, it's already validated and in the graph. + if self.digest_to_node_ix.contains_key(&digest) { return Ok(()); } @@ -55,35 +67,51 @@ impl ClosureValidator { .map_err(|e| Error::InvalidRequest(e.to_string()))?; // Ensure the directory only refers to directories which we already accepted. + // We lookup their node indices and add them to a HashSet. + let mut child_ixs = HashSet::new(); for dir in &directory.directories { - let dir_dgst = B3Digest::try_from(dir.digest.to_owned()).unwrap(); // validated + let child_digest = B3Digest::try_from(dir.digest.to_owned()).unwrap(); // validated // Ensure the digest has already been seen - let (_, recorded_dir_size) = - self.directories_and_sizes.get(&dir_dgst).ok_or_else(|| { - Error::InvalidRequest(format!( - "'{}' refers to unseen child dir: {}", - dir.name.as_bstr(), - dir_dgst - )) - })?; + let child_ix = *self.digest_to_node_ix.get(&child_digest).ok_or_else(|| { + Error::InvalidRequest(format!( + "'{}' refers to unseen child dir: {}", + dir.name.as_bstr(), + &child_digest + )) + })?; + + // Ensure the size specified in the child node matches the directory size itself. + let recorded_child_size = self + .graph + .node_weight(child_ix) + .expect("node not found") + .size(); // Ensure the size specified in the child node matches our records. - if dir.size != *recorded_dir_size { + if dir.size != recorded_child_size { return Err(Error::InvalidRequest(format!( "'{}' has wrong size, specified {}, recorded {}", dir.name.as_bstr(), dir.size, - recorded_dir_size + recorded_child_size ))); } + + child_ixs.insert(child_ix); } - trace!("inserting"); - let directory_size = directory.size(); - self.directories_and_sizes - .insert(digest.clone(), (directory, directory_size)); - self.last_directory_digest = Some(digest); + // Insert node into the graph, and add edges to all children. + let node_ix = self.graph.add_node(directory); + for child_ix in child_ixs { + self.graph.add_edge(node_ix, child_ix, ()); + } + + // Record the mapping from digest to node_ix in our lookup table. + self.digest_to_node_ix.insert(digest, node_ix); + + // Update last_directory_ix. + self.last_directory_ix = Some(node_ix); Ok(()) } @@ -93,77 +121,62 @@ impl ClosureValidator { /// order. /// In case no elements have been inserted, returns an empty list. #[instrument(level = "trace", skip_all, err)] - pub(crate) fn finalize(mut self) -> Result<Vec<Directory>, Error> { - if self.last_directory_digest.is_none() { + pub(crate) fn finalize(self) -> Result<Vec<Directory>, Error> { + // If no nodes were inserted, an empty list is returned. + let last_directory_ix = if let Some(x) = self.last_directory_ix { + x + } else { return Ok(vec![]); - } - let root_digest = self.last_directory_digest.unwrap(); - - // recursively walk all directories reachable from there, - // ensure we visited all nodes that were uploaded. - // If not, we might have received multiple disconnected graphs. - - // The list of directories we know we need to visit. - // Once we're finished working, and (there's no errors), this in reversed order will - // be a valid insertion order, and directories_and_sizes will be empty. - let mut dirs_to_visit = Vec::with_capacity(self.directories_and_sizes.len()); - dirs_to_visit.push( - self.directories_and_sizes - .remove(&root_digest) - .expect("root digest not found") - .0, - ); - // The set of digests seen while going visiting all directories. - let mut seen_dir_digests = HashSet::new(); - - // This loop moves gradually to the end of `dirs_to_visit`, while it's being - // extended. - // The loop stops once it reaches the end. - let mut i = 0; - while let Some(directory) = dirs_to_visit.get(i).map(|d| d.to_owned()) { - // lookup all child directories and put them in the back of dirs_to_visit, - // if they're not already there. - for child_dir in &directory.directories { - let child_digest = B3Digest::try_from(child_dir.digest.to_owned()).unwrap(); // validated - - // In case the digest is neither already visited nor already in dirs_to_visit, - // add it to the end of it. - // We don't need to check for the hash we're currently - // visiting, as we can't self-reference. - if !seen_dir_digests.contains(&child_digest) { - dirs_to_visit.push( - self.directories_and_sizes - .remove(&child_digest) - .expect("child digest not found") - .0, - ); - seen_dir_digests.insert(child_digest); - } - } + }; + + // do a BFS traversal of the graph, starting with the root node to get + // (the count of) all nodes reachable from there. + let mut traversal = Bfs::new(&self.graph, last_directory_ix); - i += 1; + let mut visited_directory_count = 0; + #[cfg(debug_assertions)] + let mut visited_directory_ixs = HashSet::new(); + #[cfg_attr(not(debug_assertions), allow(unused))] + while let Some(directory_ix) = traversal.next(&self.graph) { + #[cfg(debug_assertions)] + visited_directory_ixs.insert(directory_ix); + + visited_directory_count += 1; } - // check directories_and_sizes is empty. - if !self.directories_and_sizes.is_empty() { - if cfg!(debug_assertions) { + // If the number of nodes collected equals the total number of nodes in + // the graph, we know all nodes are connected. + if visited_directory_count != self.graph.node_count() { + // more or less exhaustive error reporting. + #[cfg(debug_assertions)] + { + let all_directory_ixs: HashSet<_> = self.graph.node_indices().collect(); + + let unvisited_directories: HashSet<_> = all_directory_ixs + .difference(&visited_directory_ixs) + .map(|ix| self.graph.node_weight(*ix).expect("node not found")) + .collect(); + return Err(Error::InvalidRequest(format!( - "found {} disconnected nodes: {:?}", - self.directories_and_sizes.len(), - self.directories_and_sizes + "found {} disconnected directories: {:?}", + self.graph.node_count() - visited_directory_ixs.len(), + unvisited_directories ))); - } else { + } + #[cfg(not(debug_assertions))] + { return Err(Error::InvalidRequest(format!( - "found {} disconnected nodes", - self.directories_and_sizes.len() + "found {} disconnected directories", + self.graph.node_count() - visited_directory_count ))); } } - // Reverse to have correct insertion order. - dirs_to_visit.reverse(); - - Ok(dirs_to_visit) + // Dissolve the graph, returning the nodes as a Vec. + // As the graph was populated in a valid DFS PostOrder, we can return + // nodes in that same order. + let (nodes, _edges) = self.graph.into_nodes_edges(); + Ok(nodes.into_iter().map(|x| x.weight).collect()) } } @@ -174,7 +187,7 @@ mod tests { proto::{self, Directory}, }; use lazy_static::lazy_static; - use test_case::test_case; + use rstest::rstest; lazy_static! { pub static ref BROKEN_DIRECTORY : Directory = Directory { @@ -197,28 +210,29 @@ mod tests { use super::ClosureValidator; + #[rstest] /// Uploading an empty directory should succeed. - #[test_case(vec![&DIRECTORY_A], false, Some(vec![&DIRECTORY_A]); "empty directory")] + #[case::empty_directory(&[&*DIRECTORY_A], false, Some(vec![&*DIRECTORY_A]))] /// Uploading A, then B (referring to A) should succeed. - #[test_case(vec![&DIRECTORY_A, &DIRECTORY_B], false, Some(vec![&DIRECTORY_A, &DIRECTORY_B]); "simple closure")] + #[case::simple_closure(&[&*DIRECTORY_A, &*DIRECTORY_B], false, Some(vec![&*DIRECTORY_A, &*DIRECTORY_B]))] /// Uploading A, then A, then C (referring to A twice) should succeed. /// We pretend to be a dumb client not deduping directories. - #[test_case(vec![&DIRECTORY_A, &DIRECTORY_A, &DIRECTORY_C], false, Some(vec![&DIRECTORY_A, &DIRECTORY_C]); "same child")] + #[case::same_child(&[&*DIRECTORY_A, &*DIRECTORY_A, &*DIRECTORY_C], false, Some(vec![&*DIRECTORY_A, &*DIRECTORY_C]))] /// Uploading A, then C (referring to A twice) should succeed. - #[test_case(vec![&DIRECTORY_A, &DIRECTORY_C], false, Some(vec![&DIRECTORY_A, &DIRECTORY_C]); "same child dedup")] + #[case::same_child_dedup(&[&*DIRECTORY_A, &*DIRECTORY_C], false, Some(vec![&*DIRECTORY_A, &*DIRECTORY_C]))] /// Uploading A, then C (referring to A twice), then B (itself referring to A) should fail during close, /// as B itself would be left unconnected. - #[test_case(vec![&DIRECTORY_A, &DIRECTORY_C, &DIRECTORY_B], false, None; "unconnected node")] + #[case::unconnected_node(&[&*DIRECTORY_A, &*DIRECTORY_C, &*DIRECTORY_B], false, None)] /// Uploading B (referring to A) should fail immediately, because A was never uploaded. - #[test_case(vec![&DIRECTORY_B], true, None; "dangling pointer")] + #[case::dangling_pointer(&[&*DIRECTORY_B], true, None)] /// Uploading a directory failing validation should fail immediately. - #[test_case(vec![&BROKEN_DIRECTORY], true, None; "failing validation")] + #[case::failing_validation(&[&*BROKEN_DIRECTORY], true, None)] /// Uploading a directory which refers to another Directory with a wrong size should fail. - #[test_case(vec![&DIRECTORY_A, &BROKEN_PARENT_DIRECTORY], true, None; "wrong size in parent")] + #[case::wrong_size_in_parent(&[&*DIRECTORY_A, &*BROKEN_PARENT_DIRECTORY], true, None)] fn test_uploads( - directories_to_upload: Vec<&Directory>, - exp_fail_upload_last: bool, - exp_finalize: Option<Vec<&Directory>>, // Some(_) if finalize successful, None if not. + #[case] directories_to_upload: &[&Directory], + #[case] exp_fail_upload_last: bool, + #[case] exp_finalize: Option<Vec<&Directory>>, // Some(_) if finalize successful, None if not. ) { let mut dcv = ClosureValidator::default(); let len_directories_to_upload = directories_to_upload.len(); @@ -236,13 +250,13 @@ mod tests { } } - // everything was uploaded successfully. Test drain(). + // everything was uploaded successfully. Test finalize(). let resp = dcv.finalize(); match exp_finalize { - Some(exp_drain) => { + Some(directories) => { assert_eq!( - Vec::from_iter(exp_drain.into_iter().map(|e| e.to_owned())), + Vec::from_iter(directories.iter().map(|e| (*e).to_owned())), resp.expect("drain should succeed") ); } diff --git a/tvix/castore/src/directoryservice/from_addr.rs b/tvix/castore/src/directoryservice/from_addr.rs index afb88c63ca..ae51df6376 100644 --- a/tvix/castore/src/directoryservice/from_addr.rs +++ b/tvix/castore/src/directoryservice/from_addr.rs @@ -19,7 +19,8 @@ use super::{DirectoryService, GRPCDirectoryService, MemoryDirectoryService, Sled /// - `grpc+http://host:port`, `grpc+https://host:port` /// Connects to a (remote) tvix-store gRPC service. pub async fn from_addr(uri: &str) -> Result<Box<dyn DirectoryService>, crate::Error> { - let url = Url::parse(uri) + #[allow(unused_mut)] + let mut url = Url::parse(uri) .map_err(|e| crate::Error::StorageError(format!("unable to parse url: {}", e)))?; let directory_service: Box<dyn DirectoryService> = match url.scheme() { @@ -62,6 +63,30 @@ pub async fn from_addr(uri: &str) -> Result<Box<dyn DirectoryService>, crate::Er let client = DirectoryServiceClient::new(crate::tonic::channel_from_url(&url).await?); Box::new(GRPCDirectoryService::from_client(client)) } + #[cfg(feature = "cloud")] + "bigtable" => { + use super::bigtable::BigtableParameters; + use super::BigtableDirectoryService; + + // parse the instance name from the hostname. + let instance_name = url + .host_str() + .ok_or_else(|| Error::StorageError("instance name missing".into()))? + .to_string(); + + // โฆ but add it to the query string now, so we just need to parse that. + url.query_pairs_mut() + .append_pair("instance_name", &instance_name); + + let params: BigtableParameters = serde_qs::from_str(url.query().unwrap_or_default()) + .map_err(|e| Error::InvalidRequest(format!("failed to parse parameters: {}", e)))?; + + Box::new( + BigtableDirectoryService::connect(params) + .await + .map_err(|e| Error::StorageError(e.to_string()))?, + ) + } _ => { return Err(crate::Error::StorageError(format!( "unknown scheme: {}", @@ -76,48 +101,70 @@ pub async fn from_addr(uri: &str) -> Result<Box<dyn DirectoryService>, crate::Er mod tests { use super::from_addr; use lazy_static::lazy_static; + use rstest::rstest; use tempfile::TempDir; - use test_case::test_case; lazy_static! { static ref TMPDIR_SLED_1: TempDir = TempDir::new().unwrap(); static ref TMPDIR_SLED_2: TempDir = TempDir::new().unwrap(); } + #[rstest] /// This uses an unsupported scheme. - #[test_case("http://foo.example/test", false; "unsupported scheme")] + #[case::unsupported_scheme("http://foo.example/test", false)] /// This configures sled in temporary mode. - #[test_case("sled://", true; "sled valid temporary")] + #[case::sled_valid_temporary("sled://", true)] /// This configures sled with /, which should fail. - #[test_case("sled:///", false; "sled invalid root")] + #[case::sled_invalid_root("sled:///", false)] /// This configures sled with a host, not path, which should fail. - #[test_case("sled://foo.example", false; "sled invalid host")] + #[case::sled_invalid_host("sled://foo.example", false)] /// This configures sled with a valid path path, which should succeed. - #[test_case(&format!("sled://{}", &TMPDIR_SLED_1.path().to_str().unwrap()), true; "sled valid path")] + #[case::sled_valid_path(&format!("sled://{}", &TMPDIR_SLED_1.path().to_str().unwrap()), true)] /// This configures sled with a host, and a valid path path, which should fail. - #[test_case(&format!("sled://foo.example{}", &TMPDIR_SLED_2.path().to_str().unwrap()), false; "sled invalid host with valid path")] + #[case::sled_invalid_host_with_valid_path(&format!("sled://foo.example{}", &TMPDIR_SLED_2.path().to_str().unwrap()), false)] /// This correctly sets the scheme, and doesn't set a path. - #[test_case("memory://", true; "memory valid")] + #[case::memory_valid("memory://", true)] /// This sets a memory url host to `foo` - #[test_case("memory://foo", false; "memory invalid host")] + #[case::memory_invalid_host("memory://foo", false)] /// This sets a memory url path to "/", which is invalid. - #[test_case("memory:///", false; "memory invalid root path")] + #[case::memory_invalid_root_path("memory:///", false)] /// This sets a memory url path to "/foo", which is invalid. - #[test_case("memory:///foo", false; "memory invalid root path foo")] + #[case::memory_invalid_root_path_foo("memory:///foo", false)] /// Correct scheme to connect to a unix socket. - #[test_case("grpc+unix:///path/to/somewhere", true; "grpc valid unix socket")] + #[case::grpc_valid_unix_socket("grpc+unix:///path/to/somewhere", true)] /// Correct scheme for unix socket, but setting a host too, which is invalid. - #[test_case("grpc+unix://host.example/path/to/somewhere", false; "grpc invalid unix socket and host")] + #[case::grpc_invalid_unix_socket_and_host("grpc+unix://host.example/path/to/somewhere", false)] /// Correct scheme to connect to localhost, with port 12345 - #[test_case("grpc+http://[::1]:12345", true; "grpc valid IPv6 localhost port 12345")] + #[case::grpc_valid_ipv6_localhost_port_12345("grpc+http://[::1]:12345", true)] /// Correct scheme to connect to localhost over http, without specifying a port. - #[test_case("grpc+http://localhost", true; "grpc valid http host without port")] + #[case::grpc_valid_http_host_without_port("grpc+http://localhost", true)] /// Correct scheme to connect to localhost over http, without specifying a port. - #[test_case("grpc+https://localhost", true; "grpc valid https host without port")] + #[case::grpc_valid_https_host_without_port("grpc+https://localhost", true)] /// Correct scheme to connect to localhost over http, but with additional path, which is invalid. - #[test_case("grpc+http://localhost/some-path", false; "grpc valid invalid host and path")] + #[case::grpc_invalid_host_and_path("grpc+http://localhost/some-path", false)] + /// A valid example for Bigtable + #[cfg_attr( + all(feature = "cloud", feature = "integration"), + case::bigtable_valid_url( + "bigtable://instance-1?project_id=project-1&table_name=table-1&family_name=cf1", + true + ) + )] + /// A valid example for Bigtable, specifying a custom channel size and timeout + #[cfg_attr( + all(feature = "cloud", feature = "integration"), + case::bigtable_valid_url( + "bigtable://instance-1?project_id=project-1&table_name=table-1&family_name=cf1&channel_size=10&timeout=10", + true + ) + )] + /// A invalid Bigtable example (missing fields) + #[cfg_attr( + all(feature = "cloud", feature = "integration"), + case::bigtable_invalid_url("bigtable://instance-1", false) + )] #[tokio::test] - async fn test_from_addr_tokio(uri_str: &str, exp_succeed: bool) { + async fn test_from_addr_tokio(#[case] uri_str: &str, #[case] exp_succeed: bool) { if exp_succeed { from_addr(uri_str).await.expect("should succeed"); } else { diff --git a/tvix/castore/src/directoryservice/grpc.rs b/tvix/castore/src/directoryservice/grpc.rs index 84cf01e167..7402fe1b56 100644 --- a/tvix/castore/src/directoryservice/grpc.rs +++ b/tvix/castore/src/directoryservice/grpc.rs @@ -34,6 +34,7 @@ impl GRPCDirectoryService { #[async_trait] impl DirectoryService for GRPCDirectoryService { + #[instrument(level = "trace", skip_all, fields(directory.digest = %digest))] async fn get( &self, digest: &B3Digest, @@ -82,6 +83,7 @@ impl DirectoryService for GRPCDirectoryService { } } + #[instrument(level = "trace", skip_all, fields(directory.digest = %directory.digest()))] async fn put(&self, directory: crate::proto::Directory) -> Result<B3Digest, crate::Error> { let resp = self .grpc_client @@ -101,7 +103,7 @@ impl DirectoryService for GRPCDirectoryService { } } - #[instrument(skip_all, fields(directory.digest = %root_directory_digest))] + #[instrument(level = "trace", skip_all, fields(directory.digest = %root_directory_digest))] fn get_recursive( &self, root_directory_digest: &B3Digest, diff --git a/tvix/castore/src/directoryservice/mod.rs b/tvix/castore/src/directoryservice/mod.rs index e9ac731739..cf6bea39d8 100644 --- a/tvix/castore/src/directoryservice/mod.rs +++ b/tvix/castore/src/directoryservice/mod.rs @@ -22,6 +22,12 @@ pub use self::sled::SledDirectoryService; pub use self::traverse::descend_to; pub use self::utils::traverse_directory; +#[cfg(feature = "cloud")] +mod bigtable; + +#[cfg(feature = "cloud")] +pub use self::bigtable::BigtableDirectoryService; + /// The base trait all Directory services need to implement. /// This is a simple get and put of [crate::proto::Directory], returning their /// digest. diff --git a/tvix/castore/src/directoryservice/tests/mod.rs b/tvix/castore/src/directoryservice/tests/mod.rs index cec49bb2c6..1b40d9feb0 100644 --- a/tvix/castore/src/directoryservice/tests/mod.rs +++ b/tvix/castore/src/directoryservice/tests/mod.rs @@ -16,13 +16,17 @@ use crate::{ mod utils; use self::utils::make_grpc_directory_service_client; +// TODO: add tests doing individual puts of a closure, then doing a get_recursive +// (and figure out semantics if necessary) + /// This produces a template, which will be applied to all individual test functions. /// See https://github.com/la10736/rstest/issues/130#issuecomment-968864832 #[template] #[rstest] -#[case::memory(directoryservice::from_addr("memory://").await.unwrap())] #[case::grpc(make_grpc_directory_service_client().await)] +#[case::memory(directoryservice::from_addr("memory://").await.unwrap())] #[case::sled(directoryservice::from_addr("sled://").await.unwrap())] +#[cfg_attr(all(feature = "cloud", feature = "integration"), case::bigtable(directoryservice::from_addr("bigtable://instance-1?project_id=project-1&table_name=table-1&family_name=cf1").await.unwrap()))] pub fn directory_services(#[case] directory_service: impl DirectoryService) {} /// Ensures asking for a directory that doesn't exist returns a Ok(None). diff --git a/tvix/castore/src/directoryservice/tests/utils.rs b/tvix/castore/src/directoryservice/tests/utils.rs index 72a3ff754d..0f706695ee 100644 --- a/tvix/castore/src/directoryservice/tests/utils.rs +++ b/tvix/castore/src/directoryservice/tests/utils.rs @@ -5,6 +5,7 @@ use crate::{ directoryservice::MemoryDirectoryService, proto::directory_service_server::DirectoryServiceServer, }; + use tonic::transport::{Endpoint, Server, Uri}; /// Constructs and returns a gRPC DirectoryService. diff --git a/tvix/castore/src/directoryservice/traverse.rs b/tvix/castore/src/directoryservice/traverse.rs index 573581edbd..8a668c868c 100644 --- a/tvix/castore/src/directoryservice/traverse.rs +++ b/tvix/castore/src/directoryservice/traverse.rs @@ -1,30 +1,20 @@ use super::DirectoryService; -use crate::{proto::NamedNode, B3Digest, Error}; -use std::os::unix::ffi::OsStrExt; +use crate::{proto::NamedNode, B3Digest, Error, Path}; use tracing::{instrument, warn}; /// This descends from a (root) node to the given (sub)path, returning the Node /// at that path, or none, if there's nothing at that path. -#[instrument(skip(directory_service))] +#[instrument(skip(directory_service, path), fields(%path))] pub async fn descend_to<DS>( directory_service: DS, root_node: crate::proto::node::Node, - path: &std::path::Path, + path: impl AsRef<Path> + std::fmt::Display, ) -> Result<Option<crate::proto::node::Node>, Error> where DS: AsRef<dyn DirectoryService>, { - // strip a possible `/` prefix from the path. - let path = { - if path.starts_with("/") { - path.strip_prefix("/").unwrap() - } else { - path - } - }; - let mut cur_node = root_node; - let mut it = path.components(); + let mut it = path.as_ref().components(); loop { match it.next() { @@ -59,9 +49,8 @@ where // look for first_component in the [Directory]. // FUTUREWORK: as the nodes() iterator returns in a sorted fashion, we // could stop as soon as e.name is larger than the search string. - let child_node = directory.nodes().find(|n| { - n.get_name() == first_component.as_os_str().as_bytes() - }); + let child_node = + directory.nodes().find(|n| n.get_name() == first_component); match child_node { // child node not found means there's no such element inside the directory. @@ -85,11 +74,10 @@ where #[cfg(test)] mod tests { - use std::path::PathBuf; - use crate::{ directoryservice, fixtures::{DIRECTORY_COMPLICATED, DIRECTORY_WITH_KEEP}, + PathBuf, }; use super::descend_to; @@ -132,7 +120,7 @@ mod tests { let resp = descend_to( &directory_service, node_directory_complicated.clone(), - &PathBuf::from(""), + "".parse::<PathBuf>().unwrap(), ) .await .expect("must succeed"); @@ -145,7 +133,7 @@ mod tests { let resp = descend_to( &directory_service, node_directory_complicated.clone(), - &PathBuf::from("keep"), + "keep".parse::<PathBuf>().unwrap(), ) .await .expect("must succeed"); @@ -158,7 +146,7 @@ mod tests { let resp = descend_to( &directory_service, node_directory_complicated.clone(), - &PathBuf::from("keep/.keep"), + "keep/.keep".parse::<PathBuf>().unwrap(), ) .await .expect("must succeed"); @@ -166,25 +154,12 @@ mod tests { assert_eq!(Some(node_file_keep.clone()), resp); } - // traversal to `keep/.keep` should return the node for the .keep file - { - let resp = descend_to( - &directory_service, - node_directory_complicated.clone(), - &PathBuf::from("/keep/.keep"), - ) - .await - .expect("must succeed"); - - assert_eq!(Some(node_file_keep), resp); - } - // traversal to `void` should return None (doesn't exist) { let resp = descend_to( &directory_service, node_directory_complicated.clone(), - &PathBuf::from("void"), + "void".parse::<PathBuf>().unwrap(), ) .await .expect("must succeed"); @@ -192,12 +167,12 @@ mod tests { assert_eq!(None, resp); } - // traversal to `void` should return None (doesn't exist) + // traversal to `v/oid` should return None (doesn't exist) { let resp = descend_to( &directory_service, node_directory_complicated.clone(), - &PathBuf::from("//v/oid"), + "v/oid".parse::<PathBuf>().unwrap(), ) .await .expect("must succeed"); @@ -211,25 +186,12 @@ mod tests { let resp = descend_to( &directory_service, node_directory_complicated.clone(), - &PathBuf::from("keep/.keep/foo"), + "keep/.keep/foo".parse::<PathBuf>().unwrap(), ) .await .expect("must succeed"); assert_eq!(None, resp); } - - // traversal to a subpath of '/' should return the root node. - { - let resp = descend_to( - &directory_service, - node_directory_complicated.clone(), - &PathBuf::from("/"), - ) - .await - .expect("must succeed"); - - assert_eq!(Some(node_directory_complicated), resp); - } } } diff --git a/tvix/castore/src/fs/file_attr.rs b/tvix/castore/src/fs/file_attr.rs index ad41f036a2..2e0e70e3cd 100644 --- a/tvix/castore/src/fs/file_attr.rs +++ b/tvix/castore/src/fs/file_attr.rs @@ -1,5 +1,5 @@ #![allow(clippy::unnecessary_cast)] // libc::S_IFDIR is u32 on Linux and u16 on MacOS -use super::inodes::{DirectoryInodeData, InodeData}; + use fuse_backend_rs::abi::fuse_abi::Attr; /// The [Attr] describing the root @@ -27,27 +27,3 @@ pub const ROOT_FILE_ATTR: Attr = Attr { #[cfg(target_os = "macos")] padding: 0, }; - -/// for given &Node and inode, construct an [Attr] -pub fn gen_file_attr(inode_data: &InodeData, inode: u64) -> Attr { - Attr { - ino: inode, - // FUTUREWORK: play with this numbers, as it affects read sizes for client applications. - blocks: 1024, - size: match inode_data { - InodeData::Regular(_, size, _) => *size as u64, - InodeData::Symlink(target) => target.len() as u64, - InodeData::Directory(DirectoryInodeData::Sparse(_, size)) => *size as u64, - InodeData::Directory(DirectoryInodeData::Populated(_, ref children)) => { - children.len() as u64 - } - }, - mode: match inode_data { - InodeData::Regular(_, _, false) => libc::S_IFREG as u32 | 0o444, // no-executable files - InodeData::Regular(_, _, true) => libc::S_IFREG as u32 | 0o555, // executable files - InodeData::Symlink(_) => libc::S_IFLNK as u32 | 0o444, - InodeData::Directory(_) => libc::S_IFDIR as u32 | 0o555, - }, - ..Default::default() - } -} diff --git a/tvix/castore/src/fs/inodes.rs b/tvix/castore/src/fs/inodes.rs index 9131b703ba..bdd4595434 100644 --- a/tvix/castore/src/fs/inodes.rs +++ b/tvix/castore/src/fs/inodes.rs @@ -1,5 +1,9 @@ //! This module contains all the data structures used to track information //! about inodes, which present tvix-castore nodes in a filesystem. +use std::time::Duration; + +use bytes::Bytes; + use crate::proto as castorepb; use crate::B3Digest; @@ -20,38 +24,73 @@ pub enum DirectoryInodeData { Populated(B3Digest, Vec<(u64, castorepb::node::Node)>), // [(child_inode, node)] } -impl From<&castorepb::node::Node> for InodeData { - fn from(value: &castorepb::node::Node) -> Self { - match value { - castorepb::node::Node::Directory(directory_node) => directory_node.into(), - castorepb::node::Node::File(file_node) => file_node.into(), - castorepb::node::Node::Symlink(symlink_node) => symlink_node.into(), +impl InodeData { + /// Constructs a new InodeData by consuming a [Node]. + /// It splits off the orginal name, so it can be used later. + pub fn from_node(node: castorepb::node::Node) -> (Self, Bytes) { + match node { + castorepb::node::Node::Directory(n) => ( + Self::Directory(DirectoryInodeData::Sparse( + n.digest.try_into().unwrap(), + n.size, + )), + n.name, + ), + castorepb::node::Node::File(n) => ( + Self::Regular(n.digest.try_into().unwrap(), n.size, n.executable), + n.name, + ), + castorepb::node::Node::Symlink(n) => (Self::Symlink(n.target), n.name), } } -} -impl From<&castorepb::SymlinkNode> for InodeData { - fn from(value: &castorepb::SymlinkNode) -> Self { - InodeData::Symlink(value.target.clone()) + pub fn as_fuse_file_attr(&self, inode: u64) -> fuse_backend_rs::abi::fuse_abi::Attr { + fuse_backend_rs::abi::fuse_abi::Attr { + ino: inode, + // FUTUREWORK: play with this numbers, as it affects read sizes for client applications. + blocks: 1024, + size: match self { + InodeData::Regular(_, size, _) => *size, + InodeData::Symlink(target) => target.len() as u64, + InodeData::Directory(DirectoryInodeData::Sparse(_, size)) => *size, + InodeData::Directory(DirectoryInodeData::Populated(_, ref children)) => { + children.len() as u64 + } + }, + mode: self.as_fuse_type() | self.mode(), + ..Default::default() + } } -} -impl From<&castorepb::FileNode> for InodeData { - fn from(value: &castorepb::FileNode) -> Self { - InodeData::Regular( - value.digest.clone().try_into().unwrap(), - value.size, - value.executable, - ) + fn mode(&self) -> u32 { + match self { + InodeData::Regular(_, _, false) | InodeData::Symlink(_) => 0o444, + InodeData::Regular(_, _, true) | InodeData::Directory(_) => 0o555, + } } -} -/// Converts a DirectoryNode to a sparsely populated InodeData::Directory. -impl From<&castorepb::DirectoryNode> for InodeData { - fn from(value: &castorepb::DirectoryNode) -> Self { - InodeData::Directory(DirectoryInodeData::Sparse( - value.digest.clone().try_into().unwrap(), - value.size, - )) + pub fn as_fuse_entry(&self, inode: u64) -> fuse_backend_rs::api::filesystem::Entry { + fuse_backend_rs::api::filesystem::Entry { + inode, + attr: self.as_fuse_file_attr(inode).into(), + attr_timeout: Duration::MAX, + entry_timeout: Duration::MAX, + ..Default::default() + } + } + + /// Returns the u32 fuse type + pub fn as_fuse_type(&self) -> u32 { + #[allow(clippy::let_and_return)] + let ty = match self { + InodeData::Regular(_, _, _) => libc::S_IFREG, + InodeData::Symlink(_) => libc::S_IFLNK, + InodeData::Directory(_) => libc::S_IFDIR, + }; + // libc::S_IFDIR is u32 on Linux and u16 on MacOS + #[cfg(target_os = "macos")] + let ty = ty as u32; + + ty } } diff --git a/tvix/castore/src/fs/mod.rs b/tvix/castore/src/fs/mod.rs index 699598b520..826523131f 100644 --- a/tvix/castore/src/fs/mod.rs +++ b/tvix/castore/src/fs/mod.rs @@ -12,6 +12,12 @@ pub mod virtiofs; #[cfg(test)] mod tests; +pub use self::root_nodes::RootNodes; +use self::{ + file_attr::ROOT_FILE_ATTR, + inode_tracker::InodeTracker, + inodes::{DirectoryInodeData, InodeData}, +}; use crate::proto as castorepb; use crate::{ blobservice::{BlobReader, BlobService}, @@ -19,10 +25,15 @@ use crate::{ proto::{node::Node, NamedNode}, B3Digest, }; -use fuse_backend_rs::abi::fuse_abi::stat64; -use fuse_backend_rs::api::filesystem::{Context, FileSystem, FsOptions, ROOT_ID}; +use bstr::ByteVec; +use bytes::Bytes; +use fuse_backend_rs::abi::fuse_abi::{stat64, OpenOptions}; +use fuse_backend_rs::api::filesystem::{ + Context, FileSystem, FsOptions, GetxattrReply, ListxattrReply, ROOT_ID, +}; use futures::StreamExt; use parking_lot::RwLock; +use std::sync::Mutex; use std::{ collections::HashMap, io, @@ -30,18 +41,12 @@ use std::{ sync::{atomic::Ordering, Arc}, time::Duration, }; +use std::{ffi::CStr, io::Cursor}; use tokio::{ io::{AsyncReadExt, AsyncSeekExt}, sync::mpsc, }; -use tracing::{debug, info_span, instrument, warn}; - -pub use self::root_nodes::RootNodes; -use self::{ - file_attr::{gen_file_attr, ROOT_FILE_ATTR}, - inode_tracker::InodeTracker, - inodes::{DirectoryInodeData, InodeData}, -}; +use tracing::{debug, error, instrument, warn, Span}; /// This implements a read-only FUSE filesystem for a tvix-store /// with the passed [BlobService], [DirectoryService] and [RootNodes]. @@ -83,15 +88,36 @@ pub struct TvixStoreFs<BS, DS, RN> { /// Whether to (try) listing elements in the root. list_root: bool, + /// Whether to expose blob and directory digests as extended attributes. + show_xattr: bool, + /// This maps a given basename in the root to the inode we allocated for the node. - root_nodes: RwLock<HashMap<Vec<u8>, u64>>, + root_nodes: RwLock<HashMap<Bytes, u64>>, /// This keeps track of inodes and data alongside them. inode_tracker: RwLock<InodeTracker>, + // FUTUREWORK: have a generic container type for dir/file handles and handle + // allocation. + /// Maps from the handle returned from an opendir to + /// This holds all opendir handles (for the root inode) + /// They point to the rx part of the channel producing the listing. + #[allow(clippy::type_complexity)] + dir_handles: RwLock< + HashMap< + u64, + ( + Span, + Arc<Mutex<mpsc::Receiver<(usize, Result<Node, crate::Error>)>>>, + ), + >, + >, + + next_dir_handle: AtomicU64, + /// This holds all open file handles #[allow(clippy::type_complexity)] - file_handles: RwLock<HashMap<u64, Arc<tokio::sync::Mutex<Box<dyn BlobReader>>>>>, + file_handles: RwLock<HashMap<u64, (Span, Arc<Mutex<Box<dyn BlobReader>>>)>>, next_file_handle: AtomicU64, @@ -109,6 +135,7 @@ where directory_service: DS, root_nodes_provider: RN, list_root: bool, + show_xattr: bool, ) -> Self { Self { blob_service, @@ -116,10 +143,14 @@ where root_nodes_provider, list_root, + show_xattr, root_nodes: RwLock::new(HashMap::default()), inode_tracker: RwLock::new(Default::default()), + dir_handles: RwLock::new(Default::default()), + next_dir_handle: AtomicU64::new(1), + file_handles: RwLock::new(Default::default()), next_file_handle: AtomicU64::new(1), tokio_handle: tokio::runtime::Handle::current(), @@ -153,12 +184,11 @@ where InodeData::Directory(DirectoryInodeData::Sparse(ref parent_digest, _)) => { let directory = self .tokio_handle - .block_on(self.tokio_handle.spawn({ + .block_on({ let directory_service = self.directory_service.clone(); let parent_digest = parent_digest.to_owned(); async move { directory_service.as_ref().get(&parent_digest).await } - })) - .unwrap()? + })? .ok_or_else(|| { warn!(directory.digest=%parent_digest, "directory not found"); // If the Directory can't be found, this is a hole, bail out. @@ -167,13 +197,16 @@ where // Turn the retrieved directory into a InodeData::Directory(DirectoryInodeData::Populated(..)), // allocating inodes for the children on the way. + // FUTUREWORK: there's a bunch of cloning going on here, which we can probably avoid. let children = { let mut inode_tracker = self.inode_tracker.write(); let children: Vec<(u64, castorepb::node::Node)> = directory .nodes() .map(|child_node| { - let child_ino = inode_tracker.put((&child_node).into()); + let (inode_data, _) = InodeData::from_node(child_node.clone()); + + let child_ino = inode_tracker.put(inode_data); (child_ino, child_node) }) .collect(); @@ -234,7 +267,7 @@ where // the root node doesn't exist, so the file doesn't exist. Ok(None) => Err(io::Error::from_raw_os_error(libc::ENOENT)), // The root node does exist - Ok(Some(ref root_node)) => { + Ok(Some(root_node)) => { // The name must match what's passed in the lookup, otherwise this is also a ENOENT. if root_node.get_name() != name.to_bytes() { debug!(root_node.name=?root_node.get_name(), found_node.name=%name.to_string_lossy(), "node name mismatch"); @@ -257,9 +290,9 @@ where // insert the (sparse) inode data and register in // self.root_nodes. - let inode_data: InodeData = root_node.into(); + let (inode_data, name) = InodeData::from_node(root_node); let ino = inode_tracker.put(inode_data.clone()); - root_nodes.insert(name.to_bytes().into(), ino); + root_nodes.insert(name, ino); Ok((ino, Arc::new(inode_data))) } @@ -267,6 +300,12 @@ where } } +/// Buffer size of the channel providing nodes in the mount root +const ROOT_NODES_BUFFER_SIZE: usize = 16; + +const XATTR_NAME_DIRECTORY_DIGEST: &[u8] = b"user.tvix.castore.directory.digest"; +const XATTR_NAME_BLOB_DIGEST: &[u8] = b"user.tvix.castore.blob.digest"; + impl<BS, DS, RN> FileSystem for TvixStoreFs<BS, DS, RN> where BS: AsRef<dyn BlobService> + Clone + Send + 'static, @@ -293,9 +332,9 @@ where match self.inode_tracker.read().get(inode) { None => Err(io::Error::from_raw_os_error(libc::ENOENT)), - Some(node) => { - debug!(node = ?node, "found node"); - Ok((gen_file_attr(&node, inode).into(), Duration::MAX)) + Some(inode_data) => { + debug!(inode_data = ?inode_data, "found node"); + Ok((inode_data.as_fuse_file_attr(inode).into(), Duration::MAX)) } } } @@ -318,21 +357,13 @@ where let (ino, inode_data) = self.name_in_root_to_ino_and_data(name)?; debug!(inode_data=?&inode_data, ino=ino, "Some"); - return Ok(fuse_backend_rs::api::filesystem::Entry { - inode: ino, - attr: gen_file_attr(&inode_data, ino).into(), - attr_timeout: Duration::MAX, - entry_timeout: Duration::MAX, - ..Default::default() - }); + return Ok(inode_data.as_fuse_entry(ino)); } // This is the "lookup for "a" inside inode 42. // We already know that inode 42 must be a directory. let (parent_digest, children) = self.get_directory_children(parent)?; - let span = info_span!("lookup", directory.digest = %parent_digest); - let _enter = span.enter(); - + Span::current().record("directory.digest", parent_digest.to_string()); // Search for that name in the list of children and return the FileAttrs. // in the children, find the one with the desired name. @@ -343,27 +374,69 @@ where // Reply with the file attributes for the child. // For child directories, we still have all data we need to reply. - Ok(fuse_backend_rs::api::filesystem::Entry { - inode: *child_ino, - attr: gen_file_attr(&child_inode_data, *child_ino).into(), - attr_timeout: Duration::MAX, - entry_timeout: Duration::MAX, - ..Default::default() - }) + Ok(child_inode_data.as_fuse_entry(*child_ino)) } else { // Child not found, return ENOENT. Err(io::Error::from_raw_os_error(libc::ENOENT)) } } - // TODO: readdirplus? + #[tracing::instrument(skip_all, fields(rq.inode = inode))] + fn opendir( + &self, + _ctx: &Context, + inode: Self::Inode, + _flags: u32, + ) -> io::Result<(Option<Self::Handle>, OpenOptions)> { + // In case opendir on the root is called, we provide the handle, as re-entering that listing is expensive. + // For all other directory inodes we just let readdir take care of it. + if inode == ROOT_ID { + if !self.list_root { + return Err(io::Error::from_raw_os_error(libc::EPERM)); // same error code as ipfs/kubo + } + + let root_nodes_provider = self.root_nodes_provider.clone(); + let (tx, rx) = mpsc::channel(ROOT_NODES_BUFFER_SIZE); + + // This task will run in the background immediately and will exit + // after the stream ends or if we no longer want any more entries. + self.tokio_handle.spawn(async move { + let mut stream = root_nodes_provider.list().enumerate(); + while let Some(node) = stream.next().await { + if tx.send(node).await.is_err() { + // If we get a send error, it means the sync code + // doesn't want any more entries. + break; + } + } + }); + + // Put the rx part into [self.dir_handles]. + // TODO: this will overflow after 2**64 operations, + // which is fine for now. + // See https://cl.tvl.fyi/c/depot/+/8834/comment/a6684ce0_d72469d1 + // for the discussion on alternatives. + let dh = self.next_dir_handle.fetch_add(1, Ordering::SeqCst); + + self.dir_handles + .write() + .insert(dh, (Span::current(), Arc::new(Mutex::new(rx)))); - #[tracing::instrument(skip_all, fields(rq.inode = inode, rq.offset = offset))] + return Ok(( + Some(dh), + fuse_backend_rs::api::filesystem::OpenOptions::empty(), // TODO: non-seekable + )); + } + + Ok((None, OpenOptions::empty())) + } + + #[tracing::instrument(skip_all, fields(rq.inode = inode, rq.handle = handle, rq.offset = offset), parent = self.dir_handles.read().get(&handle).and_then(|x| x.0.id()))] fn readdir( &self, _ctx: &Context, inode: Self::Inode, - _handle: Self::Handle, + handle: Self::Handle, _size: u32, offset: u64, add_entry: &mut dyn FnMut(fuse_backend_rs::api::filesystem::DirEntry) -> io::Result<usize>, @@ -373,90 +446,65 @@ where if inode == ROOT_ID { if !self.list_root { return Err(io::Error::from_raw_os_error(libc::EPERM)); // same error code as ipfs/kubo - } else { - let root_nodes_provider = self.root_nodes_provider.clone(); - let (tx, mut rx) = mpsc::channel(16); - - // This task will run in the background immediately and will exit - // after the stream ends or if we no longer want any more entries. - self.tokio_handle.spawn(async move { - let mut stream = root_nodes_provider.list().skip(offset as usize).enumerate(); - while let Some(node) = stream.next().await { - if tx.send(node).await.is_err() { - // If we get a send error, it means the sync code - // doesn't want any more entries. - break; - } - } - }); + } - while let Some((i, ref root_node)) = rx.blocking_recv() { - let root_node = match root_node { - Err(e) => { - warn!("failed to retrieve pathinfo: {}", e); - return Err(io::Error::from_raw_os_error(libc::EPERM)); - } - Ok(root_node) => root_node, - }; - - let name = root_node.get_name(); - // obtain the inode, or allocate a new one. - let ino = self.get_inode_for_root_name(name).unwrap_or_else(|| { - // insert the (sparse) inode data and register in - // self.root_nodes. - let ino = self.inode_tracker.write().put(root_node.into()); - self.root_nodes.write().insert(name.into(), ino); - ino - }); - - let ty = match root_node { - Node::Directory(_) => libc::S_IFDIR, - Node::File(_) => libc::S_IFREG, - Node::Symlink(_) => libc::S_IFLNK, - }; - - #[cfg(target_os = "macos")] - let ty = ty as u32; - - let written = add_entry(fuse_backend_rs::api::filesystem::DirEntry { - ino, - offset: offset + i as u64 + 1, - type_: ty, - name, - })?; - // If the buffer is full, add_entry will return `Ok(0)`. - if written == 0 { - break; - } + // get the handle from [self.dir_handles] + let (_span, rx) = match self.dir_handles.read().get(&handle) { + Some(rx) => rx.clone(), + None => { + warn!("dir handle {} unknown", handle); + return Err(io::Error::from_raw_os_error(libc::EIO)); } + }; + + let mut rx = rx + .lock() + .map_err(|_| crate::Error::StorageError("mutex poisoned".into()))?; + + while let Some((i, n)) = rx.blocking_recv() { + let root_node = n.map_err(|e| { + warn!("failed to retrieve root node: {}", e); + io::Error::from_raw_os_error(libc::EIO) + })?; + + let (inode_data, name) = InodeData::from_node(root_node); + + // obtain the inode, or allocate a new one. + let ino = self.get_inode_for_root_name(&name).unwrap_or_else(|| { + // insert the (sparse) inode data and register in + // self.root_nodes. + let ino = self.inode_tracker.write().put(inode_data.clone()); + self.root_nodes.write().insert(name.clone(), ino); + ino + }); - return Ok(()); + let written = add_entry(fuse_backend_rs::api::filesystem::DirEntry { + ino, + offset: offset + (i as u64) + 1, + type_: inode_data.as_fuse_type(), + name: &name, + })?; + // If the buffer is full, add_entry will return `Ok(0)`. + if written == 0 { + break; + } } + return Ok(()); } - // lookup the children, or return an error if it's not a directory. + // Non root-node case: lookup the children, or return an error if it's not a directory. let (parent_digest, children) = self.get_directory_children(inode)?; + Span::current().record("directory.digest", parent_digest.to_string()); - let span = info_span!("lookup", directory.digest = %parent_digest); - let _enter = span.enter(); + for (i, (ino, child_node)) in children.into_iter().skip(offset as usize).enumerate() { + let (inode_data, name) = InodeData::from_node(child_node); - for (i, (ino, child_node)) in children.iter().skip(offset as usize).enumerate() { // the second parameter will become the "offset" parameter on the next call. let written = add_entry(fuse_backend_rs::api::filesystem::DirEntry { - ino: *ino, - offset: offset + i as u64 + 1, - type_: match child_node { - #[allow(clippy::unnecessary_cast)] - // libc::S_IFDIR is u32 on Linux and u16 on MacOS - Node::Directory(_) => libc::S_IFDIR as u32, - #[allow(clippy::unnecessary_cast)] - // libc::S_IFDIR is u32 on Linux and u16 on MacOS - Node::File(_) => libc::S_IFREG as u32, - #[allow(clippy::unnecessary_cast)] - // libc::S_IFDIR is u32 on Linux and u16 on MacOS - Node::Symlink(_) => libc::S_IFLNK as u32, - }, - name: child_node.get_name(), + ino, + offset: offset + (i as u64) + 1, + type_: inode_data.as_fuse_type(), + name: &name, })?; // If the buffer is full, add_entry will return `Ok(0)`. if written == 0 { @@ -467,6 +515,121 @@ where Ok(()) } + #[tracing::instrument(skip_all, fields(rq.inode = inode, rq.handle = handle), parent = self.dir_handles.read().get(&handle).and_then(|x| x.0.id()))] + fn readdirplus( + &self, + _ctx: &Context, + inode: Self::Inode, + handle: Self::Handle, + _size: u32, + offset: u64, + add_entry: &mut dyn FnMut( + fuse_backend_rs::api::filesystem::DirEntry, + fuse_backend_rs::api::filesystem::Entry, + ) -> io::Result<usize>, + ) -> io::Result<()> { + debug!("readdirplus"); + + if inode == ROOT_ID { + if !self.list_root { + return Err(io::Error::from_raw_os_error(libc::EPERM)); // same error code as ipfs/kubo + } + + // get the handle from [self.dir_handles] + let (_span, rx) = match self.dir_handles.read().get(&handle) { + Some(rx) => rx.clone(), + None => { + warn!("dir handle {} unknown", handle); + return Err(io::Error::from_raw_os_error(libc::EIO)); + } + }; + + let mut rx = rx + .lock() + .map_err(|_| crate::Error::StorageError("mutex poisoned".into()))?; + + while let Some((i, n)) = rx.blocking_recv() { + let root_node = n.map_err(|e| { + warn!("failed to retrieve root node: {}", e); + io::Error::from_raw_os_error(libc::EPERM) + })?; + + let (inode_data, name) = InodeData::from_node(root_node); + + // obtain the inode, or allocate a new one. + let ino = self.get_inode_for_root_name(&name).unwrap_or_else(|| { + // insert the (sparse) inode data and register in + // self.root_nodes. + let ino = self.inode_tracker.write().put(inode_data.clone()); + self.root_nodes.write().insert(name.clone(), ino); + ino + }); + + let written = add_entry( + fuse_backend_rs::api::filesystem::DirEntry { + ino, + offset: offset + (i as u64) + 1, + type_: inode_data.as_fuse_type(), + name: &name, + }, + inode_data.as_fuse_entry(ino), + )?; + // If the buffer is full, add_entry will return `Ok(0)`. + if written == 0 { + break; + } + } + return Ok(()); + } + + // Non root-node case: lookup the children, or return an error if it's not a directory. + let (parent_digest, children) = self.get_directory_children(inode)?; + Span::current().record("directory.digest", parent_digest.to_string()); + + for (i, (ino, child_node)) in children.into_iter().skip(offset as usize).enumerate() { + let (inode_data, name) = InodeData::from_node(child_node); + + // the second parameter will become the "offset" parameter on the next call. + let written = add_entry( + fuse_backend_rs::api::filesystem::DirEntry { + ino, + offset: offset + (i as u64) + 1, + type_: inode_data.as_fuse_type(), + name: &name, + }, + inode_data.as_fuse_entry(ino), + )?; + // If the buffer is full, add_entry will return `Ok(0)`. + if written == 0 { + break; + } + } + + Ok(()) + } + + #[tracing::instrument(skip_all, fields(rq.inode = inode, rq.handle = handle), parent = self.dir_handles.read().get(&handle).and_then(|x| x.0.id()))] + fn releasedir( + &self, + _ctx: &Context, + inode: Self::Inode, + _flags: u32, + handle: Self::Handle, + ) -> io::Result<()> { + if inode == ROOT_ID { + // drop the rx part of the channel. + match self.dir_handles.write().remove(&handle) { + // drop it, which will close it. + Some(rx) => drop(rx), + None => { + warn!("dir handle not found"); + } + } + } + + Ok(()) + } + #[tracing::instrument(skip_all, fields(rq.inode = inode))] fn open( &self, @@ -490,18 +653,13 @@ where Err(io::Error::from_raw_os_error(libc::EISDIR)) } InodeData::Regular(ref blob_digest, _blob_size, _) => { - let span = info_span!("read", blob.digest = %blob_digest); - let _enter = span.enter(); + Span::current().record("blob.digest", blob_digest.to_string()); - let task = self.tokio_handle.spawn({ + match self.tokio_handle.block_on({ let blob_service = self.blob_service.clone(); let blob_digest = blob_digest.clone(); async move { blob_service.as_ref().open_read(&blob_digest).await } - }); - - let blob_reader = self.tokio_handle.block_on(task).unwrap(); - - match blob_reader { + }) { Ok(None) => { warn!("blob not found"); Err(io::Error::from_raw_os_error(libc::EIO)) @@ -518,10 +676,9 @@ where // for the discussion on alternatives. let fh = self.next_file_handle.fetch_add(1, Ordering::SeqCst); - debug!("add file handle {}", fh); self.file_handles .write() - .insert(fh, Arc::new(tokio::sync::Mutex::new(blob_reader))); + .insert(fh, (Span::current(), Arc::new(Mutex::new(blob_reader)))); Ok(( Some(fh), @@ -533,7 +690,7 @@ where } } - #[tracing::instrument(skip_all, fields(rq.inode = inode, fh = handle))] + #[tracing::instrument(skip_all, fields(rq.inode = inode, rq.handle = handle), parent = self.file_handles.read().get(&handle).and_then(|x| x.0.id()))] fn release( &self, _ctx: &Context, @@ -544,20 +701,19 @@ where _flock_release: bool, _lock_owner: Option<u64>, ) -> io::Result<()> { - // remove and get ownership on the blob reader match self.file_handles.write().remove(&handle) { - // drop it, which will close it. + // drop the blob reader, which will close it. Some(blob_reader) => drop(blob_reader), None => { // These might already be dropped if a read error occured. - debug!("file_handle {} not found", handle); + warn!("file handle not found"); } } Ok(()) } - #[tracing::instrument(skip_all, fields(rq.inode = inode, rq.offset = offset, rq.size = size))] + #[tracing::instrument(skip_all, fields(rq.inode = inode, rq.handle = handle, rq.offset = offset, rq.size = size), parent = self.file_handles.read().get(&handle).and_then(|x| x.0.id()))] fn read( &self, _ctx: &Context, @@ -574,29 +730,31 @@ where // We need to take out the blob reader from self.file_handles, so we can // interact with it in the separate task. // On success, we pass it back out of the task, so we can put it back in self.file_handles. - let blob_reader = match self.file_handles.read().get(&handle) { - Some(blob_reader) => blob_reader.clone(), - None => { + let (_span, blob_reader) = self + .file_handles + .read() + .get(&handle) + .ok_or_else(|| { warn!("file handle {} unknown", handle); - return Err(io::Error::from_raw_os_error(libc::EIO)); - } - }; + io::Error::from_raw_os_error(libc::EIO) + }) + .cloned()?; - let task = self.tokio_handle.spawn(async move { - let mut blob_reader = blob_reader.lock().await; + let mut blob_reader = blob_reader + .lock() + .map_err(|_| crate::Error::StorageError("mutex poisoned".into()))?; + let buf = self.tokio_handle.block_on(async move { // seek to the offset specified, which is relative to the start of the file. - let resp = blob_reader.seek(io::SeekFrom::Start(offset)).await; - - match resp { - Ok(pos) => { - debug_assert_eq!(offset, pos); - } - Err(e) => { + let pos = blob_reader + .seek(io::SeekFrom::Start(offset)) + .await + .map_err(|e| { warn!("failed to seek to offset {}: {}", offset, e); - return Err(io::Error::from_raw_os_error(libc::EIO)); - } - } + io::Error::from_raw_os_error(libc::EIO) + })?; + + debug_assert_eq!(offset, pos); // As written in the fuse docs, read should send exactly the number // of bytes requested except on EOF or error. @@ -606,12 +764,19 @@ where // copy things from the internal buffer into buf to fill it till up until size tokio::io::copy(&mut blob_reader.as_mut().take(size as u64), &mut buf).await?; - Ok(buf) - }); + Ok::<_, std::io::Error>(buf) + })?; - let buf = self.tokio_handle.block_on(task).unwrap()?; + // We cannot use w.write() here, we're required to call write multiple + // times until we wrote the entirety of the buffer (which is `size`, except on EOF). + let buf_len = buf.len(); + let bytes_written = io::copy(&mut Cursor::new(buf), w)?; + if bytes_written != buf_len as u64 { + error!(bytes_written=%bytes_written, "unable to write all of buf to kernel"); + return Err(io::Error::from_raw_os_error(libc::EIO)); + } - w.write(&buf) + Ok(bytes_written as usize) } #[tracing::instrument(skip_all, fields(rq.inode = inode))] @@ -628,4 +793,85 @@ where InodeData::Symlink(ref target) => Ok(target.to_vec()), } } + + #[tracing::instrument(skip_all, fields(rq.inode = inode, name=?name))] + fn getxattr( + &self, + _ctx: &Context, + inode: Self::Inode, + name: &CStr, + size: u32, + ) -> io::Result<GetxattrReply> { + if !self.show_xattr { + return Err(io::Error::from_raw_os_error(libc::ENOSYS)); + } + + // Peek at the inode requested, and construct the response. + let digest_str = match *self + .inode_tracker + .read() + .get(inode) + .ok_or_else(|| io::Error::from_raw_os_error(libc::ENODATA))? + { + InodeData::Directory(DirectoryInodeData::Sparse(ref digest, _)) + | InodeData::Directory(DirectoryInodeData::Populated(ref digest, _)) + if name.to_bytes() == XATTR_NAME_DIRECTORY_DIGEST => + { + digest.to_string() + } + InodeData::Regular(ref digest, _, _) if name.to_bytes() == XATTR_NAME_BLOB_DIGEST => { + digest.to_string() + } + _ => { + return Err(io::Error::from_raw_os_error(libc::ENODATA)); + } + }; + + if size == 0 { + Ok(GetxattrReply::Count(digest_str.len() as u32)) + } else if size < digest_str.len() as u32 { + Err(io::Error::from_raw_os_error(libc::ERANGE)) + } else { + Ok(GetxattrReply::Value(digest_str.into_bytes())) + } + } + + #[tracing::instrument(skip_all, fields(rq.inode = inode))] + fn listxattr( + &self, + _ctx: &Context, + inode: Self::Inode, + size: u32, + ) -> io::Result<ListxattrReply> { + if !self.show_xattr { + return Err(io::Error::from_raw_os_error(libc::ENOSYS)); + } + + // determine the (\0-terminated list) to of xattr keys present, depending on the type of the inode. + let xattrs_names = { + let mut out = Vec::new(); + if let Some(inode_data) = self.inode_tracker.read().get(inode) { + match *inode_data { + InodeData::Directory(_) => { + out.extend_from_slice(XATTR_NAME_DIRECTORY_DIGEST); + out.push_byte(b'\x00'); + } + InodeData::Regular(..) => { + out.extend_from_slice(XATTR_NAME_BLOB_DIGEST); + out.push_byte(b'\x00'); + } + _ => {} + } + } + out + }; + + if size == 0 { + Ok(ListxattrReply::Count(xattrs_names.len() as u32)) + } else if size < xattrs_names.len() as u32 { + Err(io::Error::from_raw_os_error(libc::ERANGE)) + } else { + Ok(ListxattrReply::Names(xattrs_names.to_vec())) + } + } } diff --git a/tvix/castore/src/fs/tests.rs b/tvix/castore/src/fs/tests.rs index 924454caa6..d6eeb8a411 100644 --- a/tvix/castore/src/fs/tests.rs +++ b/tvix/castore/src/fs/tests.rs @@ -1,18 +1,19 @@ +use bstr::ByteSlice; +use bytes::Bytes; use std::{ collections::BTreeMap, + ffi::{OsStr, OsString}, io::{self, Cursor}, - os::unix::fs::MetadataExt, + os::unix::{ffi::OsStrExt, fs::MetadataExt}, path::Path, sync::Arc, }; - -use bytes::Bytes; use tempfile::TempDir; use tokio_stream::{wrappers::ReadDirStream, StreamExt}; use super::{fuse::FuseDaemon, TvixStoreFs}; +use crate::proto as castorepb; use crate::proto::node::Node; -use crate::proto::{self as castorepb}; use crate::{ blobservice::{BlobService, MemoryBlobService}, directoryservice::{DirectoryService, MemoryDirectoryService}, @@ -40,6 +41,7 @@ fn do_mount<P: AsRef<Path>, BS, DS>( root_nodes: BTreeMap<bytes::Bytes, Node>, mountpoint: P, list_root: bool, + show_xattr: bool, ) -> io::Result<FuseDaemon> where BS: AsRef<dyn BlobService> + Send + Sync + Clone + 'static, @@ -50,6 +52,7 @@ where directory_service, Arc::new(root_nodes), list_root, + show_xattr, ); FuseDaemon::new(Arc::new(fs), mountpoint.as_ref(), 4, false) } @@ -250,6 +253,7 @@ async fn mount() { BTreeMap::default(), tmpdir.path(), false, + false, ) .expect("must succeed"); @@ -272,18 +276,13 @@ async fn root() { BTreeMap::default(), tmpdir.path(), false, + false, ) .expect("must succeed"); { - // read_dir succeeds, but getting the first element will fail. - let mut it = ReadDirStream::new(tokio::fs::read_dir(tmpdir).await.expect("must succeed")); - - let err = it - .next() - .await - .expect("must be some") - .expect_err("must be err"); + // read_dir fails (as opendir fails). + let err = tokio::fs::read_dir(tmpdir).await.expect_err("must fail"); assert_eq!(std::io::ErrorKind::PermissionDenied, err.kind()); } @@ -311,6 +310,7 @@ async fn root_with_listing() { root_nodes, tmpdir.path(), true, /* allow listing */ + false, ) .expect("must succeed"); @@ -354,6 +354,7 @@ async fn stat_file_at_root() { root_nodes, tmpdir.path(), false, + false, ) .expect("must succeed"); @@ -390,6 +391,7 @@ async fn read_file_at_root() { root_nodes, tmpdir.path(), false, + false, ) .expect("must succeed"); @@ -426,6 +428,7 @@ async fn read_large_file_at_root() { root_nodes, tmpdir.path(), false, + false, ) .expect("must succeed"); @@ -470,6 +473,7 @@ async fn symlink_readlink() { root_nodes, tmpdir.path(), false, + false, ) .expect("must succeed"); @@ -516,6 +520,7 @@ async fn read_stat_through_symlink() { root_nodes, tmpdir.path(), false, + false, ) .expect("must succeed"); @@ -560,6 +565,7 @@ async fn read_stat_directory() { root_nodes, tmpdir.path(), false, + false, ) .expect("must succeed"); @@ -573,6 +579,91 @@ async fn read_stat_directory() { fuse_daemon.unmount().expect("unmount"); } +/// Read a directory and file in the root, and ensure the xattrs expose blob or +/// directory digests. +#[tokio::test] +async fn xattr() { + // https://plume.benboeckel.net/~/JustAnotherBlog/skipping-tests-in-rust + if !std::path::Path::new("/dev/fuse").exists() { + eprintln!("skipping test"); + return; + } + let tmpdir = TempDir::new().unwrap(); + + let (blob_service, directory_service) = gen_svcs(); + let mut root_nodes = BTreeMap::default(); + + populate_directory_with_keep(&blob_service, &directory_service, &mut root_nodes).await; + populate_blob_a(&blob_service, &mut root_nodes).await; + + let mut fuse_daemon = do_mount( + blob_service, + directory_service, + root_nodes, + tmpdir.path(), + false, + true, /* support xattr */ + ) + .expect("must succeed"); + + // peek at the directory + { + let p = tmpdir.path().join(DIRECTORY_WITH_KEEP_NAME); + + let xattr_names: Vec<OsString> = xattr::list(&p).expect("must succeed").collect(); + // There should be 1 key, XATTR_NAME_DIRECTORY_DIGEST. + assert_eq!(1, xattr_names.len(), "there should be 1 xattr name"); + assert_eq!( + super::XATTR_NAME_DIRECTORY_DIGEST, + xattr_names.first().unwrap().as_encoded_bytes() + ); + + // The key should equal to the string-formatted b3 digest. + let val = xattr::get(&p, OsStr::from_bytes(super::XATTR_NAME_DIRECTORY_DIGEST)) + .expect("must succeed") + .expect("must be some"); + assert_eq!( + fixtures::DIRECTORY_WITH_KEEP + .digest() + .to_string() + .as_bytes() + .as_bstr(), + val.as_bstr() + ); + + // Reading another xattr key is gonna return None. + let val = xattr::get(&p, OsStr::from_bytes(b"user.cheesecake")).expect("must succeed"); + assert_eq!(None, val); + } + // peek at the file + { + let p = tmpdir.path().join(BLOB_A_NAME); + + let xattr_names: Vec<OsString> = xattr::list(&p).expect("must succeed").collect(); + // There should be 1 key, XATTR_NAME_BLOB_DIGEST. + assert_eq!(1, xattr_names.len(), "there should be 1 xattr name"); + assert_eq!( + super::XATTR_NAME_BLOB_DIGEST, + xattr_names.first().unwrap().as_encoded_bytes() + ); + + // The key should equal to the string-formatted b3 digest. + let val = xattr::get(&p, OsStr::from_bytes(super::XATTR_NAME_BLOB_DIGEST)) + .expect("must succeed") + .expect("must be some"); + assert_eq!( + fixtures::BLOB_A_DIGEST.to_string().as_bytes().as_bstr(), + val.as_bstr() + ); + + // Reading another xattr key is gonna return None. + let val = xattr::get(&p, OsStr::from_bytes(b"user.cheesecake")).expect("must succeed"); + assert_eq!(None, val); + } + + fuse_daemon.unmount().expect("unmount"); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] /// Read a blob inside a directory. This ensures we successfully populate directory data. async fn read_blob_inside_dir() { @@ -594,6 +685,7 @@ async fn read_blob_inside_dir() { root_nodes, tmpdir.path(), false, + false, ) .expect("must succeed"); @@ -633,6 +725,7 @@ async fn read_blob_deep_inside_dir() { root_nodes, tmpdir.path(), false, + false, ) .expect("must succeed"); @@ -675,6 +768,7 @@ async fn readdir() { root_nodes, tmpdir.path(), false, + false, ) .expect("must succeed"); @@ -734,6 +828,7 @@ async fn readdir_deep() { root_nodes, tmpdir.path(), false, + false, ) .expect("must succeed"); @@ -783,6 +878,7 @@ async fn check_attributes() { root_nodes, tmpdir.path(), false, + false, ) .expect("must succeed"); @@ -857,6 +953,7 @@ async fn compare_inodes_directories() { root_nodes, tmpdir.path(), false, + false, ) .expect("must succeed"); @@ -900,6 +997,7 @@ async fn compare_inodes_files() { root_nodes, tmpdir.path(), false, + false, ) .expect("must succeed"); @@ -948,6 +1046,7 @@ async fn compare_inodes_symlinks() { root_nodes, tmpdir.path(), false, + false, ) .expect("must succeed"); @@ -990,6 +1089,7 @@ async fn read_wrong_paths_in_root() { root_nodes, tmpdir.path(), false, + false, ) .expect("must succeed"); @@ -1044,6 +1144,7 @@ async fn disallow_writes() { root_nodes, tmpdir.path(), false, + false, ) .expect("must succeed"); @@ -1075,6 +1176,7 @@ async fn missing_directory() { root_nodes, tmpdir.path(), false, + false, ) .expect("must succeed"); @@ -1122,6 +1224,7 @@ async fn missing_blob() { root_nodes, tmpdir.path(), false, + false, ) .expect("must succeed"); diff --git a/tvix/castore/src/fs/virtiofs.rs b/tvix/castore/src/fs/virtiofs.rs index 846270d285..d63e2f2bdd 100644 --- a/tvix/castore/src/fs/virtiofs.rs +++ b/tvix/castore/src/fs/virtiofs.rs @@ -34,6 +34,7 @@ enum Error { /// Invalid descriptor chain. InvalidDescriptorChain, /// Failed to handle filesystem requests. + #[allow(dead_code)] HandleRequests(fuse_backend_rs::Error), /// Failed to construct new vhost user daemon. NewDaemon, diff --git a/tvix/castore/src/hashing_reader.rs b/tvix/castore/src/hashing_reader.rs index 5352363247..7d78cae587 100644 --- a/tvix/castore/src/hashing_reader.rs +++ b/tvix/castore/src/hashing_reader.rs @@ -62,7 +62,7 @@ where mod tests { use std::io::Cursor; - use test_case::test_case; + use rstest::rstest; use crate::fixtures::BLOB_A; use crate::fixtures::BLOB_A_DIGEST; @@ -71,11 +71,12 @@ mod tests { use crate::fixtures::EMPTY_BLOB_DIGEST; use crate::{B3Digest, B3HashingReader}; - #[test_case(&BLOB_A, &BLOB_A_DIGEST; "blob a")] - #[test_case(&BLOB_B, &BLOB_B_DIGEST; "blob b")] - #[test_case(&[], &EMPTY_BLOB_DIGEST; "empty blob")] + #[rstest] + #[case::blob_a(&BLOB_A, &BLOB_A_DIGEST)] + #[case::blob_b(&BLOB_B, &BLOB_B_DIGEST)] + #[case::empty_blob(&[], &EMPTY_BLOB_DIGEST)] #[tokio::test] - async fn test_b3_hashing_reader(data: &[u8], b3_digest: &B3Digest) { + async fn test_b3_hashing_reader(#[case] data: &[u8], #[case] b3_digest: &B3Digest) { let r = Cursor::new(data); let mut hr = B3HashingReader::from(r); diff --git a/tvix/castore/src/import.rs b/tvix/castore/src/import.rs deleted file mode 100644 index 3ba6087efc..0000000000 --- a/tvix/castore/src/import.rs +++ /dev/null @@ -1,385 +0,0 @@ -use crate::blobservice::BlobService; -use crate::directoryservice::DirectoryPutter; -use crate::directoryservice::DirectoryService; -use crate::proto::node::Node; -use crate::proto::Directory; -use crate::proto::DirectoryNode; -use crate::proto::FileNode; -use crate::proto::SymlinkNode; -use crate::Error as CastoreError; -use async_stream::stream; -use futures::pin_mut; -use futures::Stream; -use std::fs::FileType; -use tracing::Level; - -#[cfg(target_family = "unix")] -use std::os::unix::ffi::OsStrExt; - -use std::{ - collections::HashMap, - fmt::Debug, - os::unix::prelude::PermissionsExt, - path::{Path, PathBuf}, -}; -use tokio_stream::StreamExt; -use tracing::instrument; -use walkdir::DirEntry; -use walkdir::WalkDir; - -#[cfg(debug_assertions)] -use std::collections::HashSet; - -#[derive(Debug, thiserror::Error)] -pub enum Error { - #[error("failed to upload directory at {0}: {1}")] - UploadDirectoryError(PathBuf, CastoreError), - - #[error("invalid encoding encountered for entry {0:?}")] - InvalidEncoding(PathBuf), - - #[error("unable to stat {0}: {1}")] - UnableToStat(PathBuf, std::io::Error), - - #[error("unable to open {0}: {1}")] - UnableToOpen(PathBuf, std::io::Error), - - #[error("unable to read {0}: {1}")] - UnableToRead(PathBuf, std::io::Error), - - #[error("unsupported file {0} type: {1:?}")] - UnsupportedFileType(PathBuf, FileType), -} - -impl From<CastoreError> for Error { - fn from(value: CastoreError) -> Self { - match value { - CastoreError::InvalidRequest(_) => panic!("tvix bug"), - CastoreError::StorageError(_) => panic!("error"), - } - } -} - -impl From<Error> for std::io::Error { - fn from(value: Error) -> Self { - std::io::Error::new(std::io::ErrorKind::Other, value) - } -} - -/// This processes a given [walkdir::DirEntry] and returns a -/// proto::node::Node, depending on the type of the entry. -/// -/// If the entry is a file, its contents are uploaded. -/// If the entry is a directory, the Directory is uploaded as well. -/// For this to work, it relies on the caller to provide the directory object -/// with the previously returned (child) nodes. -/// -/// It assumes to be called only if all children of it have already been processed. If the entry is -/// indeed a directory, it'll also upload that directory to the store. For this, the -/// so-far-assembled Directory object for this path needs to be passed in. -/// -/// It assumes the caller adds returned nodes to the directories it assembles. -#[instrument(skip_all, fields(entry.file_type=?&entry.file_type(),entry.path=?entry.path()))] -async fn process_entry<'a, BS>( - blob_service: BS, - directory_putter: &'a mut Box<dyn DirectoryPutter>, - entry: &'a walkdir::DirEntry, - maybe_directory: Option<Directory>, -) -> Result<Node, Error> -where - BS: AsRef<dyn BlobService> + Clone, -{ - let file_type = entry.file_type(); - - if file_type.is_dir() { - let directory = maybe_directory - .expect("tvix bug: must be called with some directory in the case of directory"); - let directory_digest = directory.digest(); - let directory_size = directory.size(); - - // upload this directory - directory_putter - .put(directory) - .await - .map_err(|e| Error::UploadDirectoryError(entry.path().to_path_buf(), e))?; - - return Ok(Node::Directory(DirectoryNode { - name: entry.file_name().as_bytes().to_owned().into(), - digest: directory_digest.into(), - size: directory_size, - })); - } - - if file_type.is_symlink() { - let target: bytes::Bytes = std::fs::read_link(entry.path()) - .map_err(|e| Error::UnableToStat(entry.path().to_path_buf(), e))? - .as_os_str() - .as_bytes() - .to_owned() - .into(); - - return Ok(Node::Symlink(SymlinkNode { - name: entry.file_name().as_bytes().to_owned().into(), - target, - })); - } - - if file_type.is_file() { - let metadata = entry - .metadata() - .map_err(|e| Error::UnableToStat(entry.path().to_path_buf(), e.into()))?; - - let mut file = tokio::fs::File::open(entry.path()) - .await - .map_err(|e| Error::UnableToOpen(entry.path().to_path_buf(), e))?; - - let mut writer = blob_service.as_ref().open_write().await; - - if let Err(e) = tokio::io::copy(&mut file, &mut writer).await { - return Err(Error::UnableToRead(entry.path().to_path_buf(), e)); - }; - - let digest = writer - .close() - .await - .map_err(|e| Error::UnableToRead(entry.path().to_path_buf(), e))?; - - return Ok(Node::File(FileNode { - name: entry.file_name().as_bytes().to_vec().into(), - digest: digest.into(), - size: metadata.len(), - // If it's executable by the user, it'll become executable. - // This matches nix's dump() function behaviour. - executable: metadata.permissions().mode() & 64 != 0, - })); - } - - // Nix says things like: error: file '/home/raito/dev/code.tvl.fyi/tvix/glue/src/tests/import_fixtures/a_devnode' has an unsupported type - Err(Error::UnsupportedFileType( - entry.path().to_path_buf(), - file_type, - )) -} - -/// Walk the filesystem at a given path and returns a level-keyed list of directory entries. -/// -/// This is how [`ingest_path`] assembles the set of entries to pass on [`ingest_entries`]. -/// This low-level function can be used if additional filtering or processing is required on the -/// entries. -/// -/// Level here is in the context of graph theory, e.g. 2-level nodes -/// are nodes that are at depth 2. -/// -/// This function will walk the filesystem using `walkdir` and will consume -/// `O(#number of entries)` space. -#[instrument(fields(path), err)] -pub fn walk_path_for_ingestion<P>(path: P) -> Result<Vec<Vec<DirEntry>>, Error> -where - P: AsRef<Path> + std::fmt::Debug, -{ - let mut entries_per_depths: Vec<Vec<DirEntry>> = vec![Vec::new()]; - for entry in WalkDir::new(path.as_ref()) - .follow_links(false) - .follow_root_links(false) - .contents_first(false) - .sort_by_file_name() - .into_iter() - { - // Entry could be a NotFound, if the root path specified does not exist. - let entry = entry.map_err(|e| { - Error::UnableToOpen( - PathBuf::from(path.as_ref()), - e.into_io_error().expect("walkdir err must be some"), - ) - })?; - - if entry.depth() >= entries_per_depths.len() { - debug_assert!( - entry.depth() == entries_per_depths.len(), - "Received unexpected entry with depth {} during descent, previously at {}", - entry.depth(), - entries_per_depths.len() - ); - - entries_per_depths.push(vec![entry]); - } else { - entries_per_depths[entry.depth()].push(entry); - } - } - - Ok(entries_per_depths) -} - -/// Convert a leveled-key vector of filesystem entries into a stream of -/// [DirEntry] in a way that honors the Merkle invariant, i.e. from bottom to top. -pub fn leveled_entries_to_stream( - entries_per_depths: Vec<Vec<DirEntry>>, -) -> impl Stream<Item = DirEntry> { - stream! { - for level in entries_per_depths.into_iter().rev() { - for entry in level.into_iter() { - yield entry; - } - } - } -} - -/// Ingests the contents at a given path into the tvix store, interacting with a [BlobService] and -/// [DirectoryService]. It returns the root node or an error. -/// -/// It does not follow symlinks at the root, they will be ingested as actual symlinks. -#[instrument(skip(blob_service, directory_service), fields(path), err)] -pub async fn ingest_path<'a, BS, DS, P>( - blob_service: BS, - directory_service: DS, - path: P, -) -> Result<Node, Error> -where - P: AsRef<Path> + std::fmt::Debug, - BS: AsRef<dyn BlobService> + Clone, - DS: AsRef<dyn DirectoryService>, -{ - let entries = walk_path_for_ingestion(path)?; - let entries_stream = leveled_entries_to_stream(entries); - pin_mut!(entries_stream); - - ingest_entries(blob_service, directory_service, entries_stream).await -} - -/// The Merkle invariant checker is an internal structure to perform bookkeeping of all directory -/// entries we are ingesting and verifying we are ingesting them in the right order. -/// -/// That is, whenever we process an entry `L`, we would like to verify if we didn't process earlier -/// an entry `P` such that `P` is an **ancestor** of `L`. -/// -/// If such a thing happened, it means that we have processed something like: -/// -///```no_trust -/// A -/// / \ -/// B C -/// / \ \ -/// G F P <--------- processed before this one -/// / \ | -/// D E | -/// \ | -/// L <-----------------------------+ -/// ``` -/// -/// This is exactly what must never happen. -/// -/// Note: this checker is local, it can only see what happens on our side, not on the remote side, -/// i.e. the different remote services. -#[derive(Default)] -#[cfg(debug_assertions)] -struct MerkleInvariantChecker { - seen: HashSet<PathBuf>, -} - -#[cfg(debug_assertions)] -impl MerkleInvariantChecker { - /// See a directory entry and remember it. - fn see(&mut self, node: &DirEntry) { - self.seen.insert(node.path().to_owned()); - } - - /// Returns a potential ancestor already seen for that directory entry. - fn find_ancestor(&self, node: &DirEntry) -> Option<PathBuf> { - for anc in node.path().ancestors() { - if self.seen.contains(anc) { - return Some(anc.to_owned()); - } - } - - None - } -} - -/// Ingests elements from the given stream of [`DirEntry`] into a the passed [`BlobService`] and -/// [`DirectoryService`]. -/// It does not follow symlinks at the root, they will be ingested as actual symlinks. -#[instrument(skip_all, ret(level = Level::TRACE), err)] -pub async fn ingest_entries<'a, BS, DS, S>( - blob_service: BS, - directory_service: DS, - mut entries_async_iterator: S, -) -> Result<Node, Error> -where - BS: AsRef<dyn BlobService> + Clone, - DS: AsRef<dyn DirectoryService>, - S: Stream<Item = DirEntry> + std::marker::Unpin, -{ - let mut directories: HashMap<PathBuf, Directory> = HashMap::default(); - - let mut directory_putter = directory_service.as_ref().put_multiple_start(); - - #[cfg(debug_assertions)] - let mut invariant_checker: MerkleInvariantChecker = Default::default(); - - // We need to process a directory's children before processing - // the directory itself in order to have all the data needed - // to compute the hash. - while let Some(entry) = entries_async_iterator.next().await { - #[cfg(debug_assertions)] - { - // If we find an ancestor before we see this entry, this means that the caller - // broke the contract, refer to the documentation of the invariant checker to - // understand the reasoning here. - if let Some(ancestor) = invariant_checker.find_ancestor(&entry) { - panic!("Tvix bug: merkle invariant checker discovered that {} was processed before {}!", - ancestor.display(), - entry.path().display() - ); - } - - invariant_checker.see(&entry); - } - - // FUTUREWORK: handle directory putting here, rather than piping it through process_entry. - let node = process_entry( - blob_service.clone(), - &mut directory_putter, - &entry, - // process_entry wants an Option<Directory> in case the entry points to a directory. - // make sure to provide it. - // If the directory has contents, we already have it in - // `directories` because we iterate over depth in reverse order (deepest to - // shallowest). - if entry.file_type().is_dir() { - Some( - directories - .remove(entry.path()) - // In that case, it contained no children - .unwrap_or_default(), - ) - } else { - None - }, - ) - .await?; - - if entry.depth() == 0 { - // Make sure all the directories are flushed. - // FUTUREWORK: `debug_assert!` the resulting Ok(b3_digest) to be equal - // to `directories.get(entry.path())`. - if entry.file_type().is_dir() { - directory_putter.close().await?; - } - return Ok(node); - } else { - // calculate the parent path, and make sure we register the node there. - // NOTE: entry.depth() > 0 - let parent_path = entry.path().parent().unwrap().to_path_buf(); - - // record node in parent directory, creating a new [proto:Directory] if not there yet. - let parent_directory = directories.entry(parent_path).or_default(); - match node { - Node::Directory(e) => parent_directory.directories.push(e), - Node::File(e) => parent_directory.files.push(e), - Node::Symlink(e) => parent_directory.symlinks.push(e), - } - } - } - // unreachable, we already bailed out before if root doesn't exist. - unreachable!("Tvix bug: no root node emitted during ingestion") -} diff --git a/tvix/castore/src/import/archive.rs b/tvix/castore/src/import/archive.rs new file mode 100644 index 0000000000..0ebb4a2361 --- /dev/null +++ b/tvix/castore/src/import/archive.rs @@ -0,0 +1,458 @@ +//! Imports from an archive (tarballs) + +use std::collections::HashMap; +use std::io::{Cursor, Write}; +use std::sync::Arc; + +use petgraph::graph::{DiGraph, NodeIndex}; +use petgraph::visit::{DfsPostOrder, EdgeRef}; +use petgraph::Direction; +use tokio::io::AsyncRead; +use tokio::sync::Semaphore; +use tokio::task::JoinSet; +use tokio_stream::StreamExt; +use tokio_tar::Archive; +use tokio_util::io::InspectReader; +use tracing::{instrument, warn, Level}; + +use crate::blobservice::BlobService; +use crate::directoryservice::DirectoryService; +use crate::import::{ingest_entries, IngestionEntry, IngestionError}; +use crate::proto::node::Node; +use crate::B3Digest; + +type TarPathBuf = std::path::PathBuf; + +/// Files smaller than this threshold, in bytes, are uploaded to the [BlobService] in the +/// background. +/// +/// This is a u32 since we acquire a weighted semaphore using the size of the blob. +/// [Semaphore::acquire_many_owned] takes a u32, so we need to ensure the size of +/// the blob can be represented using a u32 and will not cause an overflow. +const CONCURRENT_BLOB_UPLOAD_THRESHOLD: u32 = 1024 * 1024; + +/// The maximum amount of bytes allowed to be buffered in memory to perform async blob uploads. +const MAX_TARBALL_BUFFER_SIZE: usize = 128 * 1024 * 1024; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("unable to construct stream of entries: {0}")] + Entries(std::io::Error), + + #[error("unable to read next entry: {0}")] + NextEntry(std::io::Error), + + #[error("unable to read path for entry: {0}")] + PathRead(std::io::Error), + + #[error("unable to convert path {0} for entry: {1}")] + PathConvert(TarPathBuf, std::io::Error), + + #[error("unable to read size field for {0}: {1}")] + Size(TarPathBuf, std::io::Error), + + #[error("unable to read mode field for {0}: {1}")] + Mode(TarPathBuf, std::io::Error), + + #[error("unable to read link name field for {0}: {1}")] + LinkName(TarPathBuf, std::io::Error), + + #[error("unable to read blob contents for {0}: {1}")] + BlobRead(TarPathBuf, std::io::Error), + + // FUTUREWORK: proper error for blob finalize + #[error("unable to finalize blob {0}: {1}")] + BlobFinalize(TarPathBuf, std::io::Error), + + #[error("unsupported tar entry {0} type: {1:?}")] + EntryType(TarPathBuf, tokio_tar::EntryType), + + #[error("symlink missing target {0}")] + MissingSymlinkTarget(TarPathBuf), + + #[error("unexpected number of top level directory entries")] + UnexpectedNumberOfTopLevelEntries, +} + +/// Ingests elements from the given tar [`Archive`] into a the passed [`BlobService`] and +/// [`DirectoryService`]. +#[instrument(skip_all, ret(level = Level::TRACE), err)] +pub async fn ingest_archive<BS, DS, R>( + blob_service: BS, + directory_service: DS, + mut archive: Archive<R>, +) -> Result<Node, IngestionError<Error>> +where + BS: BlobService + Clone + 'static, + DS: DirectoryService, + R: AsyncRead + Unpin, +{ + // Since tarballs can have entries in any arbitrary order, we need to + // buffer all of the directory metadata so we can reorder directory + // contents and entries to meet the requires of the castore. + + // In the first phase, collect up all the regular files and symlinks. + let mut nodes = IngestionEntryGraph::new(); + + let semaphore = Arc::new(Semaphore::new(MAX_TARBALL_BUFFER_SIZE)); + let mut async_blob_uploads: JoinSet<Result<(), Error>> = JoinSet::new(); + + let mut entries_iter = archive.entries().map_err(Error::Entries)?; + while let Some(mut entry) = entries_iter.try_next().await.map_err(Error::NextEntry)? { + let tar_path: TarPathBuf = entry.path().map_err(Error::PathRead)?.into(); + + // construct a castore PathBuf, which we use in the produced IngestionEntry. + let path = crate::path::PathBuf::from_host_path(tar_path.as_path(), true) + .map_err(|e| Error::PathConvert(tar_path.clone(), e))?; + + let header = entry.header(); + let entry = match header.entry_type() { + tokio_tar::EntryType::Regular + | tokio_tar::EntryType::GNUSparse + | tokio_tar::EntryType::Continuous => { + let header_size = header + .size() + .map_err(|e| Error::Size(tar_path.clone(), e))?; + + // If the blob is small enough, read it off the wire, compute the digest, + // and upload it to the [BlobService] in the background. + let (size, digest) = if header_size <= CONCURRENT_BLOB_UPLOAD_THRESHOLD as u64 { + let mut buffer = Vec::with_capacity(header_size as usize); + let mut hasher = blake3::Hasher::new(); + let mut reader = InspectReader::new(&mut entry, |bytes| { + hasher.write_all(bytes).unwrap(); + }); + + // Ensure that we don't buffer into memory until we've acquired a permit. + // This prevents consuming too much memory when performing concurrent + // blob uploads. + let permit = semaphore + .clone() + // This cast is safe because ensure the header_size is less than + // CONCURRENT_BLOB_UPLOAD_THRESHOLD which is a u32. + .acquire_many_owned(header_size as u32) + .await + .unwrap(); + let size = tokio::io::copy(&mut reader, &mut buffer) + .await + .map_err(|e| Error::Size(tar_path.clone(), e))?; + + let digest: B3Digest = hasher.finalize().as_bytes().into(); + + { + let blob_service = blob_service.clone(); + let digest = digest.clone(); + async_blob_uploads.spawn({ + let tar_path = tar_path.clone(); + async move { + let mut writer = blob_service.open_write().await; + + tokio::io::copy(&mut Cursor::new(buffer), &mut writer) + .await + .map_err(|e| Error::BlobRead(tar_path.clone(), e))?; + + let blob_digest = writer + .close() + .await + .map_err(|e| Error::BlobFinalize(tar_path, e))?; + + assert_eq!(digest, blob_digest, "Tvix bug: blob digest mismatch"); + + // Make sure we hold the permit until we finish writing the blob + // to the [BlobService]. + drop(permit); + Ok(()) + } + }); + } + + (size, digest) + } else { + let mut writer = blob_service.open_write().await; + + let size = tokio::io::copy(&mut entry, &mut writer) + .await + .map_err(|e| Error::BlobRead(tar_path.clone(), e))?; + + let digest = writer + .close() + .await + .map_err(|e| Error::BlobFinalize(tar_path.clone(), e))?; + + (size, digest) + }; + + let executable = entry + .header() + .mode() + .map_err(|e| Error::Mode(tar_path, e))? + & 64 + != 0; + + IngestionEntry::Regular { + path, + size, + executable, + digest, + } + } + tokio_tar::EntryType::Symlink => IngestionEntry::Symlink { + target: entry + .link_name() + .map_err(|e| Error::LinkName(tar_path.clone(), e))? + .ok_or_else(|| Error::MissingSymlinkTarget(tar_path.clone()))? + .into_owned() + .into_os_string() + .into_encoded_bytes(), + path, + }, + // Push a bogus directory marker so we can make sure this directoy gets + // created. We don't know the digest and size until after reading the full + // tarball. + tokio_tar::EntryType::Directory => IngestionEntry::Dir { path }, + + tokio_tar::EntryType::XGlobalHeader | tokio_tar::EntryType::XHeader => continue, + + entry_type => return Err(Error::EntryType(tar_path, entry_type).into()), + }; + + nodes.add(entry)?; + } + + while let Some(result) = async_blob_uploads.join_next().await { + result.expect("task panicked")?; + } + + let root_node = ingest_entries( + directory_service, + futures::stream::iter(nodes.finalize()?.into_iter().map(Ok)), + ) + .await?; + + Ok(root_node) +} + +/// Keep track of the directory structure of a file tree being ingested. This is used +/// for ingestion sources which do not provide any ordering or uniqueness guarantees +/// like tarballs. +/// +/// If we ingest multiple entries with the same paths and both entries are not directories, +/// the newer entry will replace the latter entry, disconnecting the old node's children +/// from the graph. +/// +/// Once all nodes are ingested a call to [IngestionEntryGraph::finalize] will return +/// a list of entries compute by performaing a DFS post order traversal of the graph +/// from the top-level directory entry. +/// +/// This expects the directory structure to contain a single top-level directory entry. +/// An error is returned if this is not the case and ingestion will fail. +struct IngestionEntryGraph { + graph: DiGraph<IngestionEntry, ()>, + path_to_index: HashMap<crate::path::PathBuf, NodeIndex>, + root_node: Option<NodeIndex>, +} + +impl Default for IngestionEntryGraph { + fn default() -> Self { + Self::new() + } +} + +impl IngestionEntryGraph { + /// Creates a new ingestion entry graph. + pub fn new() -> Self { + IngestionEntryGraph { + graph: DiGraph::new(), + path_to_index: HashMap::new(), + root_node: None, + } + } + + /// Adds a new entry to the graph. Parent directories are automatically inserted. + /// If a node exists in the graph with the same name as the new entry and both the old + /// and new nodes are not directories, the node is replaced and is disconnected from its + /// children. + pub fn add(&mut self, entry: IngestionEntry) -> Result<NodeIndex, Error> { + let path = entry.path().to_owned(); + + let index = match self.path_to_index.get(entry.path()) { + Some(&index) => { + // If either the old entry or new entry are not directories, we'll replace the old + // entry. + if !entry.is_dir() || !self.get_node(index).is_dir() { + self.replace_node(index, entry); + } + + index + } + None => self.graph.add_node(entry), + }; + + // for archives, a path with 1 component is the root node + if path.components().count() == 1 { + // We expect archives to contain a single root node, if there is another root node + // entry with a different path name, this is unsupported. + if let Some(root_node) = self.root_node { + if self.get_node(root_node).path() != path.as_ref() { + return Err(Error::UnexpectedNumberOfTopLevelEntries); + } + } + + self.root_node = Some(index) + } else if let Some(parent_path) = path.parent() { + // Recursively add the parent node until it hits the root node. + let parent_index = self.add(IngestionEntry::Dir { + path: parent_path.to_owned(), + })?; + + // Insert an edge from the parent directory to the child entry. + self.graph.add_edge(parent_index, index, ()); + } + + self.path_to_index.insert(path, index); + + Ok(index) + } + + /// Traverses the graph in DFS post order and collects the entries into a [Vec<IngestionEntry>]. + /// + /// Unreachable parts of the graph are not included in the result. + pub fn finalize(self) -> Result<Vec<IngestionEntry>, Error> { + // There must be a root node. + let Some(root_node_index) = self.root_node else { + return Err(Error::UnexpectedNumberOfTopLevelEntries); + }; + + // The root node must be a directory. + if !self.get_node(root_node_index).is_dir() { + return Err(Error::UnexpectedNumberOfTopLevelEntries); + } + + let mut traversal = DfsPostOrder::new(&self.graph, root_node_index); + let mut nodes = Vec::with_capacity(self.graph.node_count()); + while let Some(node_index) = traversal.next(&self.graph) { + nodes.push(self.get_node(node_index).clone()); + } + + Ok(nodes) + } + + /// Replaces the node with the specified entry. The node's children are disconnected. + /// + /// This should never be called if both the old and new nodes are directories. + fn replace_node(&mut self, index: NodeIndex, new_entry: IngestionEntry) { + let entry = self + .graph + .node_weight_mut(index) + .expect("Tvix bug: missing node entry"); + + debug_assert!(!(entry.is_dir() && new_entry.is_dir())); + + // Replace the node itself. + warn!( + "saw duplicate entry in archive at path {:?}. old: {:?} new: {:?}", + entry.path(), + &entry, + &new_entry + ); + *entry = new_entry; + + // Remove any outgoing edges to disconnect the old node's children. + let edges = self + .graph + .edges_directed(index, Direction::Outgoing) + .map(|edge| edge.id()) + .collect::<Vec<_>>(); + for edge in edges { + self.graph.remove_edge(edge); + } + } + + fn get_node(&self, index: NodeIndex) -> &IngestionEntry { + self.graph + .node_weight(index) + .expect("Tvix bug: missing node entry") + } +} + +#[cfg(test)] +mod test { + use crate::import::IngestionEntry; + use crate::B3Digest; + + use super::{Error, IngestionEntryGraph}; + + use lazy_static::lazy_static; + use rstest::rstest; + + lazy_static! { + pub static ref EMPTY_DIGEST: B3Digest = blake3::hash(&[]).as_bytes().into(); + pub static ref DIR_A: IngestionEntry = IngestionEntry::Dir { + path: "a".parse().unwrap() + }; + pub static ref DIR_B: IngestionEntry = IngestionEntry::Dir { + path: "b".parse().unwrap() + }; + pub static ref DIR_A_B: IngestionEntry = IngestionEntry::Dir { + path: "a/b".parse().unwrap() + }; + pub static ref FILE_A: IngestionEntry = IngestionEntry::Regular { + path: "a".parse().unwrap(), + size: 0, + executable: false, + digest: EMPTY_DIGEST.clone(), + }; + pub static ref FILE_A_B: IngestionEntry = IngestionEntry::Regular { + path: "a/b".parse().unwrap(), + size: 0, + executable: false, + digest: EMPTY_DIGEST.clone(), + }; + pub static ref FILE_A_B_C: IngestionEntry = IngestionEntry::Regular { + path: "a/b/c".parse().unwrap(), + size: 0, + executable: false, + digest: EMPTY_DIGEST.clone(), + }; + } + + #[rstest] + #[case::implicit_directories(&[&*FILE_A_B_C], &[&*FILE_A_B_C, &*DIR_A_B, &*DIR_A])] + #[case::explicit_directories(&[&*DIR_A, &*DIR_A_B, &*FILE_A_B_C], &[&*FILE_A_B_C, &*DIR_A_B, &*DIR_A])] + #[case::inaccesible_tree(&[&*DIR_A, &*DIR_A_B, &*FILE_A_B], &[&*FILE_A_B, &*DIR_A])] + fn node_ingestion_success( + #[case] in_entries: &[&IngestionEntry], + #[case] exp_entries: &[&IngestionEntry], + ) { + let mut nodes = IngestionEntryGraph::new(); + + for entry in in_entries { + nodes.add((*entry).clone()).expect("failed to add entry"); + } + + let entries = nodes.finalize().expect("invalid entries"); + + let exp_entries: Vec<IngestionEntry> = + exp_entries.iter().map(|entry| (*entry).clone()).collect(); + + assert_eq!(entries, exp_entries); + } + + #[rstest] + #[case::no_top_level_entries(&[], Error::UnexpectedNumberOfTopLevelEntries)] + #[case::multiple_top_level_dirs(&[&*DIR_A, &*DIR_B], Error::UnexpectedNumberOfTopLevelEntries)] + #[case::top_level_file_entry(&[&*FILE_A], Error::UnexpectedNumberOfTopLevelEntries)] + fn node_ingestion_error(#[case] in_entries: &[&IngestionEntry], #[case] exp_error: Error) { + let mut nodes = IngestionEntryGraph::new(); + + let result = (|| { + for entry in in_entries { + nodes.add((*entry).clone())?; + } + nodes.finalize() + })(); + + let error = result.expect_err("expected error"); + assert_eq!(error.to_string(), exp_error.to_string()); + } +} diff --git a/tvix/castore/src/import/error.rs b/tvix/castore/src/import/error.rs new file mode 100644 index 0000000000..e3fba617e0 --- /dev/null +++ b/tvix/castore/src/import/error.rs @@ -0,0 +1,20 @@ +use super::PathBuf; + +use crate::Error as CastoreError; + +/// Represents all error types that emitted by ingest_entries. +/// It can represent errors uploading individual Directories and finalizing +/// the upload. +/// It also contains a generic error kind that'll carry ingestion-method +/// specific errors. +#[derive(Debug, thiserror::Error)] +pub enum IngestionError<E: std::fmt::Display> { + #[error("error from producer: {0}")] + Producer(#[from] E), + + #[error("failed to upload directory at {0}: {1}")] + UploadDirectoryError(PathBuf, CastoreError), + + #[error("failed to finalize directory upload: {0}")] + FinalizeDirectoryUpload(CastoreError), +} diff --git a/tvix/castore/src/import/fs.rs b/tvix/castore/src/import/fs.rs new file mode 100644 index 0000000000..9d3ecfe6ab --- /dev/null +++ b/tvix/castore/src/import/fs.rs @@ -0,0 +1,185 @@ +//! Import from a real filesystem. + +use futures::stream::BoxStream; +use futures::StreamExt; +use std::fs::FileType; +use std::os::unix::ffi::OsStringExt; +use std::os::unix::fs::MetadataExt; +use std::os::unix::fs::PermissionsExt; +use tracing::instrument; +use walkdir::DirEntry; +use walkdir::WalkDir; + +use crate::blobservice::BlobService; +use crate::directoryservice::DirectoryService; +use crate::proto::node::Node; +use crate::B3Digest; + +use super::ingest_entries; +use super::IngestionEntry; +use super::IngestionError; + +/// Ingests the contents at a given path into the tvix store, interacting with a [BlobService] and +/// [DirectoryService]. It returns the root node or an error. +/// +/// It does not follow symlinks at the root, they will be ingested as actual symlinks. +/// +/// This function will walk the filesystem using `walkdir` and will consume +/// `O(#number of entries)` space. +#[instrument(skip(blob_service, directory_service), fields(path), err)] +pub async fn ingest_path<BS, DS, P>( + blob_service: BS, + directory_service: DS, + path: P, +) -> Result<Node, IngestionError<Error>> +where + P: AsRef<std::path::Path> + std::fmt::Debug, + BS: BlobService + Clone, + DS: DirectoryService, +{ + let iter = WalkDir::new(path.as_ref()) + .follow_links(false) + .follow_root_links(false) + .contents_first(true) + .into_iter(); + + let entries = dir_entries_to_ingestion_stream(blob_service, iter, path.as_ref()); + ingest_entries(directory_service, entries).await +} + +/// Converts an iterator of [walkdir::DirEntry]s into a stream of ingestion entries. +/// This can then be fed into [ingest_entries] to ingest all the entries into the castore. +/// +/// The produced stream is buffered, so uploads can happen concurrently. +/// +/// The root is the [Path] in the filesystem that is being ingested into the castore. +pub fn dir_entries_to_ingestion_stream<'a, BS, I>( + blob_service: BS, + iter: I, + root: &'a std::path::Path, +) -> BoxStream<'a, Result<IngestionEntry, Error>> +where + BS: BlobService + Clone + 'a, + I: Iterator<Item = Result<DirEntry, walkdir::Error>> + Send + 'a, +{ + let prefix = root.parent().unwrap_or_else(|| std::path::Path::new("")); + + Box::pin( + futures::stream::iter(iter) + .map(move |x| { + let blob_service = blob_service.clone(); + async move { + match x { + Ok(dir_entry) => { + dir_entry_to_ingestion_entry(blob_service, &dir_entry, prefix).await + } + Err(e) => Err(Error::Stat( + prefix.to_path_buf(), + e.into_io_error().expect("walkdir err must be some"), + )), + } + } + }) + .buffered(50), + ) +} + +/// Converts a [walkdir::DirEntry] into an [IngestionEntry], uploading blobs to the +/// provided [BlobService]. +/// +/// The prefix path is stripped from the path of each entry. This is usually the parent path +/// of the path being ingested so that the last element of the stream only has one component. +pub async fn dir_entry_to_ingestion_entry<BS>( + blob_service: BS, + entry: &DirEntry, + prefix: &std::path::Path, +) -> Result<IngestionEntry, Error> +where + BS: BlobService, +{ + let file_type = entry.file_type(); + + let fs_path = entry + .path() + .strip_prefix(prefix) + .expect("Tvix bug: failed to strip root path prefix"); + + // convert to castore PathBuf + let path = crate::path::PathBuf::from_host_path(fs_path, false) + .unwrap_or_else(|e| panic!("Tvix bug: walkdir direntry cannot be parsed: {}", e)); + + if file_type.is_dir() { + Ok(IngestionEntry::Dir { path }) + } else if file_type.is_symlink() { + let target = std::fs::read_link(entry.path()) + .map_err(|e| Error::Stat(entry.path().to_path_buf(), e))? + .into_os_string() + .into_vec(); + + Ok(IngestionEntry::Symlink { path, target }) + } else if file_type.is_file() { + let metadata = entry + .metadata() + .map_err(|e| Error::Stat(entry.path().to_path_buf(), e.into()))?; + + let digest = upload_blob(blob_service, entry.path().to_path_buf()).await?; + + Ok(IngestionEntry::Regular { + path, + size: metadata.size(), + // If it's executable by the user, it'll become executable. + // This matches nix's dump() function behaviour. + executable: metadata.permissions().mode() & 64 != 0, + digest, + }) + } else { + return Err(Error::FileType(fs_path.to_path_buf(), file_type)); + } +} + +/// Uploads the file at the provided [Path] the the [BlobService]. +#[instrument(skip(blob_service), fields(path), err)] +async fn upload_blob<BS>( + blob_service: BS, + path: impl AsRef<std::path::Path>, +) -> Result<B3Digest, Error> +where + BS: BlobService, +{ + let mut file = match tokio::fs::File::open(path.as_ref()).await { + Ok(file) => file, + Err(e) => return Err(Error::BlobRead(path.as_ref().to_path_buf(), e)), + }; + + let mut writer = blob_service.open_write().await; + + if let Err(e) = tokio::io::copy(&mut file, &mut writer).await { + return Err(Error::BlobRead(path.as_ref().to_path_buf(), e)); + }; + + let digest = writer + .close() + .await + .map_err(|e| Error::BlobFinalize(path.as_ref().to_path_buf(), e))?; + + Ok(digest) +} + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("unsupported file type at {0}: {1:?}")] + FileType(std::path::PathBuf, FileType), + + #[error("unable to stat {0}: {1}")] + Stat(std::path::PathBuf, std::io::Error), + + #[error("unable to open {0}: {1}")] + Open(std::path::PathBuf, std::io::Error), + + #[error("unable to read {0}: {1}")] + BlobRead(std::path::PathBuf, std::io::Error), + + // TODO: proper error for blob finalize + #[error("unable to finalize blob {0}: {1}")] + BlobFinalize(std::path::PathBuf, std::io::Error), +} diff --git a/tvix/castore/src/import/mod.rs b/tvix/castore/src/import/mod.rs new file mode 100644 index 0000000000..e8b27e469c --- /dev/null +++ b/tvix/castore/src/import/mod.rs @@ -0,0 +1,340 @@ +//! The main library function here is [ingest_entries], receiving a stream of +//! [IngestionEntry]. +//! +//! Specific implementations, such as ingesting from the filesystem, live in +//! child modules. + +use crate::directoryservice::DirectoryPutter; +use crate::directoryservice::DirectoryService; +use crate::path::{Path, PathBuf}; +use crate::proto::node::Node; +use crate::proto::Directory; +use crate::proto::DirectoryNode; +use crate::proto::FileNode; +use crate::proto::SymlinkNode; +use crate::B3Digest; +use futures::{Stream, StreamExt}; + +use tracing::Level; + +use std::collections::HashMap; +use tracing::instrument; + +mod error; +pub use error::IngestionError; + +pub mod archive; +pub mod fs; + +/// Ingests [IngestionEntry] from the given stream into a the passed [DirectoryService]. +/// On success, returns the root [Node]. +/// +/// The stream must have the following invariants: +/// - All children entries must come before their parents. +/// - The last entry must be the root node which must have a single path component. +/// - Every entry should have a unique path, and only consist of normal components. +/// This means, no windows path prefixes, absolute paths, `.` or `..`. +/// - All referenced directories must have an associated directory entry in the stream. +/// This means if there is a file entry for `foo/bar`, there must also be a `foo` directory +/// entry. +/// +/// Internally we maintain a [HashMap] of [PathBuf] to partially populated [Directory] at that +/// path. Once we receive an [IngestionEntry] for the directory itself, we remove it from the +/// map and upload it to the [DirectoryService] through a lazily created [DirectoryPutter]. +/// +/// On success, returns the root node. +#[instrument(skip_all, ret(level = Level::TRACE), err)] +pub async fn ingest_entries<DS, S, E>( + directory_service: DS, + mut entries: S, +) -> Result<Node, IngestionError<E>> +where + DS: DirectoryService, + S: Stream<Item = Result<IngestionEntry, E>> + Send + std::marker::Unpin, + E: std::error::Error, +{ + // For a given path, this holds the [Directory] structs as they are populated. + let mut directories: HashMap<PathBuf, Directory> = HashMap::default(); + let mut maybe_directory_putter: Option<Box<dyn DirectoryPutter>> = None; + + let root_node = loop { + let mut entry = entries + .next() + .await + // The last entry of the stream must have 1 path component, after which + // we break the loop manually. + .expect("Tvix bug: unexpected end of stream")?; + + let name = entry + .path() + .file_name() + // If this is the root node, it will have an empty name. + .unwrap_or_default() + .to_owned() + .into(); + + let node = match &mut entry { + IngestionEntry::Dir { .. } => { + // If the entry is a directory, we traversed all its children (and + // populated it in `directories`). + // If we don't have it in directories, it's a directory without + // children. + let directory = directories + .remove(entry.path()) + // In that case, it contained no children + .unwrap_or_default(); + + let directory_size = directory.size(); + let directory_digest = directory.digest(); + + // Use the directory_putter to upload the directory. + // If we don't have one yet (as that's the first one to upload), + // initialize the putter. + maybe_directory_putter + .get_or_insert_with(|| directory_service.put_multiple_start()) + .put(directory) + .await + .map_err(|e| { + IngestionError::UploadDirectoryError(entry.path().to_owned(), e) + })?; + + Node::Directory(DirectoryNode { + name, + digest: directory_digest.into(), + size: directory_size, + }) + } + IngestionEntry::Symlink { ref target, .. } => Node::Symlink(SymlinkNode { + name, + target: target.to_owned().into(), + }), + IngestionEntry::Regular { + size, + executable, + digest, + .. + } => Node::File(FileNode { + name, + digest: digest.to_owned().into(), + size: *size, + executable: *executable, + }), + }; + + let parent = entry + .path() + .parent() + .expect("Tvix bug: got entry with root node"); + + if parent == crate::Path::ROOT { + break node; + } else { + // record node in parent directory, creating a new [Directory] if not there yet. + directories.entry(parent.to_owned()).or_default().add(node); + } + }; + + assert!( + entries.count().await == 0, + "Tvix bug: left over elements in the stream" + ); + + assert!( + directories.is_empty(), + "Tvix bug: left over directories after processing ingestion stream" + ); + + // if there were directories uploaded, make sure we flush the putter, so + // they're all persisted to the backend. + if let Some(mut directory_putter) = maybe_directory_putter { + #[cfg_attr(not(debug_assertions), allow(unused))] + let root_directory_digest = directory_putter + .close() + .await + .map_err(|e| IngestionError::FinalizeDirectoryUpload(e))?; + + #[cfg(debug_assertions)] + { + if let Node::Directory(directory_node) = &root_node { + debug_assert_eq!( + root_directory_digest, + directory_node + .digest + .to_vec() + .try_into() + .expect("invalid digest len") + ) + } else { + unreachable!("Tvix bug: directory putter initialized but no root directory node"); + } + } + }; + + Ok(root_node) +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum IngestionEntry { + Regular { + path: PathBuf, + size: u64, + executable: bool, + digest: B3Digest, + }, + Symlink { + path: PathBuf, + target: Vec<u8>, + }, + Dir { + path: PathBuf, + }, +} + +impl IngestionEntry { + fn path(&self) -> &Path { + match self { + IngestionEntry::Regular { path, .. } => path, + IngestionEntry::Symlink { path, .. } => path, + IngestionEntry::Dir { path } => path, + } + } + + fn is_dir(&self) -> bool { + matches!(self, IngestionEntry::Dir { .. }) + } +} + +#[cfg(test)] +mod test { + use rstest::rstest; + + use crate::fixtures::{DIRECTORY_COMPLICATED, DIRECTORY_WITH_KEEP, EMPTY_BLOB_DIGEST}; + use crate::proto::node::Node; + use crate::proto::{Directory, DirectoryNode, FileNode, SymlinkNode}; + use crate::{directoryservice::MemoryDirectoryService, fixtures::DUMMY_DIGEST}; + + use super::ingest_entries; + use super::IngestionEntry; + + #[rstest] + #[case::single_file(vec![IngestionEntry::Regular { + path: "foo".parse().unwrap(), + size: 42, + executable: true, + digest: DUMMY_DIGEST.clone(), + }], + Node::File(FileNode { name: "foo".into(), digest: DUMMY_DIGEST.clone().into(), size: 42, executable: true } + ))] + #[case::single_symlink(vec![IngestionEntry::Symlink { + path: "foo".parse().unwrap(), + target: b"blub".into(), + }], + Node::Symlink(SymlinkNode { name: "foo".into(), target: "blub".into()}) + )] + #[case::single_dir(vec![IngestionEntry::Dir { + path: "foo".parse().unwrap(), + }], + Node::Directory(DirectoryNode { name: "foo".into(), digest: Directory::default().digest().into(), size: Directory::default().size()}) + )] + #[case::dir_with_keep(vec![ + IngestionEntry::Regular { + path: "foo/.keep".parse().unwrap(), + size: 0, + executable: false, + digest: EMPTY_BLOB_DIGEST.clone(), + }, + IngestionEntry::Dir { + path: "foo".parse().unwrap(), + }, + ], + Node::Directory(DirectoryNode { name: "foo".into(), digest: DIRECTORY_WITH_KEEP.digest().into(), size: DIRECTORY_WITH_KEEP.size() }) + )] + /// This is intentionally a bit unsorted, though it still satisfies all + /// requirements we have on the order of elements in the stream. + #[case::directory_complicated(vec![ + IngestionEntry::Regular { + path: "blub/.keep".parse().unwrap(), + size: 0, + executable: false, + digest: EMPTY_BLOB_DIGEST.clone(), + }, + IngestionEntry::Regular { + path: "blub/keep/.keep".parse().unwrap(), + size: 0, + executable: false, + digest: EMPTY_BLOB_DIGEST.clone(), + }, + IngestionEntry::Dir { + path: "blub/keep".parse().unwrap(), + }, + IngestionEntry::Symlink { + path: "blub/aa".parse().unwrap(), + target: b"/nix/store/somewhereelse".into(), + }, + IngestionEntry::Dir { + path: "blub".parse().unwrap(), + }, + ], + Node::Directory(DirectoryNode { name: "blub".into(), digest: DIRECTORY_COMPLICATED.digest().into(), size:DIRECTORY_COMPLICATED.size() }) + )] + #[tokio::test] + async fn test_ingestion(#[case] entries: Vec<IngestionEntry>, #[case] exp_root_node: Node) { + let directory_service = MemoryDirectoryService::default(); + + let root_node = ingest_entries( + directory_service.clone(), + futures::stream::iter(entries.into_iter().map(Ok::<_, std::io::Error>)), + ) + .await + .expect("must succeed"); + + assert_eq!(exp_root_node, root_node, "root node should match"); + } + + #[rstest] + #[should_panic] + #[case::empty_entries(vec![])] + #[should_panic] + #[case::missing_intermediate_dir(vec![ + IngestionEntry::Regular { + path: "blub/.keep".parse().unwrap(), + size: 0, + executable: false, + digest: EMPTY_BLOB_DIGEST.clone(), + }, + ])] + #[should_panic] + #[case::leaf_after_parent(vec![ + IngestionEntry::Dir { + path: "blub".parse().unwrap(), + }, + IngestionEntry::Regular { + path: "blub/.keep".parse().unwrap(), + size: 0, + executable: false, + digest: EMPTY_BLOB_DIGEST.clone(), + }, + ])] + #[should_panic] + #[case::root_in_entry(vec![ + IngestionEntry::Regular { + path: ".keep".parse().unwrap(), + size: 0, + executable: false, + digest: EMPTY_BLOB_DIGEST.clone(), + }, + IngestionEntry::Dir { + path: "".parse().unwrap(), + }, + ])] + #[tokio::test] + async fn test_ingestion_fail(#[case] entries: Vec<IngestionEntry>) { + let directory_service = MemoryDirectoryService::default(); + + let _ = ingest_entries( + directory_service.clone(), + futures::stream::iter(entries.into_iter().map(Ok::<_, std::io::Error>)), + ) + .await; + } +} diff --git a/tvix/castore/src/lib.rs b/tvix/castore/src/lib.rs index 1a7ac6b4b4..bdc533a8c5 100644 --- a/tvix/castore/src/lib.rs +++ b/tvix/castore/src/lib.rs @@ -9,6 +9,9 @@ pub mod fixtures; #[cfg(feature = "fs")] pub mod fs; +mod path; +pub use path::{Path, PathBuf}; + pub mod import; pub mod proto; pub mod tonic; diff --git a/tvix/castore/src/path.rs b/tvix/castore/src/path.rs new file mode 100644 index 0000000000..fcc2bd01fb --- /dev/null +++ b/tvix/castore/src/path.rs @@ -0,0 +1,446 @@ +//! Contains data structures to deal with Paths in the tvix-castore model. + +use std::{ + borrow::Borrow, + fmt::{self, Debug, Display}, + mem, + ops::Deref, + str::FromStr, +}; + +use bstr::ByteSlice; + +use crate::proto::validate_node_name; + +/// Represents a Path in the castore model. +/// These are always relative, and platform-independent, which distinguishes +/// them from the ones provided in the standard library. +#[derive(Eq, Hash, PartialEq)] +#[repr(transparent)] // SAFETY: Representation has to match [u8] +pub struct Path { + // As node names in the castore model cannot contain slashes, + // we use them as component separators here. + inner: [u8], +} + +#[allow(dead_code)] +impl Path { + // SAFETY: The empty path is valid. + pub const ROOT: &'static Path = unsafe { Path::from_bytes_unchecked(&[]) }; + + /// Convert a byte slice to a path, without checking validity. + const unsafe fn from_bytes_unchecked(bytes: &[u8]) -> &Path { + // SAFETY: &[u8] and &Path have the same representation. + unsafe { mem::transmute(bytes) } + } + + fn from_bytes(bytes: &[u8]) -> Option<&Path> { + if !bytes.is_empty() { + // Ensure all components are valid castore node names. + for component in bytes.split_str(b"/") { + validate_node_name(component).ok()?; + } + } + + // SAFETY: We have verified that the path contains no empty components. + Some(unsafe { Path::from_bytes_unchecked(bytes) }) + } + + pub fn into_boxed_bytes(self: Box<Path>) -> Box<[u8]> { + // SAFETY: Box<Path> and Box<[u8]> have the same representation. + unsafe { mem::transmute(self) } + } + + /// Returns the path without its final component, if there is one. + /// + /// Note that the parent of a bare file name is [Path::ROOT]. + /// [Path::ROOT] is the only path without a parent. + pub fn parent(&self) -> Option<&Path> { + // The root does not have a parent. + if self.inner.is_empty() { + return None; + } + + Some( + if let Some((parent, _file_name)) = self.inner.rsplit_once_str(b"/") { + // SAFETY: The parent of a valid Path is a valid Path. + unsafe { Path::from_bytes_unchecked(parent) } + } else { + // The parent of a bare file name is the root. + Path::ROOT + }, + ) + } + + /// Creates a PathBuf with `name` adjoined to self. + pub fn try_join(&self, name: &[u8]) -> Result<PathBuf, std::io::Error> { + let mut v = PathBuf::with_capacity(self.inner.len() + name.len() + 1); + v.inner.extend_from_slice(&self.inner); + v.try_push(name)?; + + Ok(v) + } + + /// Produces an iterator over the components of the path, which are + /// individual byte slices. + /// In case the path is empty, an empty iterator is returned. + pub fn components(&self) -> impl Iterator<Item = &[u8]> { + let mut iter = self.inner.split_str(&b"/"); + + // We don't want to return an empty element, consume it if it's the only one. + if self.inner.is_empty() { + let _ = iter.next(); + } + + iter + } + + /// Returns the final component of the Path, if there is one. + pub fn file_name(&self) -> Option<&[u8]> { + self.components().last() + } + + pub fn as_bytes(&self) -> &[u8] { + &self.inner + } +} + +impl Debug for Path { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(self.inner.as_bstr(), f) + } +} + +impl Display for Path { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(self.inner.as_bstr(), f) + } +} + +impl AsRef<Path> for Path { + fn as_ref(&self) -> &Path { + self + } +} + +/// Represents a owned PathBuf in the castore model. +/// These are always relative, and platform-independent, which distinguishes +/// them from the ones provided in the standard library. +#[derive(Clone, Default, Eq, Hash, PartialEq)] +pub struct PathBuf { + inner: Vec<u8>, +} + +impl Deref for PathBuf { + type Target = Path; + + fn deref(&self) -> &Self::Target { + // SAFETY: PathBuf always contains a valid Path. + unsafe { Path::from_bytes_unchecked(&self.inner) } + } +} + +impl AsRef<Path> for PathBuf { + fn as_ref(&self) -> &Path { + self + } +} + +impl ToOwned for Path { + type Owned = PathBuf; + + fn to_owned(&self) -> Self::Owned { + PathBuf { + inner: self.inner.to_owned(), + } + } +} + +impl Borrow<Path> for PathBuf { + fn borrow(&self) -> &Path { + self + } +} + +impl From<Box<Path>> for PathBuf { + fn from(value: Box<Path>) -> Self { + // SAFETY: Box<Path> is always a valid path. + unsafe { PathBuf::from_bytes_unchecked(value.into_boxed_bytes().into_vec()) } + } +} + +impl From<&Path> for PathBuf { + fn from(value: &Path) -> Self { + value.to_owned() + } +} + +impl FromStr for PathBuf { + type Err = std::io::Error; + + fn from_str(s: &str) -> Result<PathBuf, Self::Err> { + Ok(Path::from_bytes(s.as_bytes()) + .ok_or(std::io::ErrorKind::InvalidData)? + .to_owned()) + } +} + +impl Debug for PathBuf { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&**self, f) + } +} + +impl Display for PathBuf { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&**self, f) + } +} + +impl PathBuf { + pub fn new() -> PathBuf { + Self::default() + } + + pub fn with_capacity(capacity: usize) -> PathBuf { + // SAFETY: The empty path is a valid path. + Self { + inner: Vec::with_capacity(capacity), + } + } + + /// Adjoins `name` to self. + pub fn try_push(&mut self, name: &[u8]) -> Result<(), std::io::Error> { + validate_node_name(name).map_err(|_| std::io::ErrorKind::InvalidData)?; + + if !self.inner.is_empty() { + self.inner.push(b'/'); + } + + self.inner.extend_from_slice(name); + + Ok(()) + } + + /// Convert a byte vector to a PathBuf, without checking validity. + unsafe fn from_bytes_unchecked(bytes: Vec<u8>) -> PathBuf { + PathBuf { inner: bytes } + } + + /// Convert from a [&std::path::Path] to [Self]. + /// + /// - Self uses `/` as path separator. + /// - Absolute paths are always rejected, are are these with custom prefixes. + /// - Repeated separators are deduplicated. + /// - Occurrences of `.` are normalized away. + /// - A trailing slash is normalized away. + /// + /// A `canonicalize_dotdot` boolean controls whether `..` will get + /// canonicalized if possible, or should return an error. + /// + /// For more exotic paths, this conversion might produce different results + /// on different platforms, due to different underlying byte + /// representations, which is why it's restricted to unix for now. + #[cfg(unix)] + pub fn from_host_path( + host_path: &std::path::Path, + canonicalize_dotdot: bool, + ) -> Result<Self, std::io::Error> { + let mut p = PathBuf::with_capacity(host_path.as_os_str().len()); + + for component in host_path.components() { + match component { + std::path::Component::Prefix(_) | std::path::Component::RootDir => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "found disallowed prefix or rootdir", + )) + } + std::path::Component::CurDir => continue, // ignore + std::path::Component::ParentDir => { + if canonicalize_dotdot { + // Try popping the last element from the path being constructed. + // FUTUREWORK: pop method? + p = p + .parent() + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "found .. going too far up", + ) + })? + .to_owned(); + } else { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "found disallowed ..", + )); + } + } + std::path::Component::Normal(s) => { + // append the new component to the path being constructed. + p.try_push(s.as_encoded_bytes()).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "encountered invalid node in sub_path component", + ) + })? + } + } + } + + Ok(p) + } + + pub fn into_boxed_path(self) -> Box<Path> { + // SAFETY: Box<[u8]> and Box<Path> have the same representation, + // and PathBuf always contains a valid Path. + unsafe { mem::transmute(self.inner.into_boxed_slice()) } + } + + pub fn into_bytes(self) -> Vec<u8> { + self.inner + } +} + +#[cfg(test)] +mod test { + use super::{Path, PathBuf}; + use bstr::ByteSlice; + use rstest::rstest; + + // TODO: add some manual tests including invalid UTF-8 (hard to express + // with rstest) + + #[rstest] + #[case::empty("", 0)] + #[case("a", 1)] + #[case("a/b", 2)] + #[case("a/b/c", 3)] + // add two slightly more cursed variants. + // Technically nothing prevents us from representing this with castore, + // but maybe we want to disallow constructing paths like this as it's a + // bad idea. + #[case::cursed("C:\\a/b", 2)] + #[case::cursed("\\\\tvix-store", 1)] + pub fn from_str(#[case] s: &str, #[case] num_components: usize) { + let p: PathBuf = s.parse().expect("must parse"); + + assert_eq!(s.as_bytes(), p.as_bytes(), "inner bytes mismatch"); + assert_eq!( + num_components, + p.components().count(), + "number of components mismatch" + ); + } + + #[rstest] + #[case::absolute("/a/b")] + #[case::two_forward_slashes_start("//a/b")] + #[case::two_forward_slashes_middle("a/b//c/d")] + #[case::trailing_slash("a/b/")] + #[case::dot(".")] + #[case::dotdot("..")] + #[case::dot_start("./a")] + #[case::dotdot_start("../a")] + #[case::dot_middle("a/./b")] + #[case::dotdot_middle("a/../b")] + #[case::dot_end("a/b/.")] + #[case::dotdot_end("a/b/..")] + #[case::null("fo\0o")] + pub fn from_str_fail(#[case] s: &str) { + s.parse::<PathBuf>().expect_err("must fail"); + } + + #[rstest] + #[case("foo", "")] + #[case("foo/bar", "foo")] + #[case("foo2/bar2", "foo2")] + #[case("foo/bar/baz", "foo/bar")] + pub fn parent(#[case] p: PathBuf, #[case] exp_parent: PathBuf) { + assert_eq!(Some(&*exp_parent), p.parent()); + } + + #[rstest] + pub fn no_parent() { + assert!(Path::ROOT.parent().is_none()); + } + + #[rstest] + #[case("a", "b", "a/b")] + #[case("a", "b", "a/b")] + pub fn join_push(#[case] mut p: PathBuf, #[case] name: &str, #[case] exp_p: PathBuf) { + assert_eq!(exp_p, p.try_join(name.as_bytes()).expect("join failed")); + p.try_push(name.as_bytes()).expect("push failed"); + assert_eq!(exp_p, p); + } + + #[rstest] + #[case("a", "/")] + #[case("a", "")] + #[case("a", "b/c")] + #[case("", "/")] + #[case("", "")] + #[case("", "b/c")] + #[case("", ".")] + #[case("", "..")] + pub fn join_push_fail(#[case] mut p: PathBuf, #[case] name: &str) { + p.try_join(name.as_bytes()) + .expect_err("join succeeded unexpectedly"); + p.try_push(name.as_bytes()) + .expect_err("push succeeded unexpectedly"); + } + + #[rstest] + #[case::empty("", vec![])] + #[case("a", vec!["a"])] + #[case("a/b", vec!["a", "b"])] + #[case("a/b/c", vec!["a","b", "c"])] + pub fn components(#[case] p: PathBuf, #[case] exp_components: Vec<&str>) { + assert_eq!( + exp_components, + p.components() + .map(|x| x.to_str().unwrap()) + .collect::<Vec<_>>() + ); + } + + #[rstest] + #[case::empty("", "", false)] + #[case::path("a", "a", false)] + #[case::path2("a/b", "a/b", false)] + #[case::double_slash_middle("a//b", "a/b", false)] + #[case::dot(".", "", false)] + #[case::dot_start("./a/b", "a/b", false)] + #[case::dot_middle("a/./b", "a/b", false)] + #[case::dot_end("a/b/.", "a/b", false)] + #[case::trailing_slash("a/b/", "a/b", false)] + #[case::dotdot_canonicalize("a/..", "", true)] + #[case::dotdot_canonicalize2("a/../b", "b", true)] + #[cfg_attr(unix, case::faux_prefix("\\\\nix-store", "\\\\nix-store", false))] + #[cfg_attr(unix, case::faux_letter("C:\\foo.txt", "C:\\foo.txt", false))] + pub fn from_host_path( + #[case] host_path: std::path::PathBuf, + #[case] exp_path: PathBuf, + #[case] canonicalize_dotdot: bool, + ) { + let p = PathBuf::from_host_path(&host_path, canonicalize_dotdot).expect("must succeed"); + + assert_eq!(exp_path, p); + } + + #[rstest] + #[case::absolute("/", false)] + #[case::dotdot_root("..", false)] + #[case::dotdot_root_canonicalize("..", true)] + #[case::dotdot_root_no_canonicalize("a/..", false)] + #[case::invalid_name("foo/bar\0", false)] + // #[cfg_attr(windows, case::prefix("\\\\nix-store", false))] + // #[cfg_attr(windows, case::letter("C:\\foo.txt", false))] + pub fn from_host_path_fail( + #[case] host_path: std::path::PathBuf, + #[case] canonicalize_dotdot: bool, + ) { + PathBuf::from_host_path(&host_path, canonicalize_dotdot).expect_err("must fail"); + } +} diff --git a/tvix/castore/src/proto/grpc_blobservice_wrapper.rs b/tvix/castore/src/proto/grpc_blobservice_wrapper.rs index a7f84dbe60..41bd0698ec 100644 --- a/tvix/castore/src/proto/grpc_blobservice_wrapper.rs +++ b/tvix/castore/src/proto/grpc_blobservice_wrapper.rs @@ -1,5 +1,6 @@ use crate::blobservice::BlobService; use core::pin::pin; +use data_encoding::BASE64; use futures::{stream::BoxStream, TryFutureExt}; use std::{ collections::VecDeque, @@ -86,7 +87,7 @@ where // https://github.com/tokio-rs/tokio/issues/2723#issuecomment-1534723933 type ReadStream = BoxStream<'static, Result<super::BlobChunk, Status>>; - #[instrument(skip_all)] + #[instrument(skip_all, fields(blob.digest=format!("b3:{}", BASE64.encode(&request.get_ref().digest))))] async fn stat( &self, request: Request<super::StatBlobRequest>, @@ -110,7 +111,7 @@ where } } - #[instrument(skip_all)] + #[instrument(skip_all, fields(blob.digest=format!("b3:{}", BASE64.encode(&request.get_ref().digest))))] async fn read( &self, request: Request<super::ReadBlobRequest>, diff --git a/tvix/castore/src/proto/mod.rs b/tvix/castore/src/proto/mod.rs index 97ef183588..5374e3ae5a 100644 --- a/tvix/castore/src/proto/mod.rs +++ b/tvix/castore/src/proto/mod.rs @@ -66,7 +66,7 @@ pub enum ValidateStatBlobResponseError { /// Checks a Node name for validity as an intermediate node. /// We disallow slashes, null bytes, '.', '..' and the empty string. -fn validate_node_name(name: &[u8]) -> Result<(), ValidateNodeError> { +pub(crate) fn validate_node_name(name: &[u8]) -> Result<(), ValidateNodeError> { if name.is_empty() || name == b".." || name == b"." @@ -179,6 +179,42 @@ impl Ord for node::Node { } } +impl PartialOrd for FileNode { + fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { + Some(self.cmp(other)) + } +} + +impl Ord for FileNode { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.get_name().cmp(other.get_name()) + } +} + +impl PartialOrd for SymlinkNode { + fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { + Some(self.cmp(other)) + } +} + +impl Ord for SymlinkNode { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.get_name().cmp(other.get_name()) + } +} + +impl PartialOrd for DirectoryNode { + fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { + Some(self.cmp(other)) + } +} + +impl Ord for DirectoryNode { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.get_name().cmp(other.get_name()) + } +} + /// Accepts a name, and a mutable reference to the previous name. /// If the passed name is larger than the previous one, the reference is updated. /// If it's not, an error is returned. @@ -303,6 +339,56 @@ impl Directory { i_symlinks: self.symlinks.iter().peekable(), }; } + + /// Adds the specified [node::Node] to the [Directory], preserving sorted entries. + /// This assumes the [Directory] to be sorted prior to adding the node. + /// + /// Inserting an element that already exists with the same name in the directory is not + /// supported. + pub fn add(&mut self, node: node::Node) { + debug_assert!( + !self.files.iter().any(|x| x.get_name() == node.get_name()), + "name already exists in files" + ); + debug_assert!( + !self + .directories + .iter() + .any(|x| x.get_name() == node.get_name()), + "name already exists in directories" + ); + debug_assert!( + !self + .symlinks + .iter() + .any(|x| x.get_name() == node.get_name()), + "name already exists in symlinks" + ); + + match node { + node::Node::File(node) => { + let pos = self + .files + .binary_search(&node) + .expect_err("Tvix bug: dir entry with name already exists"); + self.files.insert(pos, node); + } + node::Node::Directory(node) => { + let pos = self + .directories + .binary_search(&node) + .expect_err("Tvix bug: dir entry with name already exists"); + self.directories.insert(pos, node); + } + node::Node::Symlink(node) => { + let pos = self + .symlinks + .binary_search(&node) + .expect_err("Tvix bug: dir entry with name already exists"); + self.symlinks.insert(pos, node); + } + } + } } impl StatBlobResponse { diff --git a/tvix/castore/src/proto/tests/directory.rs b/tvix/castore/src/proto/tests/directory.rs index 5fda394775..81b73a048d 100644 --- a/tvix/castore/src/proto/tests/directory.rs +++ b/tvix/castore/src/proto/tests/directory.rs @@ -1,5 +1,6 @@ use crate::proto::{ - Directory, DirectoryNode, FileNode, SymlinkNode, ValidateDirectoryError, ValidateNodeError, + node, Directory, DirectoryNode, FileNode, SymlinkNode, ValidateDirectoryError, + ValidateNodeError, }; use hex_literal::hex; @@ -371,3 +372,81 @@ fn validate_overflow() { _ => panic!("unexpected error"), } } + +#[test] +fn add_nodes_to_directory() { + let mut d = Directory { + ..Default::default() + }; + + d.add(node::Node::Directory(DirectoryNode { + name: "b".into(), + digest: DUMMY_DIGEST.to_vec().into(), + size: 1, + })); + d.add(node::Node::Directory(DirectoryNode { + name: "a".into(), + digest: DUMMY_DIGEST.to_vec().into(), + size: 1, + })); + d.add(node::Node::Directory(DirectoryNode { + name: "z".into(), + digest: DUMMY_DIGEST.to_vec().into(), + size: 1, + })); + + d.add(node::Node::File(FileNode { + name: "f".into(), + digest: DUMMY_DIGEST.to_vec().into(), + size: 1, + executable: true, + })); + d.add(node::Node::File(FileNode { + name: "c".into(), + digest: DUMMY_DIGEST.to_vec().into(), + size: 1, + executable: true, + })); + d.add(node::Node::File(FileNode { + name: "g".into(), + digest: DUMMY_DIGEST.to_vec().into(), + size: 1, + executable: true, + })); + + d.add(node::Node::Symlink(SymlinkNode { + name: "t".into(), + target: "a".into(), + })); + d.add(node::Node::Symlink(SymlinkNode { + name: "o".into(), + target: "a".into(), + })); + d.add(node::Node::Symlink(SymlinkNode { + name: "e".into(), + target: "a".into(), + })); + + d.validate().expect("directory should be valid"); +} + +#[test] +#[cfg_attr(not(debug_assertions), ignore)] +#[should_panic = "name already exists in directories"] +fn add_duplicate_node_to_directory_panics() { + let mut d = Directory { + ..Default::default() + }; + + d.add(node::Node::Directory(DirectoryNode { + name: "a".into(), + digest: DUMMY_DIGEST.to_vec().into(), + size: 1, + })); + d.add(node::Node::File(FileNode { + name: "a".into(), + digest: DUMMY_DIGEST.to_vec().into(), + size: 1, + executable: true, + })); +} diff --git a/tvix/castore/src/tests/import.rs b/tvix/castore/src/tests/import.rs index b44b71cd78..8b3bd5ce0f 100644 --- a/tvix/castore/src/tests/import.rs +++ b/tvix/castore/src/tests/import.rs @@ -1,7 +1,7 @@ use crate::blobservice::{self, BlobService}; use crate::directoryservice; use crate::fixtures::*; -use crate::import::ingest_path; +use crate::import::fs::ingest_path; use crate::proto; use std::sync::Arc; diff --git a/tvix/castore/src/tonic.rs b/tvix/castore/src/tonic.rs index 8ea417548c..4b65d6b028 100644 --- a/tvix/castore/src/tonic.rs +++ b/tvix/castore/src/tonic.rs @@ -87,31 +87,35 @@ impl From<tonic::transport::Error> for Error { #[cfg(test)] mod tests { use super::channel_from_url; - use test_case::test_case; + use rstest::rstest; use url::Url; + #[rstest] /// Correct scheme to connect to a unix socket. - #[test_case("grpc+unix:///path/to/somewhere", true; "grpc valid unix socket")] + #[case::valid_unix_socket("grpc+unix:///path/to/somewhere", true)] /// Connecting with wait-connect set to 0 succeeds, as that's the default. - #[test_case("grpc+unix:///path/to/somewhere?wait-connect=0", true; "grpc valid unix wait-connect=0")] + #[case::valid_unix_socket_wait_connect_0("grpc+unix:///path/to/somewhere?wait-connect=0", true)] /// Connecting with wait-connect set to 1 fails, as the path doesn't exist. - #[test_case("grpc+unix:///path/to/somewhere?wait-connect=1", false; "grpc valid unix wait-connect=1")] + #[case::valid_unix_socket_wait_connect_1( + "grpc+unix:///path/to/somewhere?wait-connect=1", + false + )] /// Correct scheme for unix socket, but setting a host too, which is invalid. - #[test_case("grpc+unix://host.example/path/to/somewhere", false; "grpc invalid unix socket and host")] + #[case::invalid_unix_socket_and_host("grpc+unix://host.example/path/to/somewhere", false)] /// Correct scheme to connect to localhost, with port 12345 - #[test_case("grpc+http://[::1]:12345", true; "grpc valid IPv6 localhost port 12345")] + #[case::valid_ipv6_localhost_port_12345("grpc+http://[::1]:12345", true)] /// Correct scheme to connect to localhost over http, without specifying a port. - #[test_case("grpc+http://localhost", true; "grpc valid http host without port")] + #[case::valid_http_host_without_port("grpc+http://localhost", true)] /// Correct scheme to connect to localhost over http, without specifying a port. - #[test_case("grpc+https://localhost", true; "grpc valid https host without port")] + #[case::valid_https_host_without_port("grpc+https://localhost", true)] /// Correct scheme to connect to localhost over http, but with additional path, which is invalid. - #[test_case("grpc+http://localhost/some-path", false; "grpc valid invalid host and path")] + #[case::invalid_host_and_path("grpc+http://localhost/some-path", false)] /// Connecting with wait-connect set to 0 succeeds, as that's the default. - #[test_case("grpc+http://localhost?wait-connect=0", true; "grpc valid host wait-connect=0")] + #[case::valid_host_wait_connect_0("grpc+http://localhost?wait-connect=0", true)] /// Connecting with wait-connect set to 1 fails, as the host doesn't exist. - #[test_case("grpc+http://nonexist.invalid?wait-connect=1", false; "grpc valid host wait-connect=1")] + #[case::valid_host_wait_connect_1_fails("grpc+http://nonexist.invalid?wait-connect=1", false)] #[tokio::test] - async fn test_from_addr_tokio(uri_str: &str, is_ok: bool) { + async fn test_from_addr_tokio(#[case] uri_str: &str, #[case] is_ok: bool) { let url = Url::parse(uri_str).expect("must parse"); assert_eq!(channel_from_url(&url).await.is_ok(), is_ok) } diff --git a/tvix/cli/Cargo.toml b/tvix/cli/Cargo.toml index d15165383b..ce8d361771 100644 --- a/tvix/cli/Cargo.toml +++ b/tvix/cli/Cargo.toml @@ -25,6 +25,3 @@ tracing-subscriber = { version = "0.3.16", features = ["json"] } [dependencies.wu-manber] git = "https://github.com/tvlfyi/wu-manber.git" - -[dev-dependencies] -test-case = "3.3.1" diff --git a/tvix/cli/default.nix b/tvix/cli/default.nix index 8782d34bf6..62e93cc213 100644 --- a/tvix/cli/default.nix +++ b/tvix/cli/default.nix @@ -1,23 +1,11 @@ { depot, pkgs, lib, ... }: -let - mkNixpkgsEvalCheck = attrset: expectedPath: { - label = ":nix: evaluate nixpkgs.${attrset} in tvix"; - needsOutput = true; - - command = pkgs.writeShellScript "tvix-eval-${builtins.replaceStrings [".drv"] ["-drv"] attrset}" '' - TVIX_OUTPUT=$(result/bin/tvix -E '(import ${pkgs.path} {}).${attrset}') - EXPECTED='${/* the verbatim expected Tvix output: */ "=> \"${builtins.unsafeDiscardStringContext expectedPath}\" :: string"}' - - echo "Tvix output: ''${TVIX_OUTPUT}" - if [ "$TVIX_OUTPUT" != "$EXPECTED" ]; then - echo "Correct would have been ''${EXPECTED}" - exit 1 - fi +(depot.tvix.crates.workspaceMembers.tvix-cli.build.override { + runTests = true; +}).overrideAttrs (finalAttrs: previousAttrs: - echo "Output was correct." - ''; - }; +let + tvix-cli = finalAttrs.finalPackage; benchmark-gnutime-format-string = description: @@ -30,24 +18,16 @@ let }; }); -in - -(depot.tvix.crates.workspaceMembers.tvix-cli.build.override { - runTests = true; -}).overrideAttrs (finalAttrs: previousAttrs: - -let - tvix-cli = finalAttrs.finalPackage; - # You can run the benchmark with a simple `nix run`, like: # - # nix run -f . tvix.cli.meta.ci.extraSteps.benchmark-nixpkgs-cross-hello-outpath + # nix-build -A tvix.cli.meta.ci.extraSteps.benchmark-nixpkgs-cross-hello-outpath # # TODO(amjoseph): store these results someplace more durable, like git trailers # mkExprBenchmark = { expr, description }: let name = "tvix-cli-benchmark-${description}"; in - (pkgs.writeShellScriptBin name '' + (pkgs.runCommand name { } '' + export SSL_CERT_FILE=${pkgs.cacert.out}/etc/ssl/certs/ca-bundle.crt ${lib.escapeShellArgs [ "${pkgs.time}/bin/time" "--format" "${benchmark-gnutime-format-string description}" @@ -55,15 +35,8 @@ let "--no-warnings" "-E" expr ]} - '').overrideAttrs (finalAttrs: previousAttrs: { - passthru = (previousAttrs.passthru or { }) // { - ci = { - label = ":nix: benchmark ${description} in tvix"; - needsOutput = true; - command = "${finalAttrs.finalPackage}/bin/${finalAttrs.meta.mainProgram}"; - }; - }; - }); + touch $out + ''); mkNixpkgsBenchmark = attrpath: mkExprBenchmark { @@ -71,6 +44,28 @@ let expr = "(import ${pkgs.path} {}).${attrpath}"; }; + # Constructs a Derivation invoking tvix-cli inside a build, ensures the + # calculated tvix output path matches what's passed in externally. + mkNixpkgsEvalTest = attrpath: expectedPath: + let + name = "tvix-eval-test-${builtins.replaceStrings [".drv"] ["-drv"] attrpath}"; + in + (pkgs.runCommand name { } '' + export SSL_CERT_FILE=${pkgs.cacert.out}/etc/ssl/certs/ca-bundle.crt + TVIX_OUTPUT=$(${tvix-cli}/bin/tvix -E '(import ${pkgs.path} {}).${attrpath}') + EXPECTED='${/* the verbatim expected Tvix output: */ "=> \"${builtins.unsafeDiscardStringContext expectedPath}\" :: string"}' + + echo "Tvix output: ''${TVIX_OUTPUT}" + if [ "$TVIX_OUTPUT" != "$EXPECTED" ]; then + echo "Correct would have been ''${EXPECTED}" + exit 1 + fi + + echo "Output was correct." + touch $out + ''); + + benchmarks = { benchmark-hello = (mkNixpkgsBenchmark "hello.outPath"); benchmark-cross-hello = (mkNixpkgsBenchmark "pkgsCross.aarch64-multiplatform.hello.outPath"); @@ -79,21 +74,22 @@ let # Example used for benchmarking LightSpan::Delayed in commit bf286a54bc2ac5eeb78c3d5c5ae66e9af24d74d4 benchmark-nixpkgs-attrnames = (mkExprBenchmark { expr = "builtins.length (builtins.attrNames (import ${pkgs.path} {}))"; description = "nixpkgs-attrnames"; }); }; + + evalTests = { + eval-nixpkgs-stdenv-drvpath = (mkNixpkgsEvalTest "stdenv.drvPath" pkgs.stdenv.drvPath); + eval-nixpkgs-stdenv-outpath = (mkNixpkgsEvalTest "stdenv.outPath" pkgs.stdenv.outPath); + eval-nixpkgs-hello-outpath = (mkNixpkgsEvalTest "hello.outPath" pkgs.hello.outPath); + eval-nixpkgs-firefox-outpath = (mkNixpkgsEvalTest "firefox.outPath" pkgs.firefox.outPath); + eval-nixpkgs-firefox-drvpath = (mkNixpkgsEvalTest "firefox.drvPath" pkgs.firefox.drvPath); + eval-nixpkgs-cross-stdenv-outpath = (mkNixpkgsEvalTest "pkgsCross.aarch64-multiplatform.stdenv.outPath" pkgs.pkgsCross.aarch64-multiplatform.stdenv.outPath); + eval-nixpkgs-cross-hello-outpath = (mkNixpkgsEvalTest "pkgsCross.aarch64-multiplatform.hello.outPath" pkgs.pkgsCross.aarch64-multiplatform.hello.outPath); + }; in { meta = { - ci.extraSteps = { - eval-nixpkgs-stdenv-drvpath = (mkNixpkgsEvalCheck "stdenv.drvPath" pkgs.stdenv.drvPath); - eval-nixpkgs-stdenv-outpath = (mkNixpkgsEvalCheck "stdenv.outPath" pkgs.stdenv.outPath); - eval-nixpkgs-hello-outpath = (mkNixpkgsEvalCheck "hello.outPath" pkgs.hello.outPath); - eval-nixpkgs-firefox-outpath = (mkNixpkgsEvalCheck "firefox.outPath" pkgs.firefox.outPath); - eval-nixpkgs-firefox-drvpath = (mkNixpkgsEvalCheck "firefox.drvPath" pkgs.firefox.drvPath); - eval-nixpkgs-cross-stdenv-outpath = (mkNixpkgsEvalCheck "pkgsCross.aarch64-multiplatform.stdenv.outPath" pkgs.pkgsCross.aarch64-multiplatform.stdenv.outPath); - eval-nixpkgs-cross-hello-outpath = (mkNixpkgsEvalCheck "pkgsCross.aarch64-multiplatform.hello.outPath" pkgs.pkgsCross.aarch64-multiplatform.hello.outPath); - }; - ci.targets = builtins.attrNames benchmarks; + ci.targets = (builtins.attrNames benchmarks) ++ (builtins.attrNames evalTests); }; - # Expose benchmarks as standard CI targets. - passthru = benchmarks; + # Expose benchmarks and evalTests as standard CI targets. + passthru = benchmarks // evalTests; }) diff --git a/tvix/cli/src/main.rs b/tvix/cli/src/main.rs index 436e895863..1f1fb3de68 100644 --- a/tvix/cli/src/main.rs +++ b/tvix/cli/src/main.rs @@ -5,6 +5,7 @@ use std::{fs, path::PathBuf}; use tracing::Level; use tracing_subscriber::fmt::writer::MakeWriterExt; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; +use tracing_subscriber::{EnvFilter, Layer}; use tvix_build::buildservice; use tvix_eval::builtins::impure_builtins; use tvix_eval::observer::{DisassemblingObserver, TracingObserver}; @@ -79,12 +80,7 @@ struct Args { build_service_addr: String, } -/// Interprets the given code snippet, printing out warnings, errors -/// and the result itself. The return value indicates whether -/// evaluation succeeded. -fn interpret(code: &str, path: Option<PathBuf>, args: &Args, explain: bool) -> bool { - let tokio_runtime = tokio::runtime::Runtime::new().expect("failed to setup tokio runtime"); - +fn init_io_handle(tokio_runtime: &tokio::runtime::Runtime, args: &Args) -> Rc<TvixStoreIO> { let (blob_service, directory_service, path_info_service) = tokio_runtime .block_on({ let blob_service_addr = args.blob_service_addr.clone(); @@ -116,14 +112,25 @@ fn interpret(code: &str, path: Option<PathBuf>, args: &Args, explain: bool) -> b }) .expect("unable to setup buildservice before interpreter setup"); - let tvix_store_io = Rc::new(TvixStoreIO::new( + Rc::new(TvixStoreIO::new( blob_service.clone(), directory_service.clone(), path_info_service.into(), build_service.into(), tokio_runtime.handle().clone(), - )); + )) +} +/// Interprets the given code snippet, printing out warnings, errors +/// and the result itself. The return value indicates whether +/// evaluation succeeded. +fn interpret( + tvix_store_io: Rc<TvixStoreIO>, + code: &str, + path: Option<PathBuf>, + args: &Args, + explain: bool, +) -> bool { let mut eval = tvix_eval::Evaluation::new( Box::new(TvixIO::new(tvix_store_io.clone() as Rc<dyn EvalIO>)) as Box<dyn EvalIO>, true, @@ -229,24 +236,34 @@ fn main() { let subscriber = tracing_subscriber::registry().with( tracing_subscriber::fmt::Layer::new() .with_writer(std::io::stderr.with_max_level(level)) - .pretty(), + .compact() + .with_filter( + EnvFilter::builder() + .with_default_directive(level.into()) + .from_env() + .expect("invalid RUST_LOG"), + ), ); subscriber .try_init() .expect("unable to set up tracing subscriber"); + let tokio_runtime = tokio::runtime::Runtime::new().expect("failed to setup tokio runtime"); + + let io_handle = init_io_handle(&tokio_runtime, &args); + if let Some(file) = &args.script { - run_file(file.clone(), &args) + run_file(io_handle, file.clone(), &args) } else if let Some(expr) = &args.expr { - if !interpret(expr, None, &args, false) { + if !interpret(io_handle, expr, None, &args, false) { std::process::exit(1); } } else { - run_prompt(&args) + run_prompt(io_handle, &args) } } -fn run_file(mut path: PathBuf, args: &Args) { +fn run_file(io_handle: Rc<TvixStoreIO>, mut path: PathBuf, args: &Args) { if path.is_dir() { path.push("default.nix"); } @@ -255,7 +272,7 @@ fn run_file(mut path: PathBuf, args: &Args) { let success = if args.compile_only { lint(&contents, Some(path), args) } else { - interpret(&contents, Some(path), args, false) + interpret(io_handle, &contents, Some(path), args, false) }; if !success { @@ -279,7 +296,7 @@ fn state_dir() -> Option<PathBuf> { path } -fn run_prompt(args: &Args) { +fn run_prompt(io_handle: Rc<TvixStoreIO>, args: &Args) { let mut rl = Editor::<()>::new().expect("should be able to launch rustyline"); if args.compile_only { @@ -310,9 +327,9 @@ fn run_prompt(args: &Args) { rl.add_history_entry(&line); if let Some(without_prefix) = line.strip_prefix(":d ") { - interpret(without_prefix, None, args, true); + interpret(Rc::clone(&io_handle), without_prefix, None, args, true); } else { - interpret(&line, None, args, false); + interpret(Rc::clone(&io_handle), &line, None, args, false); } } Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => break, diff --git a/tvix/crate-hashes.json b/tvix/crate-hashes.json index d57ed3a289..ca45e43176 100644 --- a/tvix/crate-hashes.json +++ b/tvix/crate-hashes.json @@ -1,4 +1,4 @@ { - "test-generator 0.3.0 (git+https://github.com/JamesGuthrie/test-generator.git?rev=82e799979980962aec1aa324ec6e0e4cad781f41#82e799979980962aec1aa324ec6e0e4cad781f41)": "08brp3qqa55hijc7xby3lam2cc84hvx1zzfqv6lj7smlczh8k32y", + "bigtable_rs 0.2.9 (git+https://github.com/flokli/bigtable_rs?rev=0af404741dfc40eb9fa99cf4d4140a09c5c20df7#0af404741dfc40eb9fa99cf4d4140a09c5c20df7)": "1njjam1lx2xlnm7a41lga8601vmjgqz0fvc77x24gd04pc7avxll", "wu-manber 0.1.0 (git+https://github.com/tvlfyi/wu-manber.git#0d5b22bea136659f7de60b102a7030e0daaa503d)": "1zhk83lbq99xzyjwphv2qrb8f8qgfqwa5bbbvyzm0z0bljsjv0pd" -} +} \ No newline at end of file diff --git a/tvix/default.nix b/tvix/default.nix index da3e0ca605..f562cf37de 100644 --- a/tvix/default.nix +++ b/tvix/default.nix @@ -43,6 +43,10 @@ let nativeBuildInputs = protobufDep prev; }; + prost-wkt-types = prev: { + nativeBuildInputs = protobufDep prev; + }; + tonic-reflection = prev: { nativeBuildInputs = protobufDep prev; }; @@ -84,7 +88,7 @@ let (crateName: (lib.nameValuePair "${crateName}-${crates.internal.crates.${crateName}.version}" crates.internal.crates.${crateName}.src.outputHash) ) [ - "test-generator" + "bigtable_rs" "wu-manber" ]); }; diff --git a/tvix/docs/src/SUMMARY.md b/tvix/docs/src/SUMMARY.md index f9b069d42c..b0e47a0011 100644 --- a/tvix/docs/src/SUMMARY.md +++ b/tvix/docs/src/SUMMARY.md @@ -2,6 +2,7 @@ # Tvix - [Architecture & data flow](./architecture.md) +- [TODOs](./TODO.md) # Nix - [Specification of the Nix Language](./language-spec.md) diff --git a/tvix/docs/src/TODO.md b/tvix/docs/src/TODO.md new file mode 100644 index 0000000000..6644bb6bac --- /dev/null +++ b/tvix/docs/src/TODO.md @@ -0,0 +1,129 @@ +# TODO + +This contains a rough collection of ideas on the TODO list, trying to keep track +of it somewhere. + +Of course, there's no guarantee these things will get addressed, but it helps +dumping the backlog somewhere. + +Feel free to add new ideas. Before picking something, ask in `#tvix-dev` to make +sure noone is working on this, or has some specific design in mind already. + +## Cleanups + - move some of the rstest cases in `tvix-glue` to the `.nix`/`.exp` mechanism. + - Parts requiring test fixtures need some special convention. + Some of these also cannot be checked into the repo, like the import tests + adding special files to test filtering. + - add `nix_oracle` mechanism from `tvix-eval` to `tvix-glue`. + +## Fixes towards correctness + - `builtins.toXML` is missing string context. See b/398. + - `builtins.toXML` self-closing tags need to be configurable in a more granular + fashion, requires third-party crate support. See b/399. + - `rnix` only supports string source files, but `NixString` uses bytes (and Nix + source code might be no valid UTF-8). + +## Documentation +Extend the other pages in here. Some ideas on what should be tackled: + - Document what Tvix is, and what it is not yet. What it is now, what it is not + (yet), explaining some of the architectural choices (castore, more hermetic + `Build` repr), while still being compatible. Explain how it's possible to + plug in other frontends, and use `tvix-{[ca]store,build}` without Nixlang even. + And how `nix-compat` is a useful crate for all sorts of formats and data + types of Nix. + - Update the Architecture diagram to model the current state of things. + There's no gRPC between Coordinator and Evaluator. + - Add a dedicated section/page explaining the separation between tvix-glue and + tvix-eval, and how more annoying builtins get injected into tvix-eval through + tvix-glue. + Maybe restructure to only explain the component structure potentially + crossing process boundaries (those with gRPC), and make the rest more crate + and trait-focused? + - Restructure docs on castore vs store, this seems to be duplicated a bit and + is probably still not too clear. + - Describe store composition(s) in more detail. There's some notes on granular + fetching which probably can be repurposed. + - Absorb the rest of //tvix/website into this. + +## Features + +### CLI + - `nix repl` can set variables and effectively mutates a global scope. We + should update the existing / add another repl that allows the same. We don't + want to mutate the evaluator, but should construct a new one, passing in the + root scope returned from the previous evaluation. + +### Fetchers +Some more fetcher-related builtins need work: + - `fetchGit` + - `fetchTree` (hairy, seems there's no proper spec and the URL syntax seems + subject to change/underdocumented) + +### Convert builtins:fetchurl to Fetches +We need to convert `builtins:fetchurl`-style calls to `builtins.derivation` to +fetches, not Derivations (tracked in `KnownPaths`). + +### Derivation -> Build +While we have some support for `structuredAttrs` and `fetchClosure` (at least +enough to calculate output hashes, aka produce identical ATerm), the code +populating the `Build` struct doesn't exist it yet. + +Similarly, we also don't properly populate the build environment for +`fetchClosure` yet. (Note there already is `ExportedPathInfo`, so once +`structuredAttrs` is there this should be easy. + +### Builders +Once builds are proven to work with real-world builds, and the corner cases +there are ruled out, adding other types of builders might be interesting. + + - bwrap + - gVisor + - Cloud Hypervisor (using similar technique as `//tvix//boot`). + +Long-term, we want to extend traits and gRPC protocol to expose more telemetry, +logs etc, but this is something requiring a lot of designing. + +### Store composition + - Combinators: list-by-priority, first-come-first-serve, cache + - How do describe hierarchies. URL format too one-dimensional, but we might get + quite far with a similar "substituters" concept that Nix uses, to construct + the composed stores. +### Store Config + There's already serde for some store options (bigtable uses `serde_qs`). + We might also have common options global over all backends, like chunking + parameters for chunking blobservices. Think where this would fit in. + - Rework the URL syntax for object_store. We should support the default s3/gcs + URLs at least. + +### BlobService + - On the trait side, currently there's no way to distinguish reading a + known-chunk vs blob, so we might be calling `.chunks()` unnecessarily often. + At least for the `object_store` backend, this might be a problem. + - While `object_store` recently got support for `Content-Type` + (https://github.com/apache/arrow-rs/pull/5650), there's no support on the + local filesystem yet. We'd need to add support to this (through xattrs). + +### DirectoryService + - Add an `object_store` variant, storing a Directory *closure* keyed by the + root `Directory` digest. This won't allow indexing intermediate Directory + nodes, but once we have `DirectoryService` composition, it shouldn't be an + issue. + - [redb](https://www.redb.org/) backend + +### PathInfoService + - [redb](https://www.redb.org/) backend + - sqlite backend (different schema than the Nix one, we need the root nodes data!) + +### Nix-compat +- Async NAR reader (@edef?) + +### Nix Daemon protocol +- Some work ongoing on the worker operation parsing. Partially blocked on the async NAR reader. + +### O11Y + - gRPC trace propagation (cl/10532) + - `tracing-tracy` (cl/10952) + - `[tracing-]indicatif` for progress/log reporting (floklis stash) + - unification into `tvix-tracing` crate, currently a lot of boilerplate + in `tvix-store` CLI entrypoint, and half of the boilerplate copied over to + `tvix-cli`. diff --git a/tvix/eval/Cargo.toml b/tvix/eval/Cargo.toml index e2bd07f827..677ce6ab85 100644 --- a/tvix/eval/Cargo.toml +++ b/tvix/eval/Cargo.toml @@ -3,8 +3,6 @@ name = "tvix-eval" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [lib] name = "tvix_eval" @@ -42,7 +40,7 @@ data-encoding = "2.5.0" criterion = "0.5" itertools = "0.12.0" pretty_assertions = "1.2.1" -rstest = "0.18.2" +rstest = "0.19.0" tempfile = "3.3.0" [features] diff --git a/tvix/eval/docs/bindings.md b/tvix/eval/docs/bindings.md new file mode 100644 index 0000000000..2b062cb13d --- /dev/null +++ b/tvix/eval/docs/bindings.md @@ -0,0 +1,133 @@ +Compilation of bindings +======================= + +Compilation of Nix bindings is one of the most mind-bending parts of Nix +evaluation. The implementation of just the compilation is currently almost 1000 +lines of code, excluding the various insane test cases we dreamt up for it. + +## What is a binding? + +In short, any attribute set or `let`-expression. Tvix currently does not treat +formals in function parameters (e.g. `{ name ? "fred" }: ...`) the same as these +bindings. + +They have two very difficult features: + +1. Keys can mutually refer to each other in `rec` sets or `let`-bindings, + including out of definition order. +2. Attribute sets can be nested, and parts of one attribute set can be defined + in multiple separate bindings. + +Tvix resolves as much of this logic statically (i.e. at compile-time) as +possible, but the procedure is quite complicated. + +## High-level concept + +The idea behind the way we compile bindings is to fully resolve nesting +statically, and use the usual mechanisms (i.e. recursion/thunking/value +capturing) for resolving dynamic values. + +This is done by compiling bindings in several phases: + +1. An initial compilation phase *only* for plain inherit statements (i.e. + `inherit name;`), *not* for namespaced inherits (i.e. `inherit (from) + name;`). + +2. A declaration-only phase, in which we use the compiler's scope tracking logic + to calculate the physical runtime stack indices (further referred to as + "stack slots" or just "slots") that all values will end up in. + + In this phase, whenever we encounter a nested attribute set, it is merged + into a custom data structure that acts like a synthetic AST node. + + This can be imagined similar to a rewrite like this: + + ```nix + # initial code: + { + a.b = 1; + a.c = 2; + } + + # rewritten form: + { + a = { + b = 1; + c = 2; + }; + } + ``` + + The rewrite applies to attribute sets and `let`-bindings alike. + + At the end of this phase, we know the stack slots of all namespaces for + inheriting from, all values inherited from them, and all values (and + optionall keys) of bindings at the current level. + + Only statically known keys are actually merged, so any dynamic keys that + conflict will lead to a "key already defined" error at runtime. + +3. A compilation phase, in which all values (and, when necessary, keys) are + actually compiled. In this phase the custom data structure used for merging + is encountered when compiling values. + + As this data structure acts like an AST node, the process begins recursively + for each nested attribute set. + +At the end of this process we have bytecode that leaves the required values (and +optionally keys) on the stack. In the case of attribute sets, a final operation +is emitted that constructs the actual attribute set structure at runtime. For +`let`-bindings a final operation is emitted that removes these locals from the +stack when the scope ends. + +## Moving parts + +WARNING: This documents the *current* implementation. If you only care about the +conceptual aspects, see above. + +There's a few types involved: + +* `PeekableAttrs`: peekable iterator over an attribute path (e.g. `a.b.c`) +* `BindingsKind`: enum defining the kind of bindings (attrs/recattrs/let) +* `AttributeSet`: struct holding the bindings kind, the AST nodes with inherits + (both namespaced and not), and an internal representation of bindings + (essentially a vector of tuples of the peekable attrs and the expression to + compile for the value). +* `Binding`: enum describing the kind of binding (namespaced inherit, attribute + set, plain binding of *any other value type*) +* `KeySlot`: enum describing the location in which a key slot is placed at + runtime (nowhere, statically known value in a slot, dynamic value in a slot) +* `TrackedBinding`: struct representing statically known information about a + single binding (its key slot, value slot and `Binding`) +* `TrackedBindings`: vector of tracked bindings, which implements logic for + merging attribute sets together + +And quite a few methods on `Compiler`: + +* `compile_bindings`: entry point for compiling anything that looks like a + binding, this calls out to the functions below. +* `compile_plain_inherits`: takes all inherits of a bindings node and compiles + the ones that are trivial to compile (i.e. just plain inherits without a + namespace). The `rnix` parser does not represent namespaced/plain inherits in + different nodes, so this function also aggregates the namespaced inherits and + returns them for further use +* `declare_namespaced_inherits`: passes over all namespaced inherits and + declares them on the locals stack, as well as inserts them into the provided + `TrackedBindings` +* `declare_bindings`: declares all regular key/value bindings in a bindings + scope, but without actually compiling their keys or values. + + There's a lot of heavy lifting going on here: + + 1. It invokes the various pieces of logic responsible for merging nested + attribute sets together, creating intermediate data structures in the value + slots of bindings that can be recursively processed the same way. + 2. It decides on the key slots of expressions based on the kind of bindings, + and the type of expression providing the key. +* `bind_values`: runs the actual compilation of values. Notably this function is + responsible for recursively compiling merged attribute sets when it encounters + a `Binding::Set` (on which it invokes `compile_bindings` itself). + +In addition to these several methods (such as `compile_attr_set`, +`compile_let_in`, ...) invoke the binding-kind specific logic and then call out +to the functions above. diff --git a/tvix/eval/docs/builtins.md b/tvix/eval/docs/builtins.md index 26bbd1b2d3..dba4c48c65 100644 --- a/tvix/eval/docs/builtins.md +++ b/tvix/eval/docs/builtins.md @@ -24,7 +24,7 @@ The `impl` column indicates implementation status in tvix: | addErrorContext | false | ? | | context | | all | false | 2 | true | | | any | false | 2 | true | | -| appendContext | false | ? | | context | +| appendContext | false | ? | | | | attrNames | false | 1 | true | | | attrValues | false | | true | | | baseNameOf | true | | | | @@ -120,7 +120,7 @@ The `impl` column indicates implementation status in tvix: | true | true | | | | | tryEval | false | | | | | typeOf | false | | | | -| unsafeDiscardOutputDependency | false | | | context | +| unsafeDiscardOutputDependency | false | | | | | unsafeDiscardStringContext | false | | | | | unsafeGetAttrPos | false | | | todo | | valueSize | false | | | todo | diff --git a/tvix/eval/src/builtins/hash.rs b/tvix/eval/src/builtins/hash.rs index 6d07fc9b2d..d0145f1e7d 100644 --- a/tvix/eval/src/builtins/hash.rs +++ b/tvix/eval/src/builtins/hash.rs @@ -6,18 +6,22 @@ use sha2::{digest::Output, Digest, Sha256, Sha512}; use crate::ErrorKind; -fn hash<D: Digest>(b: &[u8]) -> Output<D> { +/// Reads through all data from the passed reader, and returns the resulting [Digest]. +/// The exact hash function used is left generic over all [Digest]. +fn hash<D: Digest + std::io::Write>(mut r: impl std::io::Read) -> Result<Output<D>, ErrorKind> { let mut hasher = D::new(); - hasher.update(b); - hasher.finalize() + std::io::copy(&mut r, &mut hasher)?; + Ok(hasher.finalize()) } -pub fn hash_nix_string(algo: impl AsRef<[u8]>, s: impl AsRef<[u8]>) -> Result<String, ErrorKind> { +/// For a given algo "string" and reader for data, calculate the digest +/// and return it as a hexlower encoded [String]. +pub fn hash_nix_string(algo: impl AsRef<[u8]>, s: impl std::io::Read) -> Result<String, ErrorKind> { match algo.as_ref() { - b"md5" => Ok(HEXLOWER.encode(hash::<Md5>(s.as_ref()).as_bstr())), - b"sha1" => Ok(HEXLOWER.encode(hash::<Sha1>(s.as_ref()).as_bstr())), - b"sha256" => Ok(HEXLOWER.encode(hash::<Sha256>(s.as_ref()).as_bstr())), - b"sha512" => Ok(HEXLOWER.encode(hash::<Sha512>(s.as_ref()).as_bstr())), + b"md5" => Ok(HEXLOWER.encode(hash::<Md5>(s)?.as_bstr())), + b"sha1" => Ok(HEXLOWER.encode(hash::<Sha1>(s)?.as_bstr())), + b"sha256" => Ok(HEXLOWER.encode(hash::<Sha256>(s)?.as_bstr())), + b"sha512" => Ok(HEXLOWER.encode(hash::<Sha512>(s)?.as_bstr())), _ => Err(ErrorKind::UnknownHashType( algo.as_ref().as_bstr().to_string(), )), diff --git a/tvix/eval/src/builtins/impure.rs b/tvix/eval/src/builtins/impure.rs index aad55c7331..c82b910f5f 100644 --- a/tvix/eval/src/builtins/impure.rs +++ b/tvix/eval/src/builtins/impure.rs @@ -31,14 +31,13 @@ mod impure_builtins { } #[builtin("hashFile")] - #[allow(non_snake_case)] - async fn builtin_hashFile(co: GenCo, algo: Value, path: Value) -> Result<Value, ErrorKind> { + async fn builtin_hash_file(co: GenCo, algo: Value, path: Value) -> Result<Value, ErrorKind> { let path = match coerce_value_to_path(&co, path).await? { Err(cek) => return Ok(Value::from(cek)), Ok(p) => p, }; - let s = generators::request_read_to_string(&co, path).await; - hash_nix_string(algo.to_str()?, s.to_str()?).map(Value::from) + let r = generators::request_open_file(&co, path).await; + hash_nix_string(algo.to_str()?, r).map(Value::from) } #[builtin("pathExists")] @@ -79,7 +78,13 @@ mod impure_builtins { async fn builtin_read_file(co: GenCo, path: Value) -> Result<Value, ErrorKind> { match coerce_value_to_path(&co, path).await? { Err(cek) => Ok(Value::from(cek)), - Ok(path) => Ok(generators::request_read_to_string(&co, path).await), + Ok(path) => { + let mut buf = Vec::new(); + generators::request_open_file(&co, path) + .await + .read_to_end(&mut buf)?; + Ok(Value::from(buf)) + } } } } diff --git a/tvix/eval/src/builtins/mod.rs b/tvix/eval/src/builtins/mod.rs index bbf2eb80c4..04a0b3dd33 100644 --- a/tvix/eval/src/builtins/mod.rs +++ b/tvix/eval/src/builtins/mod.rs @@ -453,11 +453,11 @@ mod pure_builtins { #[builtin("toJSON")] async fn builtin_to_json(co: GenCo, val: Value) -> Result<Value, ErrorKind> { - match val.into_json(&co).await? { + match val.into_contextful_json(&co).await? { Err(cek) => Ok(Value::from(cek)), - Ok(json_value) => { + Ok((json_value, ctx)) => { let json_str = serde_json::to_string(&json_value)?; - Ok(json_str.into()) + Ok(NixString::new_context_from(ctx, json_str).into()) } } } @@ -692,10 +692,89 @@ mod pure_builtins { Ok(Value::attrs(NixAttrs::from_iter(elements))) } - #[builtin("hashString")] + #[builtin("appendContext")] #[allow(non_snake_case)] - async fn builtin_hashString(co: GenCo, algo: Value, s: Value) -> Result<Value, ErrorKind> { - hash_nix_string(algo.to_str()?, s.to_str()?).map(Value::from) + async fn builtin_appendContext( + co: GenCo, + origin: Value, + added_context: Value, + ) -> Result<Value, ErrorKind> { + // `appendContext` is a "grow" context function. + // It cannot remove a context element, neither replace a piece of its contents. + // + // Growing context is always a safe operation, there's no loss of dependency tracking + // information. + // + // This is why this operation is not prefixed by `unsafe` and is deemed *safe*. + // Nonetheless, it is possible to craft nonsensical context elements referring + // to inexistent derivations, output paths or output names. + // + // In Nix, those nonsensical context elements are partially mitigated by checking + // that various parameters are indeed syntatically valid store paths in the context, i.e. + // starting with the same prefix as `builtins.storeDir`, or ending with `.drv`. + // In addition, if writing to the store is possible (evaluator not in read-only mode), Nix + // will realize some paths and ensures they are present in the store. + // + // In this implementation, we do none of that, no syntax checks, no realization. + // The next `TODO` are the checks that Nix implements. + let mut ctx_elements: HashSet<NixContextElement> = HashSet::new(); + let span = generators::request_span(&co).await; + let origin = origin + .coerce_to_string( + co, + CoercionKind { + strong: true, + import_paths: true, + }, + span, + ) + .await?; + let mut origin = origin.to_contextful_str()?; + + let added_context = added_context.to_attrs()?; + for (context_key, context_element) in added_context.into_iter() { + // Invariant checks: + // - TODO: context_key must be a syntactically valid store path. + // - Perform a deep force `context_element`. + let context_element = context_element.to_attrs()?; + if let Some(path) = context_element.select("path") { + if path.as_bool()? { + ctx_elements.insert(NixContextElement::Plain(context_key.to_string())); + } + } + if let Some(all_outputs) = context_element.select("allOutputs") { + if all_outputs.as_bool()? { + // TODO: check if `context_key` is a derivation path. + // This may require realization. + ctx_elements.insert(NixContextElement::Derivation(context_key.to_string())); + } + } + if let Some(some_outputs) = context_element.select("outputs") { + let some_outputs = some_outputs.to_list()?; + // TODO: check if `context_key` is a derivation path. + // This may require realization. + for output in some_outputs.into_iter() { + let output = output.to_str()?; + ctx_elements.insert(NixContextElement::Single { + derivation: context_key.to_string(), + name: output.to_string(), + }); + } + } + } + + if let Some(origin_ctx) = origin.context_mut() { + // FUTUREWORK(performance): avoid this clone + // and extend in-place. + *origin_ctx = origin_ctx.clone().join(&mut ctx_elements.into()); + } + + Ok(origin.into()) + } + + #[builtin("hashString")] + async fn builtin_hash_string(co: GenCo, algo: Value, s: Value) -> Result<Value, ErrorKind> { + hash_nix_string(algo.to_str()?, std::io::Cursor::new(s.to_str()?)).map(Value::from) } #[builtin("head")] @@ -1570,6 +1649,49 @@ mod placeholder_builtins { Ok(Value::from(v)) } + #[builtin("unsafeDiscardOutputDependency")] + async fn builtin_unsafe_discard_output_dependency( + co: GenCo, + s: Value, + ) -> Result<Value, ErrorKind> { + let span = generators::request_span(&co).await; + let mut v = s + .coerce_to_string( + co, + // It's weak because + // lists, integers, floats and null are not + // accepted as parameters. + CoercionKind { + strong: false, + import_paths: true, + }, + span, + ) + .await? + .to_contextful_str()?; + + // If there's any context, we will swap any ... by a path one. + if let Some(ctx) = v.context_mut() { + let new_context: tvix_eval::NixContext = ctx + .iter() + .map(|elem| match elem { + // FUTUREWORK(performance): ideally, we should either: + // (a) do interior mutation of the existing context. + // (b) let the structural sharing make those clones cheap. + crate::NixContextElement::Derivation(drv_path) => { + crate::NixContextElement::Plain(drv_path.to_string()) + } + elem => elem.clone(), + }) + .collect::<HashSet<_>>() + .into(); + + *ctx = new_context; + } + + Ok(Value::from(v)) + } + #[builtin("addErrorContext")] async fn builtin_add_error_context( co: GenCo, @@ -1594,7 +1716,7 @@ mod placeholder_builtins { .await; let res = [ ("line", 42.into()), - ("col", 42.into()), + ("column", 42.into()), ("file", Value::String("/deep/thought".into())), ]; Ok(Value::attrs(NixAttrs::from_iter(res.into_iter()))) diff --git a/tvix/eval/src/builtins/to_xml.rs b/tvix/eval/src/builtins/to_xml.rs index 2f9a11e788..bb12cebfc9 100644 --- a/tvix/eval/src/builtins/to_xml.rs +++ b/tvix/eval/src/builtins/to_xml.rs @@ -137,7 +137,7 @@ fn value_variant_to_xml<W: Write>(w: &mut EventWriter<W>, value: &Value) -> Resu | Value::Blueprint(_) | Value::DeferredUpvalue(_) | Value::UnresolvedPath(_) - | Value::Json(_) + | Value::Json(..) | Value::FinaliseRequest(_) => { return Err(ErrorKind::TvixBug { msg: "internal value variant encountered in builtins.toXML", diff --git a/tvix/eval/src/compiler/import.rs b/tvix/eval/src/compiler/import.rs index c56909e958..9036eec817 100644 --- a/tvix/eval/src/compiler/import.rs +++ b/tvix/eval/src/compiler/import.rs @@ -6,7 +6,6 @@ //! instance, or observers). use super::GlobalsMap; -use bstr::ByteSlice; use genawaiter::rc::Gen; use std::rc::Weak; @@ -39,9 +38,11 @@ async fn import_impl( return Ok(cached); } - // TODO(tazjin): make this return a string directly instead - let contents: Value = generators::request_read_to_string(&co, path.clone()).await; - let contents = contents.to_str()?.to_str()?.to_owned(); + let mut reader = generators::request_open_file(&co, path.clone()).await; + // We read to a String instead of a Vec<u8> because rnix only supports + // string source files. + let mut contents = String::new(); + reader.read_to_string(&mut contents)?; let parsed = rnix::ast::Root::parse(&contents); let errors = parsed.errors(); diff --git a/tvix/eval/src/io.rs b/tvix/eval/src/io.rs index 1c38bc68de..f775077af8 100644 --- a/tvix/eval/src/io.rs +++ b/tvix/eval/src/io.rs @@ -16,6 +16,7 @@ //! how store paths are opened and so on. use std::{ + fs::File, io, path::{Path, PathBuf}, }; @@ -48,13 +49,8 @@ pub trait EvalIO { /// * `builtins.pathExists :: path -> bool` fn path_exists(&self, path: &Path) -> io::Result<bool>; - /// Read the file at the specified path to a `Vec<u8>`. - /// - /// This is used for the following language evaluation cases: - /// - /// * `builtins.readFile :: path -> string` - /// * `builtins.import :: path -> any` - fn read_to_end(&self, path: &Path) -> io::Result<Vec<u8>>; + /// Open the file at the specified path to a `io::Read`. + fn open(&self, path: &Path) -> io::Result<Box<dyn io::Read>>; /// Read the directory at the specified path and return the names /// of its entries associated with their [`FileType`]. @@ -99,8 +95,8 @@ impl EvalIO for StdIO { path.try_exists() } - fn read_to_end(&self, path: &Path) -> io::Result<Vec<u8>> { - std::fs::read(path) + fn open(&self, path: &Path) -> io::Result<Box<dyn io::Read>> { + Ok(Box::new(File::open(path)?)) } fn read_dir(&self, path: &Path) -> io::Result<Vec<(bytes::Bytes, FileType)>> { @@ -145,7 +141,7 @@ impl EvalIO for DummyIO { )) } - fn read_to_end(&self, _: &Path) -> io::Result<Vec<u8>> { + fn open(&self, _: &Path) -> io::Result<Box<dyn io::Read>> { Err(io::Error::new( io::ErrorKind::Unsupported, "I/O methods are not implemented in DummyIO", diff --git a/tvix/eval/src/value/json.rs b/tvix/eval/src/value/json.rs index 7ab1655d90..c48e9c1f4e 100644 --- a/tvix/eval/src/value/json.rs +++ b/tvix/eval/src/value/json.rs @@ -6,6 +6,7 @@ use super::{CoercionKind, Value}; use crate::errors::{CatchableErrorKind, ErrorKind}; use crate::generators::{self, GenCo}; +use crate::NixContext; use bstr::ByteSlice; use serde_json::value::to_value; @@ -13,22 +14,32 @@ use serde_json::Value as Json; // name clash with *our* `Value` use serde_json::{Map, Number}; impl Value { - pub async fn into_json( + /// Transforms the structure into a JSON + /// and accumulate all encountered context in the second's element + /// of the return type. + pub async fn into_contextful_json( self, co: &GenCo, - ) -> Result<Result<Json, CatchableErrorKind>, ErrorKind> { + ) -> Result<Result<(Json, NixContext), CatchableErrorKind>, ErrorKind> { let self_forced = generators::request_force(co, self).await; + let mut context = NixContext::new(); let value = match self_forced { Value::Null => Json::Null, Value::Bool(b) => Json::Bool(b), Value::Integer(i) => Json::Number(Number::from(i)), Value::Float(f) => to_value(f)?, - Value::String(s) => Json::String(s.to_str()?.to_owned()), + Value::String(s) => { + context.mimic(&s); + + Json::String(s.to_str()?.to_owned()) + } Value::Path(p) => { let imported = generators::request_path_import(co, *p).await; - Json::String(imported.to_string_lossy().to_string()) + let path = imported.to_string_lossy().to_string(); + context = context.append(crate::NixContextElement::Plain(path.clone())); + Json::String(path) } Value::List(l) => { @@ -36,7 +47,10 @@ impl Value { for val in l.into_iter() { match generators::request_to_json(co, val).await { - Ok(v) => out.push(v), + Ok((v, mut ctx)) => { + context = context.join(&mut ctx); + out.push(v) + } Err(cek) => return Ok(Err(cek)), } } @@ -62,7 +76,14 @@ impl Value { .await? { Value::Catchable(cek) => return Ok(Err(*cek)), - Value::String(s) => return Ok(Ok(Json::String(s.to_str()?.to_owned()))), + Value::String(s) => { + // We need a fresh context here because `__toString` will discard + // everything. + let mut fresh = NixContext::new(); + fresh.mimic(&s); + + return Ok(Ok((Json::String(s.to_str()?.to_owned()), fresh))); + } _ => panic!("Value::coerce_to_string_() returned a non-string!"), } } @@ -79,7 +100,10 @@ impl Value { out.insert( name.to_str()?.to_owned(), match generators::request_to_json(co, value).await { - Ok(v) => v, + Ok((v, mut ctx)) => { + context = context.join(&mut ctx); + v + } Err(cek) => return Ok(Err(cek)), }, ); @@ -97,21 +121,34 @@ impl Value { | val @ Value::Blueprint(_) | val @ Value::DeferredUpvalue(_) | val @ Value::UnresolvedPath(_) - | val @ Value::Json(_) + | val @ Value::Json(..) | val @ Value::FinaliseRequest(_) => { return Err(ErrorKind::NotSerialisableToJson(val.type_of())) } }; - Ok(Ok(value)) + Ok(Ok((value, context))) } /// Generator version of the above, which wraps responses in - /// Value::Json. - pub(crate) async fn into_json_generator(self, co: GenCo) -> Result<Value, ErrorKind> { - match self.into_json(&co).await? { + /// [`Value::Json`]. + pub(crate) async fn into_contextful_json_generator( + self, + co: GenCo, + ) -> Result<Value, ErrorKind> { + match self.into_contextful_json(&co).await? { Err(cek) => Ok(Value::from(cek)), - Ok(json) => Ok(Value::Json(Box::new(json))), + Ok((json, ctx)) => Ok(Value::Json(Box::new((json, ctx)))), } } + + /// Transforms the structure into a JSON + /// All the accumulated context is ignored, use [`into_contextful_json`] + /// to obtain the resulting context of the JSON object. + pub async fn into_json( + self, + co: &GenCo, + ) -> Result<Result<Json, CatchableErrorKind>, ErrorKind> { + Ok(self.into_contextful_json(co).await?.map(|(json, _)| json)) + } } diff --git a/tvix/eval/src/value/mod.rs b/tvix/eval/src/value/mod.rs index e8e27e5968..c171c9a04e 100644 --- a/tvix/eval/src/value/mod.rs +++ b/tvix/eval/src/value/mod.rs @@ -78,7 +78,7 @@ pub enum Value { #[serde(skip)] UnresolvedPath(Box<PathBuf>), #[serde(skip)] - Json(Box<serde_json::Value>), + Json(Box<(serde_json::Value, NixContext)>), #[serde(skip)] FinaliseRequest(bool), @@ -294,7 +294,7 @@ impl Value { | Value::Blueprint(_) | Value::DeferredUpvalue(_) | Value::UnresolvedPath(_) - | Value::Json(_) + | Value::Json(..) | Value::FinaliseRequest(_) => panic!( "Tvix bug: internal value left on stack: {}", value.type_of() @@ -444,7 +444,7 @@ impl Value { | (Value::Blueprint(_), _) | (Value::DeferredUpvalue(_), _) | (Value::UnresolvedPath(_), _) - | (Value::Json(_), _) + | (Value::Json(..), _) | (Value::FinaliseRequest(_), _) => { panic!("tvix bug: .coerce_to_string() called on internal value") } @@ -681,7 +681,7 @@ impl Value { Value::Blueprint(_) => "internal[blueprint]", Value::DeferredUpvalue(_) => "internal[deferred_upvalue]", Value::UnresolvedPath(_) => "internal[unresolved_path]", - Value::Json(_) => "internal[json]", + Value::Json(..) => "internal[json]", Value::FinaliseRequest(_) => "internal[finaliser_sentinel]", Value::Catchable(_) => "internal[catchable]", } @@ -877,7 +877,7 @@ impl Value { | Value::Blueprint(_) | Value::DeferredUpvalue(_) | Value::UnresolvedPath(_) - | Value::Json(_) + | Value::Json(..) | Value::FinaliseRequest(_) => "an internal Tvix evaluator value".into(), } } @@ -991,7 +991,7 @@ impl TotalDisplay for Value { Value::Blueprint(_) => f.write_str("internal[blueprint]"), Value::DeferredUpvalue(_) => f.write_str("internal[deferred_upvalue]"), Value::UnresolvedPath(_) => f.write_str("internal[unresolved_path]"), - Value::Json(_) => f.write_str("internal[json]"), + Value::Json(..) => f.write_str("internal[json]"), Value::FinaliseRequest(_) => f.write_str("internal[finaliser_sentinel]"), // Delegate thunk display to the type, as it must handle diff --git a/tvix/eval/src/value/string.rs b/tvix/eval/src/value/string.rs index 6ce0d190c0..ceb43f1ea5 100644 --- a/tvix/eval/src/value/string.rs +++ b/tvix/eval/src/value/string.rs @@ -48,6 +48,12 @@ impl From<NixContextElement> for NixContext { } } +impl From<HashSet<NixContextElement>> for NixContext { + fn from(value: HashSet<NixContextElement>) -> Self { + Self(value) + } +} + impl NixContext { /// Creates an empty context that can be populated /// and passed to form a contextful [NixString], albeit @@ -524,11 +530,7 @@ impl<'a> From<&'a NixString> for &'a BStr { } } -impl From<NixString> for String { - fn from(s: NixString) -> Self { - s.to_string() - } -} +// No impl From<NixString> for String, that one quotes. impl From<NixString> for BString { fn from(s: NixString) -> Self { diff --git a/tvix/eval/src/vm/generators.rs b/tvix/eval/src/vm/generators.rs index 4dacdef0dd..79de688692 100644 --- a/tvix/eval/src/vm/generators.rs +++ b/tvix/eval/src/vm/generators.rs @@ -102,8 +102,8 @@ pub enum VMRequest { /// Request that the VM imports the given path through its I/O interface. PathImport(PathBuf), - /// Request that the VM reads the given path to a string. - ReadToString(PathBuf), + /// Request that the VM opens the specified file and provides a reader. + OpenFile(PathBuf), /// Request that the VM checks whether the given path exists. PathExists(PathBuf), @@ -170,8 +170,8 @@ impl Display for VMRequest { write!(f, "import_cache_put({})", p.to_string_lossy()) } VMRequest::PathImport(p) => write!(f, "path_import({})", p.to_string_lossy()), - VMRequest::ReadToString(p) => { - write!(f, "read_to_string({})", p.to_string_lossy()) + VMRequest::OpenFile(p) => { + write!(f, "open_file({})", p.to_string_lossy()) } VMRequest::PathExists(p) => write!(f, "path_exists({})", p.to_string_lossy()), VMRequest::ReadDir(p) => write!(f, "read_dir({})", p.to_string_lossy()), @@ -199,6 +199,9 @@ pub enum VMResponse { /// VM response with a span to use at the current point. Span(LightSpan), + + /// [std::io::Reader] produced by the VM in response to some IO operation. + Reader(Box<dyn std::io::Read>), } impl Display for VMResponse { @@ -209,6 +212,7 @@ impl Display for VMResponse { VMResponse::Path(p) => write!(f, "path({})", p.to_string_lossy()), VMResponse::Directory(d) => write!(f, "dir(len = {})", d.len()), VMResponse::Span(_) => write!(f, "span"), + VMResponse::Reader(_) => write!(f, "reader"), } } } @@ -425,18 +429,18 @@ where message = VMResponse::Path(imported); } - VMRequest::ReadToString(path) => { - let content = self + VMRequest::OpenFile(path) => { + let reader = self .io_handle .as_ref() - .read_to_end(&path) + .open(&path) .map_err(|e| ErrorKind::IO { path: Some(path), error: e.into(), }) .with_span(&span, self)?; - message = VMResponse::Value(content.into()) + message = VMResponse::Reader(reader) } VMRequest::PathExists(path) => { @@ -489,7 +493,7 @@ where VMRequest::ToJson(value) => { self.reenqueue_generator(name, span.clone(), generator); self.enqueue_generator("to_json", span, |co| { - value.into_json_generator(co) + value.into_contextful_json_generator(co) }); return Ok(false); } @@ -730,9 +734,10 @@ pub(crate) async fn request_path_import(co: &GenCo, path: PathBuf) -> PathBuf { } } -pub(crate) async fn request_read_to_string(co: &GenCo, path: PathBuf) -> Value { - match co.yield_(VMRequest::ReadToString(path)).await { - VMResponse::Value(value) => value, +/// Request that the VM open a [std::io::Read] for the specified file. +pub async fn request_open_file(co: &GenCo, path: PathBuf) -> Box<dyn std::io::Read> { + match co.yield_(VMRequest::OpenFile(path)).await { + VMResponse::Reader(value) => value, msg => panic!( "Tvix bug: VM responded with incorrect generator message: {}", msg @@ -773,9 +778,9 @@ pub(crate) async fn request_span(co: &GenCo) -> LightSpan { pub(crate) async fn request_to_json( co: &GenCo, value: Value, -) -> Result<serde_json::Value, CatchableErrorKind> { +) -> Result<(serde_json::Value, NixContext), CatchableErrorKind> { match co.yield_(VMRequest::ToJson(value)).await { - VMResponse::Value(Value::Json(json)) => Ok(*json), + VMResponse::Value(Value::Json(json_with_ctx)) => Ok(*json_with_ctx), VMResponse::Value(Value::Catchable(cek)) => Err(*cek), msg => panic!( "Tvix bug: VM responded with incorrect generator message: {}", diff --git a/tvix/eval/src/vm/mod.rs b/tvix/eval/src/vm/mod.rs index c10b79cd99..5c244cc3ca 100644 --- a/tvix/eval/src/vm/mod.rs +++ b/tvix/eval/src/vm/mod.rs @@ -1148,7 +1148,7 @@ where let mut captured_with_stack = frame .upvalues .with_stack() - .map(Clone::clone) + .cloned() // ... or make an empty one if there isn't one already. .unwrap_or_else(|| Vec::with_capacity(self.with_stack.len())); diff --git a/tvix/eval/tests/nix_oracle.rs b/tvix/eval/tests/nix_oracle.rs index 6bab75cfd9..5a5cc0a822 100644 --- a/tvix/eval/tests/nix_oracle.rs +++ b/tvix/eval/tests/nix_oracle.rs @@ -30,7 +30,14 @@ fn nix_eval(expr: &str, strictness: Strictness) -> String { .arg(format!("({expr})")) .env( "NIX_REMOTE", - format!("local?root={}", store_dir.path().display()), + format!( + "local?root={}", + store_dir + .path() + .canonicalize() + .expect("valid path") + .display() + ), ) .output() .unwrap(); diff --git a/tvix/glue/Cargo.toml b/tvix/glue/Cargo.toml index 1ec3832d91..0afdefeaaa 100644 --- a/tvix/glue/Cargo.toml +++ b/tvix/glue/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -async-recursion = "1.0.5" +async-compression = { version = "0.4.9", features = ["tokio", "gzip", "bzip2", "xz"]} bstr = "1.6.0" bytes = "1.4.0" data-encoding = "2.3.3" @@ -25,12 +25,11 @@ thiserror = "1.0.38" serde = "1.0.195" serde_json = "1.0" sha2 = "0.10.8" +sha1 = "0.10.6" +md-5 = "0.10.6" +url = "2.4.0" walkdir = "2.4.0" -[dependencies.async-compression] -version = "0.4.6" -features = ["tokio", "gzip", "bzip2", "xz"] - [dependencies.wu-manber] git = "https://github.com/tvlfyi/wu-manber.git" @@ -40,9 +39,8 @@ hex-literal = "0.4.1" lazy_static = "1.4.0" nix = { version = "0.27.1", features = [ "fs" ] } pretty_assertions = "1.4.0" -rstest = "0.18.2" +rstest = "0.19.0" tempfile = "3.8.1" -test-case = "3.3.1" [features] default = ["nix_tests"] diff --git a/tvix/glue/src/builtins/derivation.rs b/tvix/glue/src/builtins/derivation.rs index 4ea9611a94..a7742ae40a 100644 --- a/tvix/glue/src/builtins/derivation.rs +++ b/tvix/glue/src/builtins/derivation.rs @@ -1,5 +1,6 @@ //! Implements `builtins.derivation`, the core of what makes Nix build packages. use crate::builtins::DerivationError; +use crate::known_paths::KnownPaths; use crate::tvix_store_io::TvixStoreIO; use bstr::BString; use nix_compat::derivation::{Derivation, Output}; @@ -19,7 +20,7 @@ const IGNORE_NULLS: &str = "__ignoreNulls"; /// Populate the inputs of a derivation from the build references /// found when scanning the derivation's parameters and extracting their contexts. -fn populate_inputs(drv: &mut Derivation, full_context: NixContext) { +fn populate_inputs(drv: &mut Derivation, full_context: NixContext, known_paths: &KnownPaths) { for element in full_context.iter() { match element { NixContextElement::Plain(source) => { @@ -57,12 +58,37 @@ fn populate_inputs(drv: &mut Derivation, full_context: NixContext) { } } - NixContextElement::Derivation(_drv_path) => { - // This is a hard one, it means that - // we are depending on a drvPath of ourselves - // *or* another derivation's drvPath. - // What to do here? - panic!("please do not depend on drvPath, I have 2 hours of sleep in blood"); + NixContextElement::Derivation(drv_path) => { + let (derivation, _rest) = + StorePath::from_absolute_path_full(drv_path).expect("valid store path"); + + #[cfg(debug_assertions)] + assert!( + _rest.iter().next().is_none(), + "Extra path not empty for {}", + drv_path + ); + + // We need to know all the outputs *names* of that derivation. + let output_names = known_paths + .get_drv_by_drvpath(&derivation) + .expect("no known derivation associated to that derivation path") + .outputs + .keys(); + + // FUTUREWORK(performance): ideally, we should be able to clone + // cheaply those outputs rather than duplicate them all around. + match drv.input_derivations.entry(derivation.clone()) { + btree_map::Entry::Vacant(entry) => { + entry.insert(output_names.cloned().collect()); + } + + btree_map::Entry::Occupied(mut entry) => { + entry.get_mut().extend(output_names.cloned()); + } + } + + drv.input_sources.insert(derivation); } } } @@ -290,8 +316,16 @@ pub(crate) mod derivation_builtins { output_names.push(output_name.to_str()?.to_owned()); } - // Add drv.environment[outputs] unconditionally. - insert_env(&mut drv, arg_name, output_names.join(" ").into())?; + match structured_attrs.as_mut() { + // add outputs to the json itself (as a list of strings) + Some(structured_attrs) => { + structured_attrs.insert(arg_name.into(), output_names.into()); + } + // add drv.environment["outputs"] as a space-separated list + None => { + insert_env(&mut drv, arg_name, output_names.join(" ").into())?; + } + } // drv.environment[$output_name] is added after the loop, // with whatever is in drv.outputs[$output_name]. } @@ -338,14 +372,13 @@ pub(crate) mod derivation_builtins { return Ok(val); } - // TODO(raitobezarius): context for json values? - // input_context.mimic(&val); - - let val_json = match val.into_json(&co).await? { + let (val_json, mut context) = match val.into_contextful_json(&co).await? { Ok(v) => v, Err(cek) => return Ok(Value::from(cek)), }; + input_context = input_context.join(&mut context); + // No need to check for dups, we only iterate over every attribute name once structured_attrs.insert(arg_name.to_owned(), val_json); } else { @@ -416,63 +449,67 @@ pub(crate) mod derivation_builtins { ); } - populate_inputs(&mut drv, input_context); let mut known_paths = state.as_ref().known_paths.borrow_mut(); + populate_inputs(&mut drv, input_context, &known_paths); // At this point, derivation fields are fully populated from // eval data structures. drv.validate(false) .map_err(DerivationError::InvalidDerivation)?; - // Calculate the derivation_or_fod_hash for the current derivation. - // This one is still intermediate (so not added to known_paths) - let derivation_or_fod_hash_tmp = drv.derivation_or_fod_hash(|drv_path| { - known_paths - .get_hash_derivation_modulo(&drv_path.to_owned()) - .unwrap_or_else(|| panic!("{} not found", drv_path)) - .to_owned() - }); + // Calculate the hash_derivation_modulo for the current derivation.. + debug_assert!( + drv.outputs.values().all(|output| { output.path.is_none() }), + "outputs should still be unset" + ); // Mutate the Derivation struct and set output paths - drv.calculate_output_paths(name, &derivation_or_fod_hash_tmp) - .map_err(DerivationError::InvalidDerivation)?; + drv.calculate_output_paths( + name, + // This one is still intermediate (so not added to known_paths), + // as the outputs are still unset. + &drv.hash_derivation_modulo(|drv_path| { + *known_paths + .get_hash_derivation_modulo(&drv_path.to_owned()) + .unwrap_or_else(|| panic!("{} not found", drv_path)) + }), + ) + .map_err(DerivationError::InvalidDerivation)?; let drv_path = drv .calculate_derivation_path(name) .map_err(DerivationError::InvalidDerivation)?; - // TODO: avoid cloning - known_paths.add(drv_path.clone(), drv.clone()); - - let mut new_attrs: Vec<(String, NixString)> = drv - .outputs - .into_iter() - .map(|(name, output)| { - ( - name.clone(), + // Assemble the attrset to return from this builtin. + let out = Value::Attrs(Box::new(NixAttrs::from_iter( + drv.outputs + .iter() + .map(|(name, output)| { + ( + name.clone(), + NixString::new_context_from( + NixContextElement::Single { + name: name.clone(), + derivation: drv_path.to_absolute_path(), + } + .into(), + output.path.as_ref().unwrap().to_absolute_path(), + ), + ) + }) + .chain(std::iter::once(( + "drvPath".to_owned(), NixString::new_context_from( - NixContextElement::Single { - name, - derivation: drv_path.to_absolute_path(), - } - .into(), - output.path.unwrap().to_absolute_path(), + NixContextElement::Derivation(drv_path.to_absolute_path()).into(), + drv_path.to_absolute_path(), ), - ) - }) - .collect(); - - new_attrs.push(( - "drvPath".to_string(), - NixString::new_context_from( - NixContextElement::Derivation(drv_path.to_absolute_path()).into(), - drv_path.to_absolute_path(), - ), - )); - - Ok(Value::Attrs(Box::new(NixAttrs::from_iter( - new_attrs.into_iter(), - )))) + ))), + ))); + + // Register the Derivation in known_paths. + known_paths.add_derivation(drv_path, drv); + + Ok(out) } #[builtin("toFile")] diff --git a/tvix/glue/src/builtins/errors.rs b/tvix/glue/src/builtins/errors.rs index c753a125e0..f6d5745c56 100644 --- a/tvix/glue/src/builtins/errors.rs +++ b/tvix/glue/src/builtins/errors.rs @@ -3,8 +3,10 @@ use nix_compat::{ nixhash::{self, NixHash}, store_path::BuildStorePathError, }; +use reqwest::Url; use std::rc::Rc; use thiserror::Error; +use tvix_castore::import; /// Errors related to derivation construction #[derive(Debug, Error)] @@ -33,7 +35,7 @@ impl From<DerivationError> for tvix_eval::ErrorKind { pub enum FetcherError { #[error("hash mismatch in file downloaded from {url}:\n wanted: {wanted}\n got: {got}")] HashMismatch { - url: String, + url: Url, wanted: NixHash, got: NixHash, }, @@ -41,15 +43,34 @@ pub enum FetcherError { #[error("Invalid hash type '{0}' for fetcher")] InvalidHashType(&'static str), - #[error("Error in store path for fetcher output: {0}")] - StorePath(#[from] BuildStorePathError), + #[error("Unable to parse URL: {0}")] + InvalidUrl(#[from] url::ParseError), #[error(transparent)] Http(#[from] reqwest::Error), + + #[error(transparent)] + Io(#[from] std::io::Error), + + #[error(transparent)] + Import(#[from] tvix_castore::import::IngestionError<import::archive::Error>), + + #[error("Error calculating store path for fetcher output: {0}")] + StorePath(#[from] BuildStorePathError), +} + +/// Errors related to `builtins.path` and `builtins.filterSource`, +/// a.k.a. "importing" builtins. +#[derive(Debug, Error)] +pub enum ImportError { + #[error("non-file '{0}' cannot be imported in 'flat' mode")] + FlatImportOfNonFile(String), + #[error("hash mismatch at ingestion of '{0}', expected: '{1}', got: '{2}'")] + HashMismatch(String, NixHash, NixHash), } -impl From<FetcherError> for tvix_eval::ErrorKind { - fn from(err: FetcherError) -> Self { +impl From<ImportError> for tvix_eval::ErrorKind { + fn from(err: ImportError) -> Self { tvix_eval::ErrorKind::TvixError(Rc::new(err)) } } diff --git a/tvix/glue/src/builtins/fetchers.rs b/tvix/glue/src/builtins/fetchers.rs index 07074d93de..c7602c03e8 100644 --- a/tvix/glue/src/builtins/fetchers.rs +++ b/tvix/glue/src/builtins/fetchers.rs @@ -1,196 +1,128 @@ -//! Contains builtins that fetch paths from the Internet +//! Contains builtins that fetch paths from the Internet, or local filesystem. -use crate::tvix_store_io::TvixStoreIO; -use bstr::ByteSlice; -use nix_compat::nixhash::{self, CAHash}; -use nix_compat::store_path::{build_ca_path, StorePathRef}; +use super::utils::select_string; +use crate::{ + fetchers::{url_basename, Fetch}, + tvix_store_io::TvixStoreIO, +}; +use nix_compat::nixhash; +use nix_compat::nixhash::NixHash; use std::rc::Rc; +use tracing::info; use tvix_eval::builtin_macros::builtins; +use tvix_eval::generators::Gen; use tvix_eval::generators::GenCo; -use tvix_eval::{CatchableErrorKind, ErrorKind, NixContextElement, NixString, Value}; - -use super::utils::select_string; -use super::{DerivationError, FetcherError}; - -/// Attempts to mimic `nix::libutil::baseNameOf` -fn url_basename(s: &str) -> &str { - if s.is_empty() { - return ""; - } - - let mut last = s.len() - 1; - if s.chars().nth(last).unwrap() == '/' && last > 0 { - last -= 1; - } - - if last == 0 { - return ""; - } +use tvix_eval::{CatchableErrorKind, ErrorKind, Value}; - let pos = match s[..=last].rfind('/') { - Some(pos) => { - if pos == last - 1 { - 0 - } else { - pos - } - } - None => 0, - }; - - &s[(pos + 1)..=last] -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum HashMode { - Flat, - Recursive, +struct NixFetchArgs { + url_str: String, + name: Option<String>, + sha256: Option<[u8; 32]>, } -/// Struct representing the arguments passed to fetcher functions -#[derive(Debug, PartialEq, Eq)] -struct FetchArgs { - url: String, - name: String, - hash: Option<CAHash>, -} - -impl FetchArgs { - pub fn new( - url: String, - name: Option<String>, - sha256: Option<String>, - mode: HashMode, - ) -> nixhash::Result<Self> { - Ok(Self { - name: name.unwrap_or_else(|| url_basename(&url).to_owned()), - url, - hash: sha256 - .map(|h| { - let hash = nixhash::from_str(&h, Some("sha256"))?; - Ok(match mode { - HashMode::Flat => Some(nixhash::CAHash::Flat(hash)), - HashMode::Recursive => Some(nixhash::CAHash::Nar(hash)), - }) - }) - .transpose()? - .flatten(), - }) - } - - fn store_path(&self) -> Result<Option<StorePathRef>, ErrorKind> { - let Some(h) = &self.hash else { - return Ok(None); - }; - build_ca_path(&self.name, h, Vec::<String>::new(), false) - .map(Some) - .map_err(|e| FetcherError::from(e).into()) +// `fetchurl` and `fetchTarball` accept a single argument, which can either be the URL (as string), +// or an attrset, where `url`, `sha256` and `name` keys are allowed. +async fn extract_fetch_args( + co: &GenCo, + args: Value, +) -> Result<Result<NixFetchArgs, CatchableErrorKind>, ErrorKind> { + if let Ok(url_str) = args.to_str() { + // Get the raw bytes, not the ToString repr. + let url_str = + String::from_utf8(url_str.as_bytes().to_vec()).map_err(|_| ErrorKind::Utf8)?; + return Ok(Ok(NixFetchArgs { + url_str, + name: None, + sha256: None, + })); } - async fn extract( - co: &GenCo, - args: Value, - default_name: Option<&str>, - mode: HashMode, - ) -> Result<Result<Self, CatchableErrorKind>, ErrorKind> { - if let Ok(url) = args.to_str() { - return Ok(Ok(FetchArgs::new( - url.to_str()?.to_owned(), - None, - None, - mode, - ) - .map_err(DerivationError::InvalidOutputHash)?)); - } + let attrs = args.to_attrs().map_err(|_| ErrorKind::TypeError { + expected: "attribute set or contextless string", + actual: args.type_of(), + })?; - let attrs = args.to_attrs().map_err(|_| ErrorKind::TypeError { - expected: "attribute set or string", - actual: args.type_of(), - })?; - - let url = match select_string(co, &attrs, "url").await? { - Ok(s) => s.ok_or_else(|| ErrorKind::AttributeNotFound { name: "url".into() })?, - Err(cek) => return Ok(Err(cek)), - }; - let name = match select_string(co, &attrs, "name").await? { - Ok(s) => s.or_else(|| default_name.map(|s| s.to_owned())), - Err(cek) => return Ok(Err(cek)), - }; - let sha256 = match select_string(co, &attrs, "sha256").await? { - Ok(s) => s, - Err(cek) => return Ok(Err(cek)), - }; - - Ok(Ok( - FetchArgs::new(url, name, sha256, mode).map_err(DerivationError::InvalidOutputHash)? - )) - } -} + let url_str = match select_string(co, &attrs, "url").await? { + Ok(s) => s.ok_or_else(|| ErrorKind::AttributeNotFound { name: "url".into() })?, + Err(cek) => return Ok(Err(cek)), + }; + let name = match select_string(co, &attrs, "name").await? { + Ok(s) => s, + Err(cek) => return Ok(Err(cek)), + }; + let sha256_str = match select_string(co, &attrs, "sha256").await? { + Ok(s) => s, + Err(cek) => return Ok(Err(cek)), + }; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum FetchMode { - Url, - Tarball, -} + // TODO: disallow other attrset keys, to match Nix' behaviour. -impl From<FetchMode> for HashMode { - fn from(value: FetchMode) -> Self { - match value { - FetchMode::Url => HashMode::Flat, - FetchMode::Tarball => HashMode::Recursive, - } - } -} + // parse the sha256 string into a digest. + let sha256 = match sha256_str { + Some(sha256_str) => { + let nixhash = nixhash::from_str(&sha256_str, Some("sha256")) + // TODO: DerivationError::InvalidOutputHash should be moved to ErrorKind::InvalidHash and used here instead + .map_err(|e| ErrorKind::TvixError(Rc::new(e)))?; -impl FetchMode { - fn default_name(self) -> Option<&'static str> { - match self { - FetchMode::Url => None, - FetchMode::Tarball => Some("source"), + Some(nixhash.digest_as_bytes().try_into().expect("is sha256")) } - } -} - -fn string_from_store_path(store_path: StorePathRef) -> NixString { - NixString::new_context_from( - NixContextElement::Plain(store_path.to_absolute_path()).into(), - store_path.to_absolute_path(), - ) -} - -async fn fetch( - state: Rc<TvixStoreIO>, - co: GenCo, - args: Value, - mode: FetchMode, -) -> Result<Value, ErrorKind> { - let args = match FetchArgs::extract(&co, args, mode.default_name(), mode.into()).await? { - Ok(args) => args, - Err(cek) => return Ok(cek.into()), + None => None, }; - if let Some(store_path) = args.store_path()? { - if state.store_path_exists(store_path).await? { - return Ok(string_from_store_path(store_path).into()); - } - } - - let hash = args.hash.as_ref().map(|h| h.hash()); - let store_path = Rc::clone(&state).tokio_handle.block_on(state.fetch_url( - &args.url, - &args.name, - hash.as_deref(), - ))?; - - Ok(string_from_store_path(store_path.as_ref()).into()) + Ok(Ok(NixFetchArgs { + url_str, + name, + sha256, + })) } #[allow(unused_variables)] // for the `state` arg, for now #[builtins(state = "Rc<TvixStoreIO>")] pub(crate) mod fetcher_builtins { + use crate::builtins::FetcherError; + use url::Url; + use super::*; - use tvix_eval::generators::Gen; + /// Consumes a fetch. + /// If there is enough info to calculate the store path without fetching, + /// queue the fetch to be fetched lazily, and return the store path. + /// If there's not enough info to calculate it, do the fetch now, and then + /// return the store path. + fn fetch_lazy(state: Rc<TvixStoreIO>, name: String, fetch: Fetch) -> Result<Value, ErrorKind> { + match fetch + .store_path(&name) + .map_err(|e| ErrorKind::TvixError(Rc::new(e)))? + { + Some(store_path) => { + // Move the fetch to KnownPaths, so it can be actually fetched later. + let sp = state + .known_paths + .borrow_mut() + .add_fetch(fetch, &name) + .expect("Tvix bug: should only fail if the store path cannot be calculated"); + + debug_assert_eq!( + sp, store_path, + "calculated store path by KnownPaths should match" + ); + + // Emit the calculated Store Path. + Ok(Value::Path(Box::new(store_path.to_absolute_path().into()))) + } + None => { + // If we don't have enough info, do the fetch now. + info!(?fetch, "triggering required fetch"); + + let (store_path, _root_node) = state + .tokio_handle + .block_on(async { state.fetcher.ingest_and_persist(&name, fetch).await }) + .map_err(|e| ErrorKind::TvixError(Rc::new(e)))?; + + Ok(Value::Path(Box::new(store_path.to_absolute_path().into()))) + } + } + } #[builtin("fetchurl")] async fn builtin_fetchurl( @@ -198,7 +130,25 @@ pub(crate) mod fetcher_builtins { co: GenCo, args: Value, ) -> Result<Value, ErrorKind> { - fetch(state, co, args, FetchMode::Url).await + let args = match extract_fetch_args(&co, args).await? { + Ok(args) => args, + Err(cek) => return Ok(Value::from(cek)), + }; + + // Derive the name from the URL basename if not set explicitly. + let name = args + .name + .unwrap_or_else(|| url_basename(&args.url_str).to_owned()); + + // Parse the URL. + let url = Url::parse(&args.url_str) + .map_err(|e| ErrorKind::TvixError(Rc::new(FetcherError::InvalidUrl(e))))?; + + fetch_lazy( + state, + name, + Fetch::URL(url, args.sha256.map(NixHash::Sha256)), + ) } #[builtin("fetchTarball")] @@ -207,7 +157,22 @@ pub(crate) mod fetcher_builtins { co: GenCo, args: Value, ) -> Result<Value, ErrorKind> { - fetch(state, co, args, FetchMode::Tarball).await + let args = match extract_fetch_args(&co, args).await? { + Ok(args) => args, + Err(cek) => return Ok(Value::from(cek)), + }; + + // Name defaults to "source" if not set explicitly. + const DEFAULT_NAME_FETCH_TARBALL: &str = "source"; + let name = args + .name + .unwrap_or_else(|| DEFAULT_NAME_FETCH_TARBALL.to_owned()); + + // Parse the URL. + let url = Url::parse(&args.url_str) + .map_err(|e| ErrorKind::TvixError(Rc::new(FetcherError::InvalidUrl(e))))?; + + fetch_lazy(state, name, Fetch::Tarball(url, args.sha256)) } #[builtin("fetchGit")] @@ -219,71 +184,3 @@ pub(crate) mod fetcher_builtins { Err(ErrorKind::NotImplemented("fetchGit")) } } - -#[cfg(test)] -mod tests { - use std::str::FromStr; - - use nix_compat::store_path::StorePath; - - use super::*; - - #[test] - fn fetchurl_store_path() { - let url = "https://raw.githubusercontent.com/aaptel/notmuch-extract-patch/f732a53e12a7c91a06755ebfab2007adc9b3063b/notmuch-extract-patch"; - let sha256 = "0nawkl04sj7psw6ikzay7kydj3dhd0fkwghcsf5rzaw4bmp4kbax"; - let args = FetchArgs::new(url.into(), None, Some(sha256.into()), HashMode::Flat).unwrap(); - - assert_eq!( - args.store_path().unwrap().unwrap().to_owned(), - StorePath::from_str("06qi00hylriyfm0nl827crgjvbax84mz-notmuch-extract-patch").unwrap() - ) - } - - #[test] - fn fetch_tarball_store_path() { - let url = "https://github.com/NixOS/nixpkgs/archive/91050ea1e57e50388fa87a3302ba12d188ef723a.tar.gz"; - let sha256 = "1hf6cgaci1n186kkkjq106ryf8mmlq9vnwgfwh625wa8hfgdn4dm"; - let args = FetchArgs::new( - url.into(), - Some("source".into()), - Some(sha256.into()), - HashMode::Recursive, - ) - .unwrap(); - - assert_eq!( - args.store_path().unwrap().unwrap().to_owned(), - StorePath::from_str("7adgvk5zdfq4pwrhsm3n9lzypb12gw0g-source").unwrap() - ) - } - - mod url_basename { - use super::*; - - #[test] - fn empty_path() { - assert_eq!(url_basename(""), ""); - } - - #[test] - fn path_on_root() { - assert_eq!(url_basename("/dir"), "dir"); - } - - #[test] - fn relative_path() { - assert_eq!(url_basename("dir/foo"), "foo"); - } - - #[test] - fn root_with_trailing_slash() { - assert_eq!(url_basename("/"), ""); - } - - #[test] - fn trailing_slash() { - assert_eq!(url_basename("/dir/"), "dir"); - } - } -} diff --git a/tvix/glue/src/builtins/import.rs b/tvix/glue/src/builtins/import.rs index 08f8a40636..219695b69f 100644 --- a/tvix/glue/src/builtins/import.rs +++ b/tvix/glue/src/builtins/import.rs @@ -1,11 +1,12 @@ //! Implements builtins used to import paths in the store. -use futures::pin_mut; +use crate::builtins::errors::ImportError; use std::path::Path; +use tvix_castore::import::ingest_entries; use tvix_eval::{ builtin_macros::builtins, generators::{self, GenCo}, - ErrorKind, Value, + ErrorKind, EvalIO, Value, }; use std::rc::Rc; @@ -16,16 +17,15 @@ async fn filtered_ingest( path: &Path, filter: Option<&Value>, ) -> Result<tvix_castore::proto::node::Node, ErrorKind> { - let mut entries_per_depths: Vec<Vec<walkdir::DirEntry>> = vec![Vec::new()]; + let mut entries: Vec<walkdir::DirEntry> = vec![]; let mut it = walkdir::WalkDir::new(path) .follow_links(false) .follow_root_links(false) .contents_first(false) - .sort_by_file_name() .into_iter(); // Skip root node. - entries_per_depths[0].push( + entries.push( it.next() .ok_or_else(|| ErrorKind::IO { path: Some(path.to_path_buf()), @@ -82,32 +82,24 @@ async fn filtered_ingest( continue; } - if entry.depth() >= entries_per_depths.len() { - debug_assert!( - entry.depth() == entries_per_depths.len(), - "Received unexpected entry with depth {} during descent, previously at {}", - entry.depth(), - entries_per_depths.len() - ); - - entries_per_depths.push(vec![entry]); - } else { - entries_per_depths[entry.depth()].push(entry); - } - - // FUTUREWORK: determine when it's the right moment to flush a level to the ingester. + entries.push(entry); } - let entries_stream = tvix_castore::import::leveled_entries_to_stream(entries_per_depths); - - pin_mut!(entries_stream); + let dir_entries = entries.into_iter().rev().map(Ok); - state - .ingest_entries_sync(entries_stream) - .map_err(|err| ErrorKind::IO { - path: Some(path.to_path_buf()), - error: err.into(), - }) + state.tokio_handle.block_on(async { + let entries = tvix_castore::import::fs::dir_entries_to_ingestion_stream( + &state.blob_service, + dir_entries, + path, + ); + ingest_entries(&state.directory_service, entries) + .await + .map_err(|e| ErrorKind::IO { + path: Some(path.to_path_buf()), + error: Rc::new(std::io::Error::new(std::io::ErrorKind::Other, e)), + }) + }) } #[builtins(state = "Rc<TvixStoreIO>")] @@ -116,11 +108,138 @@ mod import_builtins { use super::*; + use nix_compat::nixhash::{CAHash, NixHash}; use tvix_eval::generators::Gen; use tvix_eval::{generators::GenCo, ErrorKind, Value}; + use tvix_eval::{NixContextElement, NixString}; + + use tvix_castore::B3Digest; use crate::tvix_store_io::TvixStoreIO; + #[builtin("path")] + async fn builtin_path( + state: Rc<TvixStoreIO>, + co: GenCo, + args: Value, + ) -> Result<Value, ErrorKind> { + let args = args.to_attrs()?; + let path = args.select_required("path")?; + let path = generators::request_force(&co, path.clone()) + .await + .to_path()?; + let name: String = if let Some(name) = args.select("name") { + generators::request_force(&co, name.clone()) + .await + .to_str()? + .as_bstr() + .to_string() + } else { + tvix_store::import::path_to_name(&path) + .expect("Failed to derive the default name out of the path") + .to_string() + }; + let filter = args.select("filter"); + let recursive_ingestion = args + .select("recursive") + .map(|r| r.as_bool()) + .transpose()? + .unwrap_or(true); // Yes, yes, Nix, by default, puts `recursive = true;`. + let expected_sha256 = args + .select("sha256") + .map(|h| { + h.to_str().and_then(|expected| { + let expected = expected.into_bstring().to_string(); + // TODO: ensure that we fail if this is not a valid str. + nix_compat::nixhash::from_str(&expected, None).map_err(|_err| { + // TODO: a better error would be nice, we use + // DerivationError::InvalidOutputHash usually for derivation construction. + // This is not a derivation construction, should we move it outside and + // generalize? + ErrorKind::TypeError { + expected: "sha256", + actual: "not a sha256", + } + }) + }) + }) + .transpose()?; + + // FUTUREWORK(performance): this opens the file instead of using a stat-like + // system call to the file. + if !recursive_ingestion && state.open(path.as_ref()).is_err() { + Err(ImportError::FlatImportOfNonFile( + path.to_string_lossy().to_string(), + ))?; + } + + let root_node = filtered_ingest(state.clone(), co, path.as_ref(), filter).await?; + let ca: CAHash = if recursive_ingestion { + CAHash::Nar(NixHash::Sha256(state.tokio_handle.block_on(async { + Ok::<_, tvix_eval::ErrorKind>( + state + .path_info_service + .as_ref() + .calculate_nar(&root_node) + .await + .map_err(|e| ErrorKind::TvixError(Rc::new(e)))? + .1, + ) + })?)) + } else { + let digest: B3Digest = match root_node { + tvix_castore::proto::node::Node::File(ref fnode) => { + // It's already validated. + fnode.digest.clone().try_into().unwrap() + } + // We cannot hash anything else than file in flat import mode. + _ => { + return Err(ImportError::FlatImportOfNonFile( + path.to_string_lossy().to_string(), + ) + .into()) + } + }; + + // FUTUREWORK: avoid hashing again. + CAHash::Flat(NixHash::Sha256( + state + .tokio_handle + .block_on(async { state.blob_to_sha256_hash(digest).await })?, + )) + }; + + let obtained_hash = ca.hash().clone().into_owned(); + let (path_info, _hash, output_path) = state.tokio_handle.block_on(async { + state + .node_to_path_info(name.as_ref(), path.as_ref(), ca, root_node) + .await + })?; + + if let Some(expected_sha256) = expected_sha256 { + if obtained_hash != expected_sha256 { + Err(ImportError::HashMismatch( + path.to_string_lossy().to_string(), + expected_sha256, + obtained_hash, + ))?; + } + } + + let _: tvix_store::proto::PathInfo = state.tokio_handle.block_on(async { + // This is necessary to cause the coercion of the error type. + Ok::<_, std::io::Error>(state.path_info_service.as_ref().put(path_info).await?) + })?; + + // We need to attach context to the final output path. + let outpath = output_path.to_absolute_path(); + + Ok( + NixString::new_context_from(NixContextElement::Plain(outpath.clone()).into(), outpath) + .into(), + ) + } + #[builtin("filterSource")] async fn builtin_filter_source( state: Rc<TvixStoreIO>, @@ -132,14 +251,34 @@ mod import_builtins { let root_node = filtered_ingest(Rc::clone(&state), co, &p, Some(&filter)).await?; let name = tvix_store::import::path_to_name(&p)?; - Ok(state - .register_node_in_path_info_service_sync(name, &p, root_node) + let outpath = state + .tokio_handle + .block_on(async { + let (_, nar_sha256) = state + .path_info_service + .as_ref() + .calculate_nar(&root_node) + .await?; + + state + .register_node_in_path_info_service( + name, + &p, + CAHash::Nar(NixHash::Sha256(nar_sha256)), + root_node, + ) + .await + }) .map_err(|err| ErrorKind::IO { path: Some(p.to_path_buf()), error: err.into(), })? - .to_absolute_path() - .into()) + .to_absolute_path(); + + Ok( + NixString::new_context_from(NixContextElement::Plain(outpath.clone()).into(), outpath) + .into(), + ) } } diff --git a/tvix/glue/src/builtins/mod.rs b/tvix/glue/src/builtins/mod.rs index 138a52633d..0c7bcc880a 100644 --- a/tvix/glue/src/builtins/mod.rs +++ b/tvix/glue/src/builtins/mod.rs @@ -10,7 +10,7 @@ mod fetchers; mod import; mod utils; -pub use errors::{DerivationError, FetcherError}; +pub use errors::{DerivationError, FetcherError, ImportError}; /// Adds derivation-related builtins to the passed [tvix_eval::Evaluation]. /// @@ -56,8 +56,8 @@ mod tests { use super::{add_derivation_builtins, add_fetcher_builtins, add_import_builtins}; use nix_compat::store_path::hash_placeholder; + use rstest::rstest; use tempfile::TempDir; - use test_case::test_case; use tvix_build::buildservice::DummyBuildService; use tvix_eval::{EvalIO, EvaluationResult}; use tvix_store::utils::construct_services; @@ -119,26 +119,27 @@ mod tests { /// construct some calls to builtins.derivation and compare produced output /// paths. - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha256"; outputHash = "sha256-Q3QXOoy+iN4VK2CflvRulYvPZXYgF0dO7FoF7CvWFTA="; }).outPath"#, "/nix/store/17wgs52s7kcamcyin4ja58njkf91ipq8-foo"; "r:sha256")] - #[test_case(r#"(builtins.derivation { name = "foo2"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha256"; outputHash = "sha256-Q3QXOoy+iN4VK2CflvRulYvPZXYgF0dO7FoF7CvWFTA="; }).outPath"#, "/nix/store/gi0p8vd635vpk1nq029cz3aa3jkhar5k-foo2"; "r:sha256 other name")] - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha1"; outputHash = "sha1-VUCRC+16gU5lcrLYHlPSUyx0Y/Q="; }).outPath"#, "/nix/store/p5sammmhpa84ama7ymkbgwwzrilva24x-foo"; "r:sha1")] - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "md5"; outputHash = "md5-07BzhNET7exJ6qYjitX/AA=="; }).outPath"#, "/nix/store/gmmxgpy1jrzs86r5y05wy6wiy2m15xgi-foo"; "r:md5")] - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha512"; outputHash = "sha512-DPkYCnZKuoY6Z7bXLwkYvBMcZ3JkLLLc5aNPCnAvlHDdwr8SXBIZixmVwjPDS0r9NGxUojNMNQqUilG26LTmtg=="; }).outPath"#, "/nix/store/lfi2bfyyap88y45mfdwi4j99gkaxaj19-foo"; "r:sha512")] - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha256"; outputHash = "4374173a8cbe88de152b609f96f46e958bcf65762017474eec5a05ec2bd61530"; }).outPath"#, "/nix/store/17wgs52s7kcamcyin4ja58njkf91ipq8-foo"; "r:sha256 base16")] - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha256"; outputHash = "0c0msqmyq1asxi74f5r0frjwz2wmdvs9d7v05caxx25yihx1fx23"; }).outPath"#, "/nix/store/17wgs52s7kcamcyin4ja58njkf91ipq8-foo"; "r:sha256 nixbase32")] - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha256"; outputHash = "Q3QXOoy+iN4VK2CflvRulYvPZXYgF0dO7FoF7CvWFTA="; }).outPath"#, "/nix/store/17wgs52s7kcamcyin4ja58njkf91ipq8-foo"; "r:sha256 base64")] - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha256"; outputHash = "sha256-fgIr3TyFGDAXP5+qoAaiMKDg/a1MlT6Fv/S/DaA24S8="; }).outPath"#, "/nix/store/xm1l9dx4zgycv9qdhcqqvji1z88z534b-foo"; "r:sha256 base64 nopad")] - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "flat"; outputHashAlgo = "sha256"; outputHash = "sha256-Q3QXOoy+iN4VK2CflvRulYvPZXYgF0dO7FoF7CvWFTA="; }).outPath"#, "/nix/store/q4pkwkxdib797fhk22p0k3g1q32jmxvf-foo"; "sha256")] - #[test_case(r#"(builtins.derivation { name = "foo2"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "flat"; outputHashAlgo = "sha256"; outputHash = "sha256-Q3QXOoy+iN4VK2CflvRulYvPZXYgF0dO7FoF7CvWFTA="; }).outPath"#, "/nix/store/znw17xlmx9r6gw8izjkqxkl6s28sza4l-foo2"; "sha256 other name")] - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "flat"; outputHashAlgo = "sha1"; outputHash = "sha1-VUCRC+16gU5lcrLYHlPSUyx0Y/Q="; }).outPath"#, "/nix/store/zgpnjjmga53d8srp8chh3m9fn7nnbdv6-foo"; "sha1")] - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "flat"; outputHashAlgo = "md5"; outputHash = "md5-07BzhNET7exJ6qYjitX/AA=="; }).outPath"#, "/nix/store/jfhcwnq1852ccy9ad9nakybp2wadngnd-foo"; "md5")] - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "flat"; outputHashAlgo = "sha512"; outputHash = "sha512-DPkYCnZKuoY6Z7bXLwkYvBMcZ3JkLLLc5aNPCnAvlHDdwr8SXBIZixmVwjPDS0r9NGxUojNMNQqUilG26LTmtg=="; }).outPath"#, "/nix/store/as736rr116ian9qzg457f96j52ki8bm3-foo"; "sha512")] - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHash = "sha256-Q3QXOoy+iN4VK2CflvRulYvPZXYgF0dO7FoF7CvWFTA="; }).outPath"#, "/nix/store/17wgs52s7kcamcyin4ja58njkf91ipq8-foo"; "r:sha256 outputHashAlgo omitted")] - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHash = "sha256-Q3QXOoy+iN4VK2CflvRulYvPZXYgF0dO7FoF7CvWFTA="; }).outPath"#, "/nix/store/q4pkwkxdib797fhk22p0k3g1q32jmxvf-foo"; "r:sha256 outputHashAlgo and outputHashMode omitted")] - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; }).outPath"#, "/nix/store/xpcvxsx5sw4rbq666blz6sxqlmsqphmr-foo"; "outputHash* omitted")] - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; outputs = ["foo" "bar"]; system = "x86_64-linux"; }).outPath"#, "/nix/store/hkwdinvz2jpzgnjy9lv34d2zxvclj4s3-foo-foo"; "multiple outputs")] - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; args = ["--foo" "42" "--bar"]; system = "x86_64-linux"; }).outPath"#, "/nix/store/365gi78n2z7vwc1bvgb98k0a9cqfp6as-foo"; "args")] - #[test_case(r#" + #[rstest] + #[case::r_sha256(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha256"; outputHash = "sha256-Q3QXOoy+iN4VK2CflvRulYvPZXYgF0dO7FoF7CvWFTA="; }).outPath"#, "/nix/store/17wgs52s7kcamcyin4ja58njkf91ipq8-foo")] + #[case::r_sha256_other_name(r#"(builtins.derivation { name = "foo2"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha256"; outputHash = "sha256-Q3QXOoy+iN4VK2CflvRulYvPZXYgF0dO7FoF7CvWFTA="; }).outPath"#, "/nix/store/gi0p8vd635vpk1nq029cz3aa3jkhar5k-foo2")] + #[case::r_sha1(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha1"; outputHash = "sha1-VUCRC+16gU5lcrLYHlPSUyx0Y/Q="; }).outPath"#, "/nix/store/p5sammmhpa84ama7ymkbgwwzrilva24x-foo")] + #[case::r_md5(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "md5"; outputHash = "md5-07BzhNET7exJ6qYjitX/AA=="; }).outPath"#, "/nix/store/gmmxgpy1jrzs86r5y05wy6wiy2m15xgi-foo")] + #[case::r_sha512(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha512"; outputHash = "sha512-DPkYCnZKuoY6Z7bXLwkYvBMcZ3JkLLLc5aNPCnAvlHDdwr8SXBIZixmVwjPDS0r9NGxUojNMNQqUilG26LTmtg=="; }).outPath"#, "/nix/store/lfi2bfyyap88y45mfdwi4j99gkaxaj19-foo")] + #[case::r_sha256_base16(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha256"; outputHash = "4374173a8cbe88de152b609f96f46e958bcf65762017474eec5a05ec2bd61530"; }).outPath"#, "/nix/store/17wgs52s7kcamcyin4ja58njkf91ipq8-foo")] + #[case::r_sha256_nixbase32(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha256"; outputHash = "0c0msqmyq1asxi74f5r0frjwz2wmdvs9d7v05caxx25yihx1fx23"; }).outPath"#, "/nix/store/17wgs52s7kcamcyin4ja58njkf91ipq8-foo")] + #[case::r_sha256_base64(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha256"; outputHash = "Q3QXOoy+iN4VK2CflvRulYvPZXYgF0dO7FoF7CvWFTA="; }).outPath"#, "/nix/store/17wgs52s7kcamcyin4ja58njkf91ipq8-foo")] + #[case::r_sha256_base64_nopad(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha256"; outputHash = "sha256-fgIr3TyFGDAXP5+qoAaiMKDg/a1MlT6Fv/S/DaA24S8="; }).outPath"#, "/nix/store/xm1l9dx4zgycv9qdhcqqvji1z88z534b-foo")] + #[case::sha256(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "flat"; outputHashAlgo = "sha256"; outputHash = "sha256-Q3QXOoy+iN4VK2CflvRulYvPZXYgF0dO7FoF7CvWFTA="; }).outPath"#, "/nix/store/q4pkwkxdib797fhk22p0k3g1q32jmxvf-foo")] + #[case::sha256_other_name(r#"(builtins.derivation { name = "foo2"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "flat"; outputHashAlgo = "sha256"; outputHash = "sha256-Q3QXOoy+iN4VK2CflvRulYvPZXYgF0dO7FoF7CvWFTA="; }).outPath"#, "/nix/store/znw17xlmx9r6gw8izjkqxkl6s28sza4l-foo2")] + #[case::sha1(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "flat"; outputHashAlgo = "sha1"; outputHash = "sha1-VUCRC+16gU5lcrLYHlPSUyx0Y/Q="; }).outPath"#, "/nix/store/zgpnjjmga53d8srp8chh3m9fn7nnbdv6-foo")] + #[case::md5(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "flat"; outputHashAlgo = "md5"; outputHash = "md5-07BzhNET7exJ6qYjitX/AA=="; }).outPath"#, "/nix/store/jfhcwnq1852ccy9ad9nakybp2wadngnd-foo")] + #[case::sha512(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "flat"; outputHashAlgo = "sha512"; outputHash = "sha512-DPkYCnZKuoY6Z7bXLwkYvBMcZ3JkLLLc5aNPCnAvlHDdwr8SXBIZixmVwjPDS0r9NGxUojNMNQqUilG26LTmtg=="; }).outPath"#, "/nix/store/as736rr116ian9qzg457f96j52ki8bm3-foo")] + #[case::r_sha256_outputhashalgo_omitted(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHash = "sha256-Q3QXOoy+iN4VK2CflvRulYvPZXYgF0dO7FoF7CvWFTA="; }).outPath"#, "/nix/store/17wgs52s7kcamcyin4ja58njkf91ipq8-foo")] + #[case::r_sha256_outputhashalgo_and_outputhashmode_omitted(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHash = "sha256-Q3QXOoy+iN4VK2CflvRulYvPZXYgF0dO7FoF7CvWFTA="; }).outPath"#, "/nix/store/q4pkwkxdib797fhk22p0k3g1q32jmxvf-foo")] + #[case::outputhash_omitted(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; }).outPath"#, "/nix/store/xpcvxsx5sw4rbq666blz6sxqlmsqphmr-foo")] + #[case::multiple_outputs(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; outputs = ["foo" "bar"]; system = "x86_64-linux"; }).outPath"#, "/nix/store/hkwdinvz2jpzgnjy9lv34d2zxvclj4s3-foo-foo")] + #[case::args(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; args = ["--foo" "42" "--bar"]; system = "x86_64-linux"; }).outPath"#, "/nix/store/365gi78n2z7vwc1bvgb98k0a9cqfp6as-foo")] + #[case::full(r#" let bar = builtins.derivation { name = "bar"; @@ -155,33 +156,34 @@ mod tests { system = ":"; inherit bar; }).outPath - "#, "/nix/store/5vyvcwah9l9kf07d52rcgdk70g2f4y13-foo"; "full")] - #[test_case(r#"(builtins.derivation { "name" = "foo"; passAsFile = ["bar"]; bar = "baz"; system = ":"; builder = ":";}).outPath"#, "/nix/store/25gf0r1ikgmh4vchrn8qlc4fnqlsa5a1-foo"; "passAsFile")] + "#, "/nix/store/5vyvcwah9l9kf07d52rcgdk70g2f4y13-foo")] + #[case::pass_as_file(r#"(builtins.derivation { "name" = "foo"; passAsFile = ["bar"]; bar = "baz"; system = ":"; builder = ":";}).outPath"#, "/nix/store/25gf0r1ikgmh4vchrn8qlc4fnqlsa5a1-foo")] // __ignoreNulls = true, but nothing set to null - #[test_case(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __ignoreNulls = true; }).drvPath"#, "/nix/store/xa96w6d7fxrlkk60z1fmx2ffp2wzmbqx-foo.drv"; "ignoreNulls no arg drvPath")] - #[test_case(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __ignoreNulls = true; }).outPath"#, "/nix/store/pk2agn9za8r9bxsflgh1y7fyyrmwcqkn-foo"; "ignoreNulls no arg outPath")] + #[case::ignore_nulls_true_no_arg_drvpath(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __ignoreNulls = true; }).drvPath"#, "/nix/store/xa96w6d7fxrlkk60z1fmx2ffp2wzmbqx-foo.drv")] + #[case::ignore_nulls_true_no_arg_outpath(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __ignoreNulls = true; }).outPath"#, "/nix/store/pk2agn9za8r9bxsflgh1y7fyyrmwcqkn-foo")] // __ignoreNulls = true, with a null arg, same paths - #[test_case(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __ignoreNulls = true; ignoreme = null; }).drvPath"#, "/nix/store/xa96w6d7fxrlkk60z1fmx2ffp2wzmbqx-foo.drv"; "ignoreNulls drvPath")] - #[test_case(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __ignoreNulls = true; ignoreme = null; }).outPath"#, "/nix/store/pk2agn9za8r9bxsflgh1y7fyyrmwcqkn-foo"; "ignoreNulls outPath")] + #[case::ignore_nulls_true_drvpath(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __ignoreNulls = true; ignoreme = null; }).drvPath"#, "/nix/store/xa96w6d7fxrlkk60z1fmx2ffp2wzmbqx-foo.drv")] + #[case::ignore_nulls_true_outpath(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __ignoreNulls = true; ignoreme = null; }).outPath"#, "/nix/store/pk2agn9za8r9bxsflgh1y7fyyrmwcqkn-foo")] // __ignoreNulls = false - #[test_case(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __ignoreNulls = false; }).drvPath"#, "/nix/store/xa96w6d7fxrlkk60z1fmx2ffp2wzmbqx-foo.drv"; "ignoreNulls false no arg drvPath")] - #[test_case(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __ignoreNulls = false; }).outPath"#, "/nix/store/pk2agn9za8r9bxsflgh1y7fyyrmwcqkn-foo"; "ignoreNulls false no arg arg outPath")] + #[case::ignore_nulls_false_no_arg_drvpath(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __ignoreNulls = false; }).drvPath"#, "/nix/store/xa96w6d7fxrlkk60z1fmx2ffp2wzmbqx-foo.drv")] + #[case::ignore_nulls_false_no_arg_outpath(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __ignoreNulls = false; }).outPath"#, "/nix/store/pk2agn9za8r9bxsflgh1y7fyyrmwcqkn-foo")] // __ignoreNulls = false, with a null arg - #[test_case(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __ignoreNulls = false; foo = null; }).drvPath"#, "/nix/store/xwkwbajfiyhdqmksrbzm0s4g4ib8d4ms-foo.drv"; "ignoreNulls false arg drvPath")] - #[test_case(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __ignoreNulls = false; foo = null; }).outPath"#, "/nix/store/2n2jqm6l7r2ahi19m58pl896ipx9cyx6-foo"; "ignoreNulls false arg arg outPath")] + #[case::ignore_nulls_fales_arg_path_drvpath(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __ignoreNulls = false; foo = null; }).drvPath"#, "/nix/store/xwkwbajfiyhdqmksrbzm0s4g4ib8d4ms-foo.drv")] + #[case::ignore_nulls_fales_arg_path_outpath(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __ignoreNulls = false; foo = null; }).outPath"#, "/nix/store/2n2jqm6l7r2ahi19m58pl896ipx9cyx6-foo")] // structured attrs set to false will render an empty string inside env - #[test_case(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __structuredAttrs = false; foo = "bar"; }).drvPath"#, "/nix/store/qs39krwr2lsw6ac910vqx4pnk6m63333-foo.drv"; "structuredAttrs-false-drvPath")] - #[test_case(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __structuredAttrs = false; foo = "bar"; }).outPath"#, "/nix/store/9yy3764rdip3fbm8ckaw4j9y7vh4d231-foo"; "structuredAttrs-false-outPath")] + #[case::structured_attrs_false_drvpath(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __structuredAttrs = false; foo = "bar"; }).drvPath"#, "/nix/store/qs39krwr2lsw6ac910vqx4pnk6m63333-foo.drv")] + #[case::structured_attrs_false_outpath(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __structuredAttrs = false; foo = "bar"; }).outPath"#, "/nix/store/9yy3764rdip3fbm8ckaw4j9y7vh4d231-foo")] // simple structured attrs - #[test_case(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __structuredAttrs = true; foo = "bar"; }).drvPath"#, "/nix/store/k6rlb4k10cb9iay283037ml1nv3xma2f-foo.drv"; "structuredAttrs-simple-drvPath")] - #[test_case(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __structuredAttrs = true; foo = "bar"; }).outPath"#, "/nix/store/6lmv3hyha1g4cb426iwjyifd7nrdv1xn-foo"; "structuredAttrs-simple-outPath")] + #[case::structured_attrs_simple_drvpath(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __structuredAttrs = true; foo = "bar"; }).drvPath"#, "/nix/store/k6rlb4k10cb9iay283037ml1nv3xma2f-foo.drv")] + #[case::structured_attrs_simple_outpath(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __structuredAttrs = true; foo = "bar"; }).outPath"#, "/nix/store/6lmv3hyha1g4cb426iwjyifd7nrdv1xn-foo")] // structured attrs with outputsCheck - #[test_case(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __structuredAttrs = true; foo = "bar"; outputChecks = {out = {maxClosureSize = 256 * 1024 * 1024; disallowedRequisites = [ "dev" ];};}; }).drvPath"#, "/nix/store/fx9qzpchh5wchchhy39bwsml978d6wp1-foo.drv"; "structuredAttrs-outputChecks-drvPath")] - #[test_case(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __structuredAttrs = true; foo = "bar"; outputChecks = {out = {maxClosureSize = 256 * 1024 * 1024; disallowedRequisites = [ "dev" ];};}; }).outPath"#, "/nix/store/pcywah1nwym69rzqdvpp03sphfjgyw1l-foo"; "structuredAttrs-outputChecks-outPath")] + #[case::structured_attrs_output_checks_drvpath(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __structuredAttrs = true; foo = "bar"; outputChecks = {out = {maxClosureSize = 256 * 1024 * 1024; disallowedRequisites = [ "dev" ];};}; }).drvPath"#, "/nix/store/fx9qzpchh5wchchhy39bwsml978d6wp1-foo.drv")] + #[case::structured_attrs_output_checks_outpath(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __structuredAttrs = true; foo = "bar"; outputChecks = {out = {maxClosureSize = 256 * 1024 * 1024; disallowedRequisites = [ "dev" ];};}; }).outPath"#, "/nix/store/pcywah1nwym69rzqdvpp03sphfjgyw1l-foo")] // structured attrs and __ignoreNulls. ignoreNulls is inactive (so foo ends up in __json, yet __ignoreNulls itself is not present. - #[test_case(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __ignoreNulls = false; foo = null; __structuredAttrs = true; }).drvPath"#, "/nix/store/rldskjdcwa3p7x5bqy3r217va1jsbjsc-foo.drv"; "structuredAttrs-and-ignore-nulls-drvPath")] - - fn test_outpath(code: &str, expected_path: &str) { + #[case::structured_attrs_and_ignore_nulls_drvpath(r#"(builtins.derivation { name = "foo"; system = ":"; builder = ":"; __ignoreNulls = false; foo = null; __structuredAttrs = true; }).drvPath"#, "/nix/store/rldskjdcwa3p7x5bqy3r217va1jsbjsc-foo.drv")] + // structured attrs, setting outputs. + #[case::structured_attrs_outputs_drvpath(r#"(builtins.derivation { name = "test"; system = "aarch64-linux"; builder = "/bin/sh"; __structuredAttrs = true; outputs = [ "out"]; }).drvPath"#, "/nix/store/6sgawp30zibsh525p7c948xxd22y2ngy-test.drv")] + fn test_outpath(#[case] code: &str, #[case] expected_path: &str) { let value = eval(code).value.expect("must succeed"); match value { @@ -193,10 +195,11 @@ mod tests { } /// construct some calls to builtins.derivation that should be rejected - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha256"; outputHash = "sha256-00"; }).outPath"#; "invalid outputhash")] - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha1"; outputHash = "sha256-Q3QXOoy+iN4VK2CflvRulYvPZXYgF0dO7FoF7CvWFTA="; }).outPath"#; "sha1 and sha256")] - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; outputs = ["foo" "foo"]; system = "x86_64-linux"; }).outPath"#; "duplicate output names")] - fn test_outpath_invalid(code: &str) { + #[rstest] + #[case::invalid_outputhash(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha256"; outputHash = "sha256-00"; }).outPath"#)] + #[case::sha1_and_sha256(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha1"; outputHash = "sha256-Q3QXOoy+iN4VK2CflvRulYvPZXYgF0dO7FoF7CvWFTA="; }).outPath"#)] + #[case::duplicate_output_names(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; outputs = ["foo" "foo"]; system = "x86_64-linux"; }).outPath"#)] + fn test_outpath_invalid(#[case] code: &str) { let resp = eval(code); assert!(resp.value.is_none(), "Value should be None"); assert!( @@ -284,7 +287,8 @@ mod tests { } } - #[test_case(r#" + #[rstest] + #[case::input_in_args(r#" let bar = builtins.derivation { name = "bar"; @@ -301,8 +305,8 @@ mod tests { args = [ "${bar}" ]; system = ":"; }).drvPath - "#, "/nix/store/50yl2gmmljyl0lzyrp1mcyhn53vhjhkd-foo.drv"; "input in `args`")] - fn test_inputs_derivation_from_context(code: &str, expected_drvpath: &str) { + "#, "/nix/store/50yl2gmmljyl0lzyrp1mcyhn53vhjhkd-foo.drv")] + fn test_inputs_derivation_from_context(#[case] code: &str, #[case] expected_drvpath: &str) { let eval_result = eval(code); let value = eval_result.value.expect("must succeed"); @@ -330,8 +334,12 @@ mod tests { } /// constructs calls to builtins.derivation that should succeed, but produce warnings - #[test_case(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha256"; outputHash = "sha256-fgIr3TyFGDAXP5+qoAaiMKDg/a1MlT6Fv/S/DaA24S8===="; }).outPath"#, "/nix/store/xm1l9dx4zgycv9qdhcqqvji1z88z534b-foo"; "r:sha256 wrong padding")] - fn builtins_derivation_hash_wrong_padding_warn(code: &str, expected_path: &str) { + #[rstest] + #[case::r_sha256_wrong_padding(r#"(builtins.derivation { name = "foo"; builder = "/bin/sh"; system = "x86_64-linux"; outputHashMode = "recursive"; outputHashAlgo = "sha256"; outputHash = "sha256-fgIr3TyFGDAXP5+qoAaiMKDg/a1MlT6Fv/S/DaA24S8===="; }).outPath"#, "/nix/store/xm1l9dx4zgycv9qdhcqqvji1z88z534b-foo")] + fn builtins_derivation_hash_wrong_padding_warn( + #[case] code: &str, + #[case] expected_path: &str, + ) { let eval_result = eval(code); let value = eval_result.value.expect("must succeed"); @@ -352,35 +360,117 @@ mod tests { /// Invokes `builtins.filterSource` on various carefully-crated subdirs, and /// ensures the resulting store paths matches what Nix produces. /// @fixtures is replaced to the fixtures directory. + #[rstest] #[cfg(target_family = "unix")] - #[test_case(r#"(builtins.filterSource (p: t: true) @fixtures)"#, "/nix/store/bqh6kd0x3vps2rzagzpl7qmbbgnx19cp-import_fixtures"; "complicated directory: filter nothing")] - #[test_case(r#"(builtins.filterSource (p: t: false) @fixtures)"#, "/nix/store/giq6czz24lpjg97xxcxk6rg950lcpib1-import_fixtures"; "complicated directory: filter everything")] - #[test_case(r#"(builtins.filterSource (p: t: t != "directory") @fixtures/a_dir)"#, "/nix/store/8vbqaxapywkvv1hacdja3pi075r14d43-a_dir"; "simple directory with one file: filter directories")] - #[test_case(r#"(builtins.filterSource (p: t: t != "regular") @fixtures/a_dir)"#, "/nix/store/zphlqc93s2iq4xm393l06hzf8hp85r4z-a_dir"; "simple directory with one file: filter files")] - #[test_case(r#"(builtins.filterSource (p: t: t != "symlink") @fixtures/a_dir)"#, "/nix/store/8vbqaxapywkvv1hacdja3pi075r14d43-a_dir"; "simple directory with one file: filter symlinks")] - #[test_case(r#"(builtins.filterSource (p: t: true) @fixtures/a_dir)"#, "/nix/store/8vbqaxapywkvv1hacdja3pi075r14d43-a_dir"; "simple directory with one file: filter nothing")] - #[test_case(r#"(builtins.filterSource (p: t: false) @fixtures/a_dir)"#, "/nix/store/zphlqc93s2iq4xm393l06hzf8hp85r4z-a_dir"; "simple directory with one file: filter everything")] - #[test_case(r#"builtins.filterSource (p: t: t != "directory") @fixtures/b_dir"#, "/nix/store/xzsfzdgrxg93icaamjm8zq1jq6xvf2fz-b_dir"; "simple directory with one directory: filter directories")] - #[test_case(r#"builtins.filterSource (p: t: t != "regular") @fixtures/b_dir"#, "/nix/store/8rjx64mm7173xp60rahv7cl3ixfkv3rf-b_dir"; "simple directory with one directory: filter files")] - #[test_case(r#"builtins.filterSource (p: t: t != "symlink") @fixtures/b_dir"#, "/nix/store/8rjx64mm7173xp60rahv7cl3ixfkv3rf-b_dir"; "simple directory with one directory: filter symlinks")] - #[test_case(r#"builtins.filterSource (p: t: true) @fixtures/b_dir"#, "/nix/store/8rjx64mm7173xp60rahv7cl3ixfkv3rf-b_dir"; "simple directory with one directory: filter nothing")] - #[test_case(r#"builtins.filterSource (p: t: false) @fixtures/b_dir"#, "/nix/store/xzsfzdgrxg93icaamjm8zq1jq6xvf2fz-b_dir"; "simple directory with one directory: filter everything")] - #[test_case(r#"builtins.filterSource (p: t: t != "directory") @fixtures/c_dir"#, "/nix/store/riigfmmzzrq65zqiffcjk5sbqr9c9h09-c_dir"; "simple directory with one symlink to a file: filter directory")] - #[test_case(r#"builtins.filterSource (p: t: t != "regular") @fixtures/c_dir"#, "/nix/store/riigfmmzzrq65zqiffcjk5sbqr9c9h09-c_dir"; "simple directory with one symlink to a file: filter files")] - #[test_case(r#"builtins.filterSource (p: t: t != "symlink") @fixtures/c_dir"#, "/nix/store/y5g1fz04vzjvf422q92qmv532axj5q26-c_dir"; "simple directory with one symlink to a file: filter symlinks")] - #[test_case(r#"builtins.filterSource (p: t: true) @fixtures/c_dir"#, "/nix/store/riigfmmzzrq65zqiffcjk5sbqr9c9h09-c_dir"; "simple directory with one symlink to a file: filter nothing")] - #[test_case(r#"builtins.filterSource (p: t: false) @fixtures/c_dir"#, "/nix/store/y5g1fz04vzjvf422q92qmv532axj5q26-c_dir"; "simple directory with one symlink to a file: filter everything")] - #[test_case(r#"builtins.filterSource (p: t: t != "directory") @fixtures/d_dir"#, "/nix/store/f2d1aixwiqy4lbzrd040ala2s4m2z199-d_dir"; "simple directory with dangling symlink: filter directory")] - #[test_case(r#"builtins.filterSource (p: t: t != "regular") @fixtures/d_dir"#, "/nix/store/f2d1aixwiqy4lbzrd040ala2s4m2z199-d_dir"; "simple directory with dangling symlink: filter file")] - #[test_case(r#"builtins.filterSource (p: t: t != "symlink") @fixtures/d_dir"#, "/nix/store/7l371xax8kknhpska4wrmyll1mzlhzvl-d_dir"; "simple directory with dangling symlink: filter symlinks")] - #[test_case(r#"builtins.filterSource (p: t: true) @fixtures/d_dir"#, "/nix/store/f2d1aixwiqy4lbzrd040ala2s4m2z199-d_dir"; "simple directory with dangling symlink: filter nothing")] - #[test_case(r#"builtins.filterSource (p: t: false) @fixtures/d_dir"#, "/nix/store/7l371xax8kknhpska4wrmyll1mzlhzvl-d_dir"; "simple directory with dangling symlink: filter everything")] - #[test_case(r#"builtins.filterSource (p: t: t != "directory") @fixtures/symlink_to_a_dir"#, "/nix/store/apmdprm8fwl2zrjpbyfcd99zrnhvf47q-symlink_to_a_dir"; "simple symlinked directory with one file: filter directories")] - #[test_case(r#"builtins.filterSource (p: t: t != "regular") @fixtures/symlink_to_a_dir"#, "/nix/store/apmdprm8fwl2zrjpbyfcd99zrnhvf47q-symlink_to_a_dir"; "simple symlinked directory with one file: filter file")] - #[test_case(r#"builtins.filterSource (p: t: t != "symlink") @fixtures/symlink_to_a_dir"#, "/nix/store/apmdprm8fwl2zrjpbyfcd99zrnhvf47q-symlink_to_a_dir"; "simple symlinked directory with one file: filter symlinks")] - #[test_case(r#"builtins.filterSource (p: t: true) @fixtures/symlink_to_a_dir"#, "/nix/store/apmdprm8fwl2zrjpbyfcd99zrnhvf47q-symlink_to_a_dir"; "simple symlinked directory with one file: filter nothing")] - #[test_case(r#"builtins.filterSource (p: t: false) @fixtures/symlink_to_a_dir"#, "/nix/store/apmdprm8fwl2zrjpbyfcd99zrnhvf47q-symlink_to_a_dir"; "simple symlinked directory with one file: filter everything")] - fn builtins_filter_source_succeed(code: &str, expected_outpath: &str) { + #[case::complicated_filter_nothing( + r#"(builtins.filterSource (p: t: true) @fixtures)"#, + "/nix/store/bqh6kd0x3vps2rzagzpl7qmbbgnx19cp-import_fixtures" + )] + #[case::complicated_filter_everything( + r#"(builtins.filterSource (p: t: false) @fixtures)"#, + "/nix/store/giq6czz24lpjg97xxcxk6rg950lcpib1-import_fixtures" + )] + #[case::simple_dir_with_one_file_filter_dirs( + r#"(builtins.filterSource (p: t: t != "directory") @fixtures/a_dir)"#, + "/nix/store/8vbqaxapywkvv1hacdja3pi075r14d43-a_dir" + )] + #[case::simple_dir_with_one_file_filter_files( + r#"(builtins.filterSource (p: t: t != "regular") @fixtures/a_dir)"#, + "/nix/store/zphlqc93s2iq4xm393l06hzf8hp85r4z-a_dir" + )] + #[case::simple_dir_with_one_file_filter_symlinks( + r#"(builtins.filterSource (p: t: t != "symlink") @fixtures/a_dir)"#, + "/nix/store/8vbqaxapywkvv1hacdja3pi075r14d43-a_dir" + )] + #[case::simple_dir_with_one_file_filter_nothing( + r#"(builtins.filterSource (p: t: true) @fixtures/a_dir)"#, + "/nix/store/8vbqaxapywkvv1hacdja3pi075r14d43-a_dir" + )] + #[case::simple_dir_with_one_file_filter_everything( + r#"(builtins.filterSource (p: t: false) @fixtures/a_dir)"#, + "/nix/store/zphlqc93s2iq4xm393l06hzf8hp85r4z-a_dir" + )] + #[case::simple_dir_with_one_dir_filter_dirs( + r#"builtins.filterSource (p: t: t != "directory") @fixtures/b_dir"#, + "/nix/store/xzsfzdgrxg93icaamjm8zq1jq6xvf2fz-b_dir" + )] + #[case::simple_dir_with_one_dir_filter_files( + r#"builtins.filterSource (p: t: t != "regular") @fixtures/b_dir"#, + "/nix/store/8rjx64mm7173xp60rahv7cl3ixfkv3rf-b_dir" + )] + #[case::simple_dir_with_one_dir_filter_symlinks( + r#"builtins.filterSource (p: t: t != "symlink") @fixtures/b_dir"#, + "/nix/store/8rjx64mm7173xp60rahv7cl3ixfkv3rf-b_dir" + )] + #[case::simple_dir_with_one_dir_filter_nothing( + r#"builtins.filterSource (p: t: true) @fixtures/b_dir"#, + "/nix/store/8rjx64mm7173xp60rahv7cl3ixfkv3rf-b_dir" + )] + #[case::simple_dir_with_one_dir_filter_everything( + r#"builtins.filterSource (p: t: false) @fixtures/b_dir"#, + "/nix/store/xzsfzdgrxg93icaamjm8zq1jq6xvf2fz-b_dir" + )] + #[case::simple_dir_with_one_symlink_to_file_filter_dirs( + r#"builtins.filterSource (p: t: t != "directory") @fixtures/c_dir"#, + "/nix/store/riigfmmzzrq65zqiffcjk5sbqr9c9h09-c_dir" + )] + #[case::simple_dir_with_one_symlink_to_file_filter_files( + r#"builtins.filterSource (p: t: t != "regular") @fixtures/c_dir"#, + "/nix/store/riigfmmzzrq65zqiffcjk5sbqr9c9h09-c_dir" + )] + #[case::simple_dir_with_one_symlink_to_file_filter_symlinks( + r#"builtins.filterSource (p: t: t != "symlink") @fixtures/c_dir"#, + "/nix/store/y5g1fz04vzjvf422q92qmv532axj5q26-c_dir" + )] + #[case::simple_dir_with_one_symlink_to_file_filter_nothing( + r#"builtins.filterSource (p: t: true) @fixtures/c_dir"#, + "/nix/store/riigfmmzzrq65zqiffcjk5sbqr9c9h09-c_dir" + )] + #[case::simple_dir_with_one_symlink_to_file_filter_everything( + r#"builtins.filterSource (p: t: false) @fixtures/c_dir"#, + "/nix/store/y5g1fz04vzjvf422q92qmv532axj5q26-c_dir" + )] + #[case::simple_dir_with_dangling_symlink_filter_dirs( + r#"builtins.filterSource (p: t: t != "directory") @fixtures/d_dir"#, + "/nix/store/f2d1aixwiqy4lbzrd040ala2s4m2z199-d_dir" + )] + #[case::simple_dir_with_dangling_symlink_filter_files( + r#"builtins.filterSource (p: t: t != "regular") @fixtures/d_dir"#, + "/nix/store/f2d1aixwiqy4lbzrd040ala2s4m2z199-d_dir" + )] + #[case::simple_dir_with_dangling_symlink_filter_symlinks( + r#"builtins.filterSource (p: t: t != "symlink") @fixtures/d_dir"#, + "/nix/store/7l371xax8kknhpska4wrmyll1mzlhzvl-d_dir" + )] + #[case::simple_dir_with_dangling_symlink_filter_nothing( + r#"builtins.filterSource (p: t: true) @fixtures/d_dir"#, + "/nix/store/f2d1aixwiqy4lbzrd040ala2s4m2z199-d_dir" + )] + #[case::simple_dir_with_dangling_symlink_filter_everything( + r#"builtins.filterSource (p: t: false) @fixtures/d_dir"#, + "/nix/store/7l371xax8kknhpska4wrmyll1mzlhzvl-d_dir" + )] + #[case::simple_symlinked_dir_with_one_file_filter_dirs( + r#"builtins.filterSource (p: t: t != "directory") @fixtures/symlink_to_a_dir"#, + "/nix/store/apmdprm8fwl2zrjpbyfcd99zrnhvf47q-symlink_to_a_dir" + )] + #[case::simple_symlinked_dir_with_one_file_filter_files( + r#"builtins.filterSource (p: t: t != "regular") @fixtures/symlink_to_a_dir"#, + "/nix/store/apmdprm8fwl2zrjpbyfcd99zrnhvf47q-symlink_to_a_dir" + )] + #[case::simple_symlinked_dir_with_one_file_filter_symlinks( + r#"builtins.filterSource (p: t: t != "symlink") @fixtures/symlink_to_a_dir"#, + "/nix/store/apmdprm8fwl2zrjpbyfcd99zrnhvf47q-symlink_to_a_dir" + )] + #[case::simple_symlinked_dir_with_one_file_filter_nothing( + r#"builtins.filterSource (p: t: true) @fixtures/symlink_to_a_dir"#, + "/nix/store/apmdprm8fwl2zrjpbyfcd99zrnhvf47q-symlink_to_a_dir" + )] + #[case::simple_symlinked_dir_with_one_file_filter_everything( + r#"builtins.filterSource (p: t: false) @fixtures/symlink_to_a_dir"#, + "/nix/store/apmdprm8fwl2zrjpbyfcd99zrnhvf47q-symlink_to_a_dir" + )] + fn builtins_filter_source_succeed(#[case] code: &str, #[case] expected_outpath: &str) { // populate the fixtures dir let temp = TempDir::new().expect("create temporary directory"); let p = temp.path().join("import_fixtures"); @@ -438,20 +528,218 @@ mod tests { assert!(eval_result.errors.is_empty(), "errors should be empty"); } + // Space is an illegal character. + #[rstest] + #[case( + r#"(builtins.path { name = "valid-name"; path = @fixtures + "/te st"; recursive = true; })"#, + true + )] + // Space is still an illegal character. + #[case( + r#"(builtins.path { name = "invalid name"; path = @fixtures + "/te st"; recursive = true; })"#, + false + )] + fn builtins_path_recursive_rename(#[case] code: &str, #[case] success: bool) { + // populate the fixtures dir + let temp = TempDir::new().expect("create temporary directory"); + let p = temp.path().join("import_fixtures"); + + // create the fixtures directory. + // We produce them at runtime rather than shipping it inside the source + // tree, as git can't model certain things - like directories without any + // items. + { + fs::create_dir(&p).expect("creating import_fixtures"); + fs::write(p.join("te st"), "").expect("creating `/te st`"); + } + // replace @fixtures with the temporary path containing the fixtures + let code_replaced = code.replace("@fixtures", &p.to_string_lossy()); + + let eval_result = eval(&code_replaced); + + let value = eval_result.value; + + if success { + match value.expect("expected successful evaluation on legal rename") { + tvix_eval::Value::String(s) => { + assert_eq!( + "/nix/store/nd5z11x7zjqqz44rkbhc6v7yifdkn659-valid-name", + s.as_bstr() + ); + } + v => panic!("unexpected value type: {:?}", v), + } + } else { + assert!(value.is_none(), "unexpected success on illegal store paths"); + } + } + + // Space is an illegal character. + #[rstest] + #[case( + r#"(builtins.path { name = "valid-name"; path = @fixtures + "/te st"; recursive = false; })"#, + true + )] + // Space is still an illegal character. + #[case( + r#"(builtins.path { name = "invalid name"; path = @fixtures + "/te st"; recursive = false; })"#, + false + )] + // The non-recursive variant passes explicitly `recursive = false;` + fn builtins_path_nonrecursive_rename(#[case] code: &str, #[case] success: bool) { + // populate the fixtures dir + let temp = TempDir::new().expect("create temporary directory"); + let p = temp.path().join("import_fixtures"); + + // create the fixtures directory. + // We produce them at runtime rather than shipping it inside the source + // tree, as git can't model certain things - like directories without any + // items. + { + fs::create_dir(&p).expect("creating import_fixtures"); + fs::write(p.join("te st"), "").expect("creating `/te st`"); + } + // replace @fixtures with the temporary path containing the fixtures + let code_replaced = code.replace("@fixtures", &p.to_string_lossy()); + + let eval_result = eval(&code_replaced); + + let value = eval_result.value; + + if success { + match value.expect("expected successful evaluation on legal rename") { + tvix_eval::Value::String(s) => { + assert_eq!( + "/nix/store/il2rmfbqgs37rshr8w7x64hd4d3b4bsa-valid-name", + s.as_bstr() + ); + } + v => panic!("unexpected value type: {:?}", v), + } + } else { + assert!(value.is_none(), "unexpected success on illegal store paths"); + } + } + + #[rstest] + #[case( + r#"(builtins.path { name = "valid-name"; path = @fixtures + "/te st"; recursive = false; sha256 = "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="; })"#, + true + )] + #[case( + r#"(builtins.path { name = "valid-name"; path = @fixtures + "/te st"; recursive = true; sha256 = "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="; })"#, + false + )] + #[case( + r#"(builtins.path { name = "valid-name"; path = @fixtures + "/te st"; recursive = true; sha256 = "sha256-d6xi4mKdjkX2JFicDIv5niSzpyI0m/Hnm8GGAIU04kY="; })"#, + true + )] + #[case( + r#"(builtins.path { name = "valid-name"; path = @fixtures + "/te st"; recursive = false; sha256 = "sha256-d6xi4mKdjkX2JFicDIv5niSzpyI0m/Hnm8GGAIU04kY="; })"#, + false + )] + fn builtins_path_fod_locking(#[case] code: &str, #[case] exp_success: bool) { + // populate the fixtures dir + let temp = TempDir::new().expect("create temporary directory"); + let p = temp.path().join("import_fixtures"); + + // create the fixtures directory. + // We produce them at runtime rather than shipping it inside the source + // tree, as git can't model certain things - like directories without any + // items. + { + fs::create_dir(&p).expect("creating import_fixtures"); + fs::write(p.join("te st"), "").expect("creating `/te st`"); + } + // replace @fixtures with the temporary path containing the fixtures + let code_replaced = code.replace("@fixtures", &p.to_string_lossy()); + + let eval_result = eval(&code_replaced); + + let value = eval_result.value; + + if exp_success { + assert!( + value.is_some(), + "expected successful evaluation on legal rename and valid FOD sha256" + ); + } else { + assert!(value.is_none(), "unexpected success on invalid FOD sha256"); + } + } + + #[rstest] + #[case( + r#"(builtins.path { name = "valid-path"; path = @fixtures + "/te st dir"; filter = _: _: true; })"#, + "/nix/store/i28jmi4fwym4fw3flkrkp2mdxx50pdy0-valid-path" + )] + #[case( + r#"(builtins.path { name = "valid-path"; path = @fixtures + "/te st dir"; filter = _: _: false; })"#, + "/nix/store/pwza2ij9gk1fmzhbjnynmfv2mq2sgcap-valid-path" + )] + fn builtins_path_filter(#[case] code: &str, #[case] expected_outpath: &str) { + // populate the fixtures dir + let temp = TempDir::new().expect("create temporary directory"); + let p = temp.path().join("import_fixtures"); + + // create the fixtures directory. + // We produce them at runtime rather than shipping it inside the source + // tree, as git can't model certain things - like directories without any + // items. + { + fs::create_dir(&p).expect("creating import_fixtures"); + fs::create_dir(p.join("te st dir")).expect("creating `/te st dir`"); + fs::write(p.join("te st dir").join("test"), "").expect("creating `/te st dir/test`"); + } + // replace @fixtures with the temporary path containing the fixtures + let code_replaced = code.replace("@fixtures", &p.to_string_lossy()); + + let eval_result = eval(&code_replaced); + + let value = eval_result.value.expect("must succeed"); + + match value { + tvix_eval::Value::String(s) => { + assert_eq!(expected_outpath, s.as_bstr()); + } + _ => panic!("unexpected value type: {:?}", value), + } + + assert!(eval_result.errors.is_empty(), "errors should be empty"); + } + // All tests filter out some unsupported (not representable in castore) nodes, confirming // invalid, but filtered-out nodes don't prevent ingestion of a path. + #[rstest] #[cfg(target_family = "unix")] // There is a set of invalid filetypes. - // We write a filter function for most subsets, excluding one that filters all of them. - // We expect these cases to make the evaluation fail as there are still invalid files present - // after the filtering. - #[test_case(r#"(builtins.filterSource (p: t: t == "unknown") @fixtures)"#, false; "complicated directory: filter unsupported types")] - #[test_case(r#"(builtins.filterSource (p: t: (builtins.baseNameOf p) != "a_charnode") @fixtures)"#, false; "complicated directory: filter character device nodes")] - #[test_case(r#"(builtins.filterSource (p: t: (builtins.baseNameOf p) != "a_socket") @fixtures)"#, false; "complicated directory: filter sockets")] - #[test_case(r#"(builtins.filterSource (p: t: (builtins.baseNameOf p) != "a_fifo") @fixtures)"#, false; "complicated directory: filter FIFOs")] + // We write various filter functions filtering them out, but usually leaving + // some behind. + // In case there's still invalid filetypes left after the filtering, we + // expect the evaluation to fail. + #[case::fail_kept_unknowns( + r#"(builtins.filterSource (p: t: t == "unknown") @fixtures)"#, + false + )] // We filter all invalid filetypes, so the evaluation has to succeed. - #[test_case(r#"(builtins.filterSource (p: t: t != "unknown") @fixtures)"#, true; "complicated directory: filter out unsupported types")] - fn builtins_filter_source_unsupported_files(code: &str, success: bool) { + #[case::succeed_filter_unknowns( + r#"(builtins.filterSource (p: t: t != "unknown") @fixtures)"#, + true + )] + #[case::fail_kept_charnode( + r#"(builtins.filterSource (p: t: (builtins.baseNameOf p) != "a_charnode") @fixtures)"#, + false + )] + #[case::fail_kept_socket( + r#"(builtins.filterSource (p: t: (builtins.baseNameOf p) != "a_socket") @fixtures)"#, + false + )] + #[case::fail_kept_fifo( + r#"(builtins.filterSource (p: t: (builtins.baseNameOf p) != "a_fifo") @fixtures)"#, + false + )] + fn builtins_filter_source_unsupported_files(#[case] code: &str, #[case] exp_success: bool) { + use nix::errno::Errno; use nix::sys::stat; use nix::unistd; use std::os::unix::net::UnixListener; @@ -478,12 +766,21 @@ mod tests { stat::Mode::S_IRWXU, 0, ) + .inspect_err(|e| { + if *e == Errno::EPERM { + eprintln!( + "\ +Missing permissions to create a character device node with mknod(2). +Please run this test as root or set CAP_MKNOD." + ); + } + }) .expect("Failed to create a character device node"); let code_replaced = code.replace("@fixtures", &temp.path().to_string_lossy()); let eval_result = eval(&code_replaced); - if success { + if exp_success { assert!( eval_result.value.is_some(), "unexpected failure on a directory of unsupported file types but all filtered: {:?}", diff --git a/tvix/glue/src/decompression.rs b/tvix/glue/src/fetchers/decompression.rs index 7e526932e7..f96fa60e34 100644 --- a/tvix/glue/src/decompression.rs +++ b/tvix/glue/src/fetchers/decompression.rs @@ -183,7 +183,7 @@ mod tests { use async_compression::tokio::bufread::GzipEncoder; use futures::TryStreamExt; - use test_case::test_case; + use rstest::rstest; use tokio::io::{AsyncReadExt, BufReader}; use tokio_tar::Archive; @@ -203,11 +203,12 @@ mod tests { assert_eq!(data[..], round_tripped[..]); } - #[test_case(include_bytes!("tests/blob.tar.gz"); "gzip")] - #[test_case(include_bytes!("tests/blob.tar.bz2"); "bzip2")] - #[test_case(include_bytes!("tests/blob.tar.xz"); "xz")] + #[rstest] + #[case::gzip(include_bytes!("../tests/blob.tar.gz"))] + #[case::bzip2(include_bytes!("../tests/blob.tar.bz2"))] + #[case::xz(include_bytes!("../tests/blob.tar.xz"))] #[tokio::test] - async fn compressed_tar(data: &[u8]) { + async fn compressed_tar(#[case] data: &[u8]) { let reader = DecompressedReader::new(BufReader::new(data)); let mut archive = Archive::new(reader); let mut entries: Vec<_> = archive.entries().unwrap().try_collect().await.unwrap(); diff --git a/tvix/glue/src/fetchers/mod.rs b/tvix/glue/src/fetchers/mod.rs new file mode 100644 index 0000000000..342dfd84e8 --- /dev/null +++ b/tvix/glue/src/fetchers/mod.rs @@ -0,0 +1,445 @@ +use futures::TryStreamExt; +use md5::Md5; +use nix_compat::{ + nixhash::{CAHash, HashAlgo, NixHash}, + store_path::{build_ca_path, BuildStorePathError, StorePathRef}, +}; +use sha1::Sha1; +use sha2::{digest::Output, Digest, Sha256, Sha512}; +use tokio::io::{AsyncBufRead, AsyncRead, AsyncWrite}; +use tokio_util::io::InspectReader; +use tracing::warn; +use tvix_castore::{ + blobservice::BlobService, + directoryservice::DirectoryService, + proto::{node::Node, FileNode}, +}; +use tvix_store::{pathinfoservice::PathInfoService, proto::PathInfo}; +use url::Url; + +use crate::builtins::FetcherError; + +mod decompression; +use decompression::DecompressedReader; + +/// Representing options for doing a fetch. +#[derive(Clone, Eq, PartialEq)] +pub enum Fetch { + /// Fetch a literal file from the given URL, with an optional expected + /// NixHash of it. + /// TODO: check if this is *always* sha256, and if so, make it [u8; 32]. + URL(Url, Option<NixHash>), + + /// Fetch a tarball from the given URL and unpack. + /// The file must be a tape archive (.tar), optionally compressed with gzip, + /// bzip2 or xz. + /// The top-level path component of the files in the tarball is removed, + /// so it is best if the tarball contains a single directory at top level. + /// Optionally, a sha256 digest can be provided to verify the unpacked + /// contents against. + Tarball(Url, Option<[u8; 32]>), + + /// TODO + Git(), +} + +// Drops potentially sensitive username and password from a URL. +fn redact_url(url: &Url) -> Url { + let mut url = url.to_owned(); + if !url.username().is_empty() { + let _ = url.set_username("redacted"); + } + + if url.password().is_some() { + let _ = url.set_password(Some("redacted")); + } + + url +} + +impl std::fmt::Debug for Fetch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Fetch::URL(url, exp_hash) => { + let url = redact_url(url); + if let Some(exp_hash) = exp_hash { + write!(f, "URL [url: {}, exp_hash: Some({})]", &url, exp_hash) + } else { + write!(f, "URL [url: {}, exp_hash: None]", &url) + } + } + Fetch::Tarball(url, exp_digest) => { + let url = redact_url(url); + if let Some(exp_digest) = exp_digest { + write!( + f, + "Tarball [url: {}, exp_hash: Some({})]", + url, + NixHash::Sha256(*exp_digest) + ) + } else { + write!(f, "Tarball [url: {}, exp_hash: None]", url) + } + } + Fetch::Git() => todo!(), + } + } +} + +impl Fetch { + /// If the [Fetch] contains an expected hash upfront, returns the resulting + /// store path. + /// This doesn't do any fetching. + pub fn store_path<'a>( + &self, + name: &'a str, + ) -> Result<Option<StorePathRef<'a>>, BuildStorePathError> { + let ca_hash = match self { + Fetch::URL(_, Some(nixhash)) => CAHash::Flat(nixhash.clone()), + Fetch::Tarball(_, Some(nar_sha256)) => CAHash::Nar(NixHash::Sha256(*nar_sha256)), + _ => return Ok(None), + }; + + // calculate the store path of this fetch + build_ca_path(name, &ca_hash, Vec::<String>::new(), false).map(Some) + } +} + +/// Knows how to fetch a given [Fetch]. +pub struct Fetcher<BS, DS, PS> { + http_client: reqwest::Client, + blob_service: BS, + directory_service: DS, + path_info_service: PS, +} + +impl<BS, DS, PS> Fetcher<BS, DS, PS> { + pub fn new(blob_service: BS, directory_service: DS, path_info_service: PS) -> Self { + Self { + http_client: reqwest::Client::new(), + blob_service, + directory_service, + path_info_service, + } + } + + /// Constructs a HTTP request to the passed URL, and returns a AsyncReadBuf to it. + /// In case the URI uses the file:// scheme, use tokio::fs to open it. + async fn download(&self, url: Url) -> Result<Box<dyn AsyncBufRead + Unpin>, FetcherError> { + match url.scheme() { + "file" => { + let f = tokio::fs::File::open(url.to_file_path().map_err(|_| { + // "Returns Err if the host is neither empty nor "localhost" + // (except on Windows, where file: URLs may have a non-local host)" + FetcherError::Io(std::io::Error::new( + std::io::ErrorKind::Other, + "invalid host for file:// scheme", + )) + })?) + .await?; + Ok(Box::new(tokio::io::BufReader::new(f))) + } + _ => { + let resp = self.http_client.get(url).send().await?; + Ok(Box::new(tokio_util::io::StreamReader::new( + resp.bytes_stream().map_err(|e| { + let e = e.without_url(); + warn!(%e, "failed to get response body"); + std::io::Error::new(std::io::ErrorKind::BrokenPipe, e) + }), + ))) + } + } + } +} + +/// Copies all data from the passed reader to the passed writer. +/// Afterwards, it also returns the resulting [Digest], as well as the number of +/// bytes copied. +/// The exact hash function used is left generic over all [Digest]. +async fn hash<D: Digest + std::io::Write>( + mut r: impl AsyncRead + Unpin, + mut w: impl AsyncWrite + Unpin, +) -> std::io::Result<(Output<D>, u64)> { + let mut hasher = D::new(); + let bytes_copied = tokio::io::copy( + &mut InspectReader::new(&mut r, |d| hasher.write_all(d).unwrap()), + &mut w, + ) + .await?; + Ok((hasher.finalize(), bytes_copied)) +} + +impl<BS, DS, PS> Fetcher<BS, DS, PS> +where + BS: BlobService + Clone + 'static, + DS: DirectoryService + Clone, + PS: PathInfoService, +{ + /// Ingest the data from a specified [Fetch]. + /// On success, return the root node, a content digest and length. + /// Returns an error if there was a failure during fetching, or the contents + /// didn't match the previously communicated hash contained inside the FetchArgs. + pub async fn ingest(&self, fetch: Fetch) -> Result<(Node, CAHash, u64), FetcherError> { + match fetch { + Fetch::URL(url, exp_hash) => { + // Construct a AsyncRead reading from the data as its downloaded. + let mut r = self.download(url.clone()).await?; + + // Construct a AsyncWrite to write into the BlobService. + let mut blob_writer = self.blob_service.open_write().await; + + // Copy the contents from the download reader to the blob writer. + // Calculate the digest of the file received, depending on the + // communicated expected hash (or sha256 if none provided). + let (actual_hash, blob_size) = match exp_hash + .as_ref() + .map(NixHash::algo) + .unwrap_or_else(|| HashAlgo::Sha256) + { + HashAlgo::Sha256 => hash::<Sha256>(&mut r, &mut blob_writer).await.map( + |(digest, bytes_written)| (NixHash::Sha256(digest.into()), bytes_written), + )?, + HashAlgo::Md5 => hash::<Md5>(&mut r, &mut blob_writer).await.map( + |(digest, bytes_written)| (NixHash::Md5(digest.into()), bytes_written), + )?, + HashAlgo::Sha1 => hash::<Sha1>(&mut r, &mut blob_writer).await.map( + |(digest, bytes_written)| (NixHash::Sha1(digest.into()), bytes_written), + )?, + HashAlgo::Sha512 => hash::<Sha512>(&mut r, &mut blob_writer).await.map( + |(digest, bytes_written)| { + (NixHash::Sha512(Box::new(digest.into())), bytes_written) + }, + )?, + }; + + if let Some(exp_hash) = exp_hash { + if exp_hash != actual_hash { + return Err(FetcherError::HashMismatch { + url, + wanted: exp_hash, + got: actual_hash, + }); + } + } + + // Construct and return the FileNode describing the downloaded contents. + Ok(( + Node::File(FileNode { + name: vec![].into(), + digest: blob_writer.close().await?.into(), + size: blob_size, + executable: false, + }), + CAHash::Flat(actual_hash), + blob_size, + )) + } + Fetch::Tarball(url, exp_nar_sha256) => { + // Construct a AsyncRead reading from the data as its downloaded. + let r = self.download(url.clone()).await?; + + // Pop compression. + let r = DecompressedReader::new(r); + // Open the archive. + let archive = tokio_tar::Archive::new(r); + + // Ingest the archive, get the root node + let node = tvix_castore::import::archive::ingest_archive( + self.blob_service.clone(), + self.directory_service.clone(), + archive, + ) + .await?; + + // If an expected NAR sha256 was provided, compare with the one + // calculated from our root node. + // Even if no expected NAR sha256 has been provided, we need + // the actual one later. + let (nar_size, actual_nar_sha256) = self + .path_info_service + .calculate_nar(&node) + .await + .map_err(|e| { + // convert the generic Store error to an IO error. + FetcherError::Io(e.into()) + })?; + + if let Some(exp_nar_sha256) = exp_nar_sha256 { + if exp_nar_sha256 != actual_nar_sha256 { + return Err(FetcherError::HashMismatch { + url, + wanted: NixHash::Sha256(exp_nar_sha256), + got: NixHash::Sha256(actual_nar_sha256), + }); + } + } + + Ok(( + node, + CAHash::Nar(NixHash::Sha256(actual_nar_sha256)), + nar_size, + )) + } + Fetch::Git() => todo!(), + } + } + + /// Ingests the data from a specified [Fetch], persists the returned node + /// in the PathInfoService, and returns the calculated StorePath, as well as + /// the root node pointing to the contents. + /// The root node can be used to descend into the data without doing the + /// lookup to the PathInfoService again. + pub async fn ingest_and_persist<'a>( + &self, + name: &'a str, + fetch: Fetch, + ) -> Result<(StorePathRef<'a>, Node), FetcherError> { + // Fetch file, return the (unnamed) (File)Node of its contents, ca hash and filesize. + let (node, ca_hash, size) = self.ingest(fetch).await?; + + // Calculate the store path to return later, which is done with the ca_hash. + let store_path = build_ca_path(name, &ca_hash, Vec::<String>::new(), false)?; + + // Rename the node name to match the Store Path. + let node = node.rename(store_path.to_string().into()); + + // If the resulting hash is not a CAHash::Nar, we also need to invoke + // `calculate_nar` to calculate this representation, as it's required in + // the [PathInfo]. + let (nar_size, nar_sha256) = match &ca_hash { + CAHash::Flat(_nix_hash) => self + .path_info_service + .calculate_nar(&node) + .await + .map_err(|e| FetcherError::Io(e.into()))?, + CAHash::Nar(NixHash::Sha256(nar_sha256)) => (size, *nar_sha256), + CAHash::Nar(_) => unreachable!("Tvix bug: fetch returned non-sha256 CAHash::Nar"), + CAHash::Text(_) => unreachable!("Tvix bug: fetch returned CAHash::Text"), + }; + + // Construct the PathInfo and persist it. + let path_info = PathInfo { + node: Some(tvix_castore::proto::Node { node: Some(node) }), + references: vec![], + narinfo: Some(tvix_store::proto::NarInfo { + nar_size, + nar_sha256: nar_sha256.to_vec().into(), + signatures: vec![], + reference_names: vec![], + deriver: None, + ca: Some(ca_hash.into()), + }), + }; + + let path_info = self + .path_info_service + .put(path_info) + .await + .map_err(|e| FetcherError::Io(e.into()))?; + + Ok((store_path, path_info.node.unwrap().node.unwrap())) + } +} + +/// Attempts to mimic `nix::libutil::baseNameOf` +pub(crate) fn url_basename(s: &str) -> &str { + if s.is_empty() { + return ""; + } + + let mut last = s.len() - 1; + if s.chars().nth(last).unwrap() == '/' && last > 0 { + last -= 1; + } + + if last == 0 { + return ""; + } + + let pos = match s[..=last].rfind('/') { + Some(pos) => { + if pos == last - 1 { + 0 + } else { + pos + } + } + None => 0, + }; + + &s[(pos + 1)..=last] +} + +#[cfg(test)] +mod tests { + mod fetch { + use nix_compat::nixbase32; + + use crate::fetchers::Fetch; + + use super::super::*; + + #[test] + fn fetchurl_store_path() { + let url = Url::parse("https://raw.githubusercontent.com/aaptel/notmuch-extract-patch/f732a53e12a7c91a06755ebfab2007adc9b3063b/notmuch-extract-patch").unwrap(); + let exp_hash = NixHash::Sha256( + nixbase32::decode_fixed("0nawkl04sj7psw6ikzay7kydj3dhd0fkwghcsf5rzaw4bmp4kbax") + .unwrap(), + ); + + let fetch = Fetch::URL(url, Some(exp_hash)); + assert_eq!( + "06qi00hylriyfm0nl827crgjvbax84mz-notmuch-extract-patch", + &fetch + .store_path("notmuch-extract-patch") + .unwrap() + .unwrap() + .to_string(), + ) + } + + #[test] + fn fetch_tarball_store_path() { + let url = Url::parse("https://github.com/NixOS/nixpkgs/archive/91050ea1e57e50388fa87a3302ba12d188ef723a.tar.gz").unwrap(); + let exp_nixbase32 = + nixbase32::decode_fixed("1hf6cgaci1n186kkkjq106ryf8mmlq9vnwgfwh625wa8hfgdn4dm") + .unwrap(); + let fetch = Fetch::Tarball(url, Some(exp_nixbase32)); + + assert_eq!( + "7adgvk5zdfq4pwrhsm3n9lzypb12gw0g-source", + &fetch.store_path("source").unwrap().unwrap().to_string(), + ) + } + } + + mod url_basename { + use super::super::*; + + #[test] + fn empty_path() { + assert_eq!(url_basename(""), ""); + } + + #[test] + fn path_on_root() { + assert_eq!(url_basename("/dir"), "dir"); + } + + #[test] + fn relative_path() { + assert_eq!(url_basename("dir/foo"), "foo"); + } + + #[test] + fn root_with_trailing_slash() { + assert_eq!(url_basename("/"), ""); + } + + #[test] + fn trailing_slash() { + assert_eq!(url_basename("/dir/"), "dir"); + } + } +} diff --git a/tvix/glue/src/known_paths.rs b/tvix/glue/src/known_paths.rs index 9cd9470fa9..290c9d5b69 100644 --- a/tvix/glue/src/known_paths.rs +++ b/tvix/glue/src/known_paths.rs @@ -8,9 +8,14 @@ //! This data is required to find the derivation needed to actually trigger the //! build, if necessary. -use nix_compat::{derivation::Derivation, store_path::StorePath}; +use nix_compat::{ + derivation::Derivation, + store_path::{BuildStorePathError, StorePath, StorePathRef}, +}; use std::collections::HashMap; +use crate::fetchers::Fetch; + /// Struct keeping track of all known Derivations in the current evaluation. /// This keeps both the Derivation struct, as well as the "Hash derivation /// modulo". @@ -26,6 +31,9 @@ pub struct KnownPaths { /// Note that in the case of FODs, multiple drvs can produce the same output /// path. We use one of them. outputs_to_drvpath: HashMap<StorePath, StorePath>, + + /// A map from output path to fetches (and their names). + outputs_to_fetches: HashMap<StorePath, (String, Fetch)>, } impl KnownPaths { @@ -50,12 +58,12 @@ impl KnownPaths { self.outputs_to_drvpath.get(output_path) } - /// Insert a new Derivation into this struct. + /// Insert a new [Derivation] into this struct. /// The Derivation struct must pass validation, and its output paths need to /// be fully calculated. /// All input derivations this refers to must also be inserted to this /// struct. - pub fn add(&mut self, drv_path: StorePath, drv: Derivation) { + pub fn add_derivation(&mut self, drv_path: StorePath, drv: Derivation) { // check input derivations to have been inserted. #[cfg(debug_assertions)] { @@ -65,7 +73,7 @@ impl KnownPaths { } // compute the hash derivation modulo - let hash_derivation_modulo = drv.derivation_or_fod_hash(|drv_path| { + let hash_derivation_modulo = drv.hash_derivation_modulo(|drv_path| { self.get_hash_derivation_modulo(&drv_path.to_owned()) .unwrap_or_else(|| panic!("{} not found", drv_path)) .to_owned() @@ -95,11 +103,40 @@ impl KnownPaths { } } } + + /// Insert a new [Fetch] into this struct, which *must* have an expected + /// hash (otherwise we wouldn't be able to calculate the store path). + /// Fetches without a known hash need to be fetched inside builtins. + pub fn add_fetch<'a>( + &mut self, + fetch: Fetch, + name: &'a str, + ) -> Result<StorePathRef<'a>, BuildStorePathError> { + let store_path = fetch + .store_path(name)? + .expect("Tvix bug: fetch must have an expected hash"); + // insert the fetch. + self.outputs_to_fetches + .insert(store_path.to_owned(), (name.to_owned(), fetch)); + + Ok(store_path) + } + + /// Return the name and fetch producing the passed output path. + /// Note there can also be (multiple) Derivations producing the same output path. + pub fn get_fetch_for_output_path(&self, output_path: &StorePath) -> Option<(String, Fetch)> { + self.outputs_to_fetches + .get(output_path) + .map(|(name, fetch)| (name.to_owned(), fetch.to_owned())) + } } #[cfg(test)] mod tests { - use nix_compat::{derivation::Derivation, store_path::StorePath}; + use nix_compat::{derivation::Derivation, nixbase32, nixhash::NixHash, store_path::StorePath}; + use url::Url; + + use crate::fetchers::Fetch; use super::KnownPaths; use hex_literal::hex; @@ -122,21 +159,33 @@ mod tests { StorePath::from_bytes(b"mp57d33657rf34lzvlbpfa1gjfv5gmpg-bar").expect("must parse"); static ref FOO_OUT_PATH: StorePath = StorePath::from_bytes(b"fhaj6gmwns62s6ypkcldbaj2ybvkhx3p-foo").expect("must parse"); + + static ref FETCH_URL : Fetch = Fetch::URL( + Url::parse("https://raw.githubusercontent.com/aaptel/notmuch-extract-patch/f732a53e12a7c91a06755ebfab2007adc9b3063b/notmuch-extract-patch").unwrap(), + Some(NixHash::Sha256(nixbase32::decode_fixed("0nawkl04sj7psw6ikzay7kydj3dhd0fkwghcsf5rzaw4bmp4kbax").unwrap())) + ); + static ref FETCH_URL_OUT_PATH: StorePath = StorePath::from_bytes(b"06qi00hylriyfm0nl827crgjvbax84mz-notmuch-extract-patch").unwrap(); + + static ref FETCH_TARBALL : Fetch = Fetch::Tarball( + Url::parse("https://github.com/NixOS/nixpkgs/archive/91050ea1e57e50388fa87a3302ba12d188ef723a.tar.gz").unwrap(), + Some(nixbase32::decode_fixed("1hf6cgaci1n186kkkjq106ryf8mmlq9vnwgfwh625wa8hfgdn4dm").unwrap()) + ); + static ref FETCH_TARBALL_OUT_PATH: StorePath = StorePath::from_bytes(b"7adgvk5zdfq4pwrhsm3n9lzypb12gw0g-source").unwrap(); } /// ensure we don't allow acdding a Derivation that depends on another, /// not-yet-added Derivation. #[test] #[should_panic] - fn reject_if_missing_input_drv() { + fn drv_reject_if_missing_input_drv() { let mut known_paths = KnownPaths::default(); // FOO_DRV depends on BAR_DRV, which wasn't added. - known_paths.add(FOO_DRV_PATH.clone(), FOO_DRV.clone()); + known_paths.add_derivation(FOO_DRV_PATH.clone(), FOO_DRV.clone()); } #[test] - fn happy_path() { + fn drv_happy_path() { let mut known_paths = KnownPaths::default(); // get_drv_by_drvpath should return None for non-existing Derivations, @@ -149,7 +198,7 @@ mod tests { ); // Add BAR_DRV - known_paths.add(BAR_DRV_PATH.clone(), BAR_DRV.clone()); + known_paths.add_derivation(BAR_DRV_PATH.clone(), BAR_DRV.clone()); // We should get it back assert_eq!( @@ -173,7 +222,7 @@ mod tests { // Now insert FOO_DRV too. It shouldn't panic, as BAR_DRV is already // added. - known_paths.add(FOO_DRV_PATH.clone(), FOO_DRV.clone()); + known_paths.add_derivation(FOO_DRV_PATH.clone(), FOO_DRV.clone()); assert_eq!( Some(&FOO_DRV.clone()), @@ -192,4 +241,49 @@ mod tests { known_paths.get_drv_path_for_output_path(&FOO_OUT_PATH) ); } + + #[test] + fn fetch_happy_path() { + let mut known_paths = KnownPaths::default(); + + // get_fetch_for_output_path should return None for new fetches. + assert!(known_paths + .get_fetch_for_output_path(&FETCH_TARBALL_OUT_PATH) + .is_none()); + + // add_fetch should return the properly calculated store paths. + assert_eq!( + *FETCH_TARBALL_OUT_PATH, + known_paths + .add_fetch(FETCH_TARBALL.clone(), "source") + .unwrap() + .to_owned() + ); + + assert_eq!( + *FETCH_URL_OUT_PATH, + known_paths + .add_fetch(FETCH_URL.clone(), "notmuch-extract-patch") + .unwrap() + .to_owned() + ); + + // We should be able to get these fetches out, when asking for their out path. + let (got_name, got_fetch) = known_paths + .get_fetch_for_output_path(&FETCH_URL_OUT_PATH) + .expect("must be some"); + + assert_eq!("notmuch-extract-patch", got_name); + assert_eq!(FETCH_URL.clone(), got_fetch); + + // โฆ multiple times. + let (got_name, got_fetch) = known_paths + .get_fetch_for_output_path(&FETCH_URL_OUT_PATH) + .expect("must be some"); + + assert_eq!("notmuch-extract-patch", got_name); + assert_eq!(FETCH_URL.clone(), got_fetch); + } + + // TODO: add test panicking about missing digest } diff --git a/tvix/glue/src/lib.rs b/tvix/glue/src/lib.rs index f04d5ec3a0..2e5a3be103 100644 --- a/tvix/glue/src/lib.rs +++ b/tvix/glue/src/lib.rs @@ -1,11 +1,11 @@ pub mod builtins; +pub mod fetchers; pub mod known_paths; pub mod refscan; pub mod tvix_build; pub mod tvix_io; pub mod tvix_store_io; -mod decompression; #[cfg(test)] mod tests; diff --git a/tvix/glue/src/tests/mod.rs b/tvix/glue/src/tests/mod.rs index 469ed17a51..8e1572b6e3 100644 --- a/tvix/glue/src/tests/mod.rs +++ b/tvix/glue/src/tests/mod.rs @@ -14,6 +14,8 @@ use rstest::rstest; use crate::{ builtins::{add_derivation_builtins, add_fetcher_builtins, add_import_builtins}, + configure_nix_path, + tvix_io::TvixIO, tvix_store_io::TvixStoreIO, }; @@ -50,12 +52,17 @@ fn eval_test(code_path: PathBuf, expect_success: bool) { Arc::new(DummyBuildService::default()), tokio_runtime.handle().clone(), )); - let mut eval = tvix_eval::Evaluation::new(tvix_store_io.clone() as Rc<dyn EvalIO>, true); + // Wrap with TvixIO, so <nix/fetchurl.nix can be imported. + let mut eval = tvix_eval::Evaluation::new( + Box::new(TvixIO::new(tvix_store_io.clone() as Rc<dyn EvalIO>)) as Box<dyn EvalIO>, + true, + ); eval.strict = true; add_derivation_builtins(&mut eval, tvix_store_io.clone()); add_fetcher_builtins(&mut eval, tvix_store_io.clone()); add_import_builtins(&mut eval, tvix_store_io.clone()); + configure_nix_path(&mut eval, &None); let result = eval.evaluate(code, Some(code_path.clone())); let failed = match result.value { @@ -134,9 +141,13 @@ fn nix_eval_okay(#[files("src/tests/nix_tests/eval-okay-*.nix")] code_path: Path // notyetpassing; this makes the test suite much more useful for // regression testing, since there should always be zero non-ignored // failing tests. -#[rstest] -fn nix_eval_okay_currently_failing( - #[files("src/tests/nix_tests/notyetpassing/eval-okay-*.nix")] code_path: PathBuf, -) { - eval_test(code_path, false) -} +// +// NOTE: There's no such test anymore. `rstest` does not handle empty directories, so, we +// just comment it for now. +// +// #[rstest] +// fn nix_eval_okay_currently_failing( +// #[files("src/tests/nix_tests/notyetpassing/eval-okay-*.nix")] code_path: PathBuf, +// ) { +// eval_test(code_path, false) +// } diff --git a/tvix/glue/src/tests/nix_tests/notyetpassing/eval-okay-context-introspection.exp b/tvix/glue/src/tests/nix_tests/eval-okay-context-introspection.exp index 03b400cc88..03b400cc88 100644 --- a/tvix/glue/src/tests/nix_tests/notyetpassing/eval-okay-context-introspection.exp +++ b/tvix/glue/src/tests/nix_tests/eval-okay-context-introspection.exp diff --git a/tvix/glue/src/tests/nix_tests/notyetpassing/eval-okay-context-introspection.nix b/tvix/glue/src/tests/nix_tests/eval-okay-context-introspection.nix index 354376b895..354376b895 100644 --- a/tvix/glue/src/tests/nix_tests/notyetpassing/eval-okay-context-introspection.nix +++ b/tvix/glue/src/tests/nix_tests/eval-okay-context-introspection.nix diff --git a/tvix/glue/src/tests/tvix_tests/eval-okay-context-introspection.exp b/tvix/glue/src/tests/tvix_tests/eval-okay-context-introspection.exp index 3f8905e9bd..a136b0035e 100644 --- a/tvix/glue/src/tests/tvix_tests/eval-okay-context-introspection.exp +++ b/tvix/glue/src/tests/tvix_tests/eval-okay-context-introspection.exp @@ -1 +1 @@ -[ true true true true true true true true true true true ] +[ true true true true true true true true true true true true true ] diff --git a/tvix/glue/src/tests/tvix_tests/eval-okay-context-introspection.nix b/tvix/glue/src/tests/tvix_tests/eval-okay-context-introspection.nix index 1b645b1a26..ecd8ab0073 100644 --- a/tvix/glue/src/tests/tvix_tests/eval-okay-context-introspection.nix +++ b/tvix/glue/src/tests/tvix_tests/eval-okay-context-introspection.nix @@ -41,6 +41,13 @@ let reconstructed-path = appendContextFrom combo-path (builtins.unsafeDiscardStringContext combo-path); + an-str = { + a = "${drv}"; + }; + an-list = { + b = [ drv ]; + }; + # Eta rule for strings with context. etaRule = str: str == appendContextFrom @@ -70,4 +77,7 @@ in (etaRule "foo") (etaRule drv.drvPath) (etaRule drv.foo.outPath) + # `toJSON` tests + (builtins.hasContext (builtins.toJSON an-str)) + (builtins.hasContext (builtins.toJSON an-list)) ] diff --git a/tvix/glue/src/tests/tvix_tests/eval-okay-context-propagation.exp b/tvix/glue/src/tests/tvix_tests/eval-okay-context-propagation.exp index 71c6ded406..ff56f6ca18 100644 --- a/tvix/glue/src/tests/tvix_tests/eval-okay-context-propagation.exp +++ b/tvix/glue/src/tests/tvix_tests/eval-okay-context-propagation.exp @@ -1 +1 @@ -[ true true true true true true true true true true true true true true true true true true true true true true true true true true ] +[ true true true true true true true true true true true true true true true true true true true true true true true true true true true true true true true true true true ] diff --git a/tvix/glue/src/tests/tvix_tests/eval-okay-context-propagation.nix b/tvix/glue/src/tests/tvix_tests/eval-okay-context-propagation.nix index 67f0ac4672..41e7f207b9 100644 --- a/tvix/glue/src/tests/tvix_tests/eval-okay-context-propagation.nix +++ b/tvix/glue/src/tests/tvix_tests/eval-okay-context-propagation.nix @@ -12,6 +12,11 @@ let system = "x86_64-linux"; outputs = [ "out" "bar" ]; }; + a-path-drv = builtins.path { + name = "a-path-drv"; + path = ./eval-okay-context-introspection.nix; + }; + another-path-drv = builtins.filterSource (_: true) ./eval-okay-context-introspection.nix; # `substr` propagates context, we truncate to an empty string and concatenate to the target # to infect it with the context of `copied`. @@ -37,6 +42,9 @@ in (builtins.hasContext "${(builtins.toFile "myself" "${./eval-okay-context-introspection.nix}")}") # `derivation` should produce context. (builtins.hasContext "${drv}") + # `builtins.path` / `builtins.filterSource` should produce context. + (builtins.hasContext "${a-path-drv}") + (builtins.hasContext "${another-path-drv}") # Low-level test to ensure that interpolation is working as expected. (builtins.length (builtins.attrNames (builtins.getContext "${drv}${other-drv}")) == 2) (builtins.getContext "${drv}${other-drv}" == mergeContext drv other-drv) @@ -85,4 +93,27 @@ in (preserveContext other-drv (builtins.concatStringsSep "${other-drv}" [ "abc" "def" ])) # `attrNames` will never ever produce context. (preserveContext "abc" (toString (builtins.attrNames { a = { }; b = { }; c = { }; }))) + # `toJSON` preserves context of its inputs. + (preserveContexts [ drv other-drv ] (builtins.toJSON { + a = [ drv ]; + b = [ other-drv ]; + })) + (preserveContexts [ drv other-drv ] (builtins.toJSON { + a.deep = [ drv ]; + b = [ other-drv ]; + })) + (preserveContexts [ drv other-drv ] (builtins.toJSON { + a = "${drv}"; + b = [ other-drv ]; + })) + (preserveContexts [ drv other-drv ] (builtins.toJSON { + a.deep = "${drv}"; + b = [ other-drv ]; + })) + (preserveContexts [ drv other-drv ] (builtins.toJSON { + a = "${drv} ${other-drv}"; + })) + (preserveContexts [ drv other-drv ] (builtins.toJSON { + a.b.c.d.e.f = "${drv} ${other-drv}"; + })) ] diff --git a/tvix/glue/src/tests/tvix_tests/eval-okay-fetchtarball.exp b/tvix/glue/src/tests/tvix_tests/eval-okay-fetchtarball.exp new file mode 100644 index 0000000000..c7332c0503 --- /dev/null +++ b/tvix/glue/src/tests/tvix_tests/eval-okay-fetchtarball.exp @@ -0,0 +1 @@ +[ /nix/store/7adgvk5zdfq4pwrhsm3n9lzypb12gw0g-source /nix/store/7adgvk5zdfq4pwrhsm3n9lzypb12gw0g-source /nix/store/7adgvk5zdfq4pwrhsm3n9lzypb12gw0g-source /nix/store/7adgvk5zdfq4pwrhsm3n9lzypb12gw0g-source /nix/store/7adgvk5zdfq4pwrhsm3n9lzypb12gw0g-source /nix/store/md9dsn2zwa6aj7zzalvjwwwx82whcyva-some-name ] diff --git a/tvix/glue/src/tests/tvix_tests/eval-okay-fetchtarball.nix b/tvix/glue/src/tests/tvix_tests/eval-okay-fetchtarball.nix new file mode 100644 index 0000000000..e454f12444 --- /dev/null +++ b/tvix/glue/src/tests/tvix_tests/eval-okay-fetchtarball.nix @@ -0,0 +1,42 @@ +[ + # (fetchTarball "url") cannot be tested, as that one has to fetch from the + # internet to calculate the path. + + # with url and sha256 + (builtins.fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/91050ea1e57e50388fa87a3302ba12d188ef723a.tar.gz"; + sha256 = "1hf6cgaci1n186kkkjq106ryf8mmlq9vnwgfwh625wa8hfgdn4dm"; + }) + + # with url and sha256 (as SRI) + (builtins.fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/91050ea1e57e50388fa87a3302ba12d188ef723a.tar.gz"; + sha256 = "sha256-tRHbnoNI8SIM5O5xuxOmtSLnswEByzmnQcGGyNRjxsE="; + }) + + # with another url, it actually doesn't matter (no .gz prefix) + (builtins.fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/91050ea1e57e50388fa87a3302ba12d188ef723a.tar"; + sha256 = "sha256-tRHbnoNI8SIM5O5xuxOmtSLnswEByzmnQcGGyNRjxsE="; + }) + + # also with an entirely different url, it doesn't change + (builtins.fetchTarball { + url = "https://test.example/owo"; + sha256 = "sha256-tRHbnoNI8SIM5O5xuxOmtSLnswEByzmnQcGGyNRjxsE="; + }) + + # โฆ because `name` defaults to source, and that (and the sha256 affect the store path) + (builtins.fetchTarball { + name = "source"; + url = "https://test.example/owo"; + sha256 = "sha256-tRHbnoNI8SIM5O5xuxOmtSLnswEByzmnQcGGyNRjxsE="; + }) + + # โฆ so changing name causes the hash to change. + (builtins.fetchTarball { + name = "some-name"; + url = "https://test.example/owo"; + sha256 = "sha256-tRHbnoNI8SIM5O5xuxOmtSLnswEByzmnQcGGyNRjxsE="; + }) +] diff --git a/tvix/glue/src/tests/tvix_tests/eval-okay-fetchurl.exp b/tvix/glue/src/tests/tvix_tests/eval-okay-fetchurl.exp new file mode 100644 index 0000000000..37a04d577c --- /dev/null +++ b/tvix/glue/src/tests/tvix_tests/eval-okay-fetchurl.exp @@ -0,0 +1 @@ +[ /nix/store/y0r1p1cqmlvm0yqkz3gxvkc1p8kg2sz8-null /nix/store/06qi00hylriyfm0nl827crgjvbax84mz-notmuch-extract-patch /nix/store/06qi00hylriyfm0nl827crgjvbax84mz-notmuch-extract-patch /nix/store/06qi00hylriyfm0nl827crgjvbax84mz-notmuch-extract-patch ] diff --git a/tvix/glue/src/tests/tvix_tests/eval-okay-fetchurl.nix b/tvix/glue/src/tests/tvix_tests/eval-okay-fetchurl.nix new file mode 100644 index 0000000000..8a39101525 --- /dev/null +++ b/tvix/glue/src/tests/tvix_tests/eval-okay-fetchurl.nix @@ -0,0 +1,25 @@ +[ + # (fetchurl "url") needs to immediately fetch, but our options without + # internet access are fairly limited. + # TODO: populate some fixtures at a known location instead. + (builtins.fetchurl "file:///dev/null") + + # fetchurl with url and sha256 + (builtins.fetchurl { + url = "https://raw.githubusercontent.com/aaptel/notmuch-extract-patch/f732a53e12a7c91a06755ebfab2007adc9b3063b/notmuch-extract-patch"; + sha256 = "0nawkl04sj7psw6ikzay7kydj3dhd0fkwghcsf5rzaw4bmp4kbax"; + }) + + # fetchurl with url and sha256 (as SRI) + (builtins.fetchurl { + url = "https://raw.githubusercontent.com/aaptel/notmuch-extract-patch/f732a53e12a7c91a06755ebfab2007adc9b3063b/notmuch-extract-patch"; + sha256 = "sha256-Xa1Jbl2Eq5+L0ww+Ph1osA3Z/Dxe/RkN1/dITQCdXFk="; + }) + + # fetchurl with another url, but same name + (builtins.fetchurl { + url = "https://test.example/owo"; + name = "notmuch-extract-patch"; + sha256 = "sha256-Xa1Jbl2Eq5+L0ww+Ph1osA3Z/Dxe/RkN1/dITQCdXFk="; + }) +] diff --git a/tvix/glue/src/tvix_io.rs b/tvix/glue/src/tvix_io.rs index 95146df728..0e5f23b990 100644 --- a/tvix/glue/src/tvix_io.rs +++ b/tvix/glue/src/tvix_io.rs @@ -8,7 +8,7 @@ //! otherwise fundamental features like nixpkgs bootstrapping and hash //! calculation will not work. -use std::io; +use std::io::{self, Cursor}; use std::path::{Path, PathBuf}; use tvix_eval::{EvalIO, FileType}; @@ -44,7 +44,7 @@ where self.actual.as_ref().path_exists(path) } - fn read_to_end(&self, path: &Path) -> io::Result<Vec<u8>> { + fn open(&self, path: &Path) -> io::Result<Box<dyn io::Read>> { // Bundled version of corepkgs/fetchurl.nix. The counterpart // of this happens in [crate::configure_nix_path], where the `nix_path` // of the evaluation has `nix=/__corepkgs__` added to it. @@ -52,13 +52,12 @@ where // This workaround is similar to what cppnix does for passing // the path through. // - // TODO: this comparison is bad and allocates, we should use - // the sane path library. + // TODO: this comparison is bad we should use the sane path library. if path.starts_with("/__corepkgs__/fetchurl.nix") { - return Ok(include_bytes!("fetchurl.nix").to_vec()); + return Ok(Box::new(Cursor::new(include_bytes!("fetchurl.nix")))); } - self.actual.as_ref().read_to_end(path) + self.actual.as_ref().open(path) } fn read_dir(&self, path: &Path) -> io::Result<Vec<(bytes::Bytes, FileType)>> { diff --git a/tvix/glue/src/tvix_store_io.rs b/tvix/glue/src/tvix_store_io.rs index 30ab97c0ca..7478fac9d2 100644 --- a/tvix/glue/src/tvix_store_io.rs +++ b/tvix/glue/src/tvix_store_io.rs @@ -1,15 +1,11 @@ //! This module provides an implementation of EvalIO talking to tvix-store. -use async_recursion::async_recursion; use bytes::Bytes; -use futures::Stream; use futures::{StreamExt, TryStreamExt}; use nix_compat::nixhash::NixHash; -use nix_compat::store_path::{build_ca_path, StorePathRef}; +use nix_compat::store_path::StorePathRef; use nix_compat::{nixhash::CAHash, store_path::StorePath}; use sha2::{Digest, Sha256}; -use std::marker::Unpin; -use std::rc::Rc; use std::{ cell::RefCell, collections::BTreeSet, @@ -17,21 +13,22 @@ use std::{ path::{Path, PathBuf}, sync::Arc, }; -use tokio::io::AsyncReadExt; -use tracing::{error, instrument, warn, Level}; +use tokio_util::io::SyncIoBridge; +use tracing::{error, info, instrument, warn, Level}; use tvix_build::buildservice::BuildService; -use tvix_eval::{ErrorKind, EvalIO, FileType, StdIO}; -use walkdir::DirEntry; +use tvix_castore::proto::node::Node; +use tvix_eval::{EvalIO, FileType, StdIO}; +use tvix_store::utils::AsyncIoBridge; use tvix_castore::{ blobservice::BlobService, directoryservice::{self, DirectoryService}, - proto::{node::Node, FileNode, NamedNode}, + proto::NamedNode, B3Digest, }; use tvix_store::{pathinfoservice::PathInfoService, proto::PathInfo}; -use crate::builtins::FetcherError; +use crate::fetchers::Fetcher; use crate::known_paths::KnownPaths; use crate::tvix_build::derivation_to_build_request; @@ -51,14 +48,19 @@ use crate::tvix_build::derivation_to_build_request; /// implementation of "Tvix Store IO" which does not necessarily bring the concept of blob service, /// directory service or path info service. pub struct TvixStoreIO { - blob_service: Arc<dyn BlobService>, - directory_service: Arc<dyn DirectoryService>, - path_info_service: Arc<dyn PathInfoService>, + // This is public so helper functions can interact with the stores directly. + pub(crate) blob_service: Arc<dyn BlobService>, + pub(crate) directory_service: Arc<dyn DirectoryService>, + pub(crate) path_info_service: Arc<dyn PathInfoService>, std_io: StdIO, #[allow(dead_code)] build_service: Arc<dyn BuildService>, pub(crate) tokio_handle: tokio::runtime::Handle, - http_client: reqwest::Client, + + pub(crate) fetcher: + Fetcher<Arc<dyn BlobService>, Arc<dyn DirectoryService>, Arc<dyn PathInfoService>>, + + // Paths known how to produce, by building or fetching. pub(crate) known_paths: RefCell<KnownPaths>, } @@ -71,13 +73,13 @@ impl TvixStoreIO { tokio_handle: tokio::runtime::Handle, ) -> Self { Self { - blob_service, - directory_service, - path_info_service, + blob_service: blob_service.clone(), + directory_service: directory_service.clone(), + path_info_service: path_info_service.clone(), std_io: StdIO {}, build_service, tokio_handle, - http_client: reqwest::Client::new(), + fetcher: Fetcher::new(blob_service, directory_service, path_info_service), known_paths: Default::default(), } } @@ -89,7 +91,6 @@ impl TvixStoreIO { /// /// In case there is no PathInfo yet, this means we need to build it /// (which currently is stubbed out still). - #[async_recursion(?Send)] #[instrument(skip(self, store_path), fields(store_path=%store_path), ret(level = Level::TRACE), err)] async fn store_path_to_node( &self, @@ -119,193 +120,208 @@ impl TvixStoreIO { // it for things like <nixpkgs> pointing to a store path. // In the future, these things will (need to) have PathInfo. None => { - // The store path doesn't exist yet, so we need to build it. - warn!("triggering build"); - - // Look up the derivation for this output path. - let (drv_path, drv) = { - let known_paths = self.known_paths.borrow(); - match known_paths.get_drv_path_for_output_path(store_path) { - Some(drv_path) => ( - drv_path.to_owned(), - known_paths.get_drv_by_drvpath(drv_path).unwrap().to_owned(), - ), - None => { - warn!(store_path=%store_path, "no drv found"); - // let StdIO take over - return Ok(None); - } + // The store path doesn't exist yet, so we need to fetch or build it. + // We check for fetches first, as we might have both native + // fetchers and FODs in KnownPaths, and prefer the former. + + let maybe_fetch = self + .known_paths + .borrow() + .get_fetch_for_output_path(store_path); + + match maybe_fetch { + Some((name, fetch)) => { + info!(?fetch, "triggering lazy fetch"); + let (sp, root_node) = self + .fetcher + .ingest_and_persist(&name, fetch) + .await + .map_err(|e| { + std::io::Error::new(std::io::ErrorKind::InvalidData, e) + })?; + + debug_assert_eq!( + sp.to_string(), + store_path.to_string(), + "store path returned from fetcher should match" + ); + + root_node } - }; - - // derivation_to_build_request needs castore nodes for all inputs. - // Provide them, which means, here is where we recursively build - // all dependencies. - #[allow(clippy::mutable_key_type)] - let input_nodes: BTreeSet<Node> = - futures::stream::iter(drv.input_derivations.iter()) - .map(|(input_drv_path, output_names)| { - // look up the derivation object - let input_drv = { - let known_paths = self.known_paths.borrow(); - known_paths - .get_drv_by_drvpath(input_drv_path) - .unwrap_or_else(|| panic!("{} not found", input_drv_path)) - .to_owned() + None => { + // Look up the derivation for this output path. + let (drv_path, drv) = { + let known_paths = self.known_paths.borrow(); + match known_paths.get_drv_path_for_output_path(store_path) { + Some(drv_path) => ( + drv_path.to_owned(), + known_paths.get_drv_by_drvpath(drv_path).unwrap().to_owned(), + ), + None => { + warn!(store_path=%store_path, "no drv found"); + // let StdIO take over + return Ok(None); + } + } + }; + + warn!("triggering build"); + + // derivation_to_build_request needs castore nodes for all inputs. + // Provide them, which means, here is where we recursively build + // all dependencies. + #[allow(clippy::mutable_key_type)] + let input_nodes: BTreeSet<Node> = + futures::stream::iter(drv.input_derivations.iter()) + .map(|(input_drv_path, output_names)| { + // look up the derivation object + let input_drv = { + let known_paths = self.known_paths.borrow(); + known_paths + .get_drv_by_drvpath(input_drv_path) + .unwrap_or_else(|| { + panic!("{} not found", input_drv_path) + }) + .to_owned() + }; + + // convert output names to actual paths + let output_paths: Vec<StorePath> = output_names + .iter() + .map(|output_name| { + input_drv + .outputs + .get(output_name) + .expect("missing output_name") + .path + .as_ref() + .expect("missing output path") + .clone() + }) + .collect(); + // For each output, ask for the castore node. + // We're in a per-derivation context, so if they're + // not built yet they'll all get built together. + // If they don't need to build, we can however still + // substitute all in parallel (if they don't need to + // be built) - so we turn this into a stream of streams. + // It's up to the builder to deduplicate same build requests. + futures::stream::iter(output_paths.into_iter()).map( + |output_path| async move { + let node = self + .store_path_to_node(&output_path, Path::new("")) + .await?; + + if let Some(node) = node { + Ok(node) + } else { + Err(io::Error::other("no node produced")) + } + }, + ) + }) + .flatten() + .buffer_unordered(10) // TODO: make configurable + .try_collect() + .await?; + + // TODO: check if input sources are sufficiently dealth with, + // I think yes, they must be imported into the store by other + // operations, so dealt with in the Some(โฆ) match arm + + // synthesize the build request. + let build_request = derivation_to_build_request(&drv, input_nodes)?; + + // create a build + let build_result = self + .build_service + .as_ref() + .do_build(build_request) + .await + .map_err(|e| std::io::Error::new(io::ErrorKind::Other, e))?; + + // TODO: refscan? + + // For each output, insert a PathInfo. + for output in &build_result.outputs { + let root_node = output.node.as_ref().expect("invalid root node"); + + // calculate the nar representation + let (nar_size, nar_sha256) = + self.path_info_service.calculate_nar(root_node).await?; + + // assemble the PathInfo to persist + let path_info = PathInfo { + node: Some(tvix_castore::proto::Node { + node: Some(root_node.clone()), + }), + references: vec![], // TODO: refscan + narinfo: Some(tvix_store::proto::NarInfo { + nar_size, + nar_sha256: Bytes::from(nar_sha256.to_vec()), + signatures: vec![], + reference_names: vec![], // TODO: refscan + deriver: Some(tvix_store::proto::StorePath { + name: drv_path + .name() + .strip_suffix(".drv") + .expect("missing .drv suffix") + .to_string(), + digest: drv_path.digest().to_vec().into(), + }), + ca: drv.fod_digest().map( + |fod_digest| -> tvix_store::proto::nar_info::Ca { + (&CAHash::Nar(nix_compat::nixhash::NixHash::Sha256( + fod_digest, + ))) + .into() + }, + ), + }), }; - // convert output names to actual paths - let output_paths: Vec<StorePath> = output_names - .iter() - .map(|output_name| { - input_drv - .outputs - .get(output_name) - .expect("missing output_name") - .path - .as_ref() - .expect("missing output path") - .clone() - }) - .collect(); - // For each output, ask for the castore node. - // We're in a per-derivation context, so if they're - // not built yet they'll all get built together. - // If they don't need to build, we can however still - // substitute all in parallel (if they don't need to - // be built) - so we turn this into a stream of streams. - // It's up to the builder to deduplicate same build requests. - futures::stream::iter(output_paths.into_iter()).map( - |output_path| async move { - let node = self - .store_path_to_node(&output_path, Path::new("")) - .await?; - - if let Some(node) = node { - Ok(node) - } else { - Err(io::Error::other("no node produced")) - } - }, - ) - }) - .flatten() - .buffer_unordered(10) // TODO: make configurable - .try_collect() - .await?; - - // TODO: check if input sources are sufficiently dealth with, - // I think yes, they must be imported into the store by other - // operations, so dealt with in the Some(โฆ) match arm - - // synthesize the build request. - let build_request = derivation_to_build_request(&drv, input_nodes)?; - - // create a build - let build_result = self - .build_service - .as_ref() - .do_build(build_request) - .await - .map_err(|e| std::io::Error::new(io::ErrorKind::Other, e))?; - - // TODO: refscan? - - // For each output, insert a PathInfo. - for output in &build_result.outputs { - let root_node = output.node.as_ref().expect("invalid root node"); - - // calculate the nar representation - let (nar_size, nar_sha256) = - self.path_info_service.calculate_nar(root_node).await?; - - // assemble the PathInfo to persist - let path_info = PathInfo { - node: Some(tvix_castore::proto::Node { - node: Some(root_node.clone()), - }), - references: vec![], // TODO: refscan - narinfo: Some(tvix_store::proto::NarInfo { - nar_size, - nar_sha256: Bytes::from(nar_sha256.to_vec()), - signatures: vec![], - reference_names: vec![], // TODO: refscan - deriver: Some(tvix_store::proto::StorePath { - name: drv_path - .name() - .strip_suffix(".drv") - .expect("missing .drv suffix") - .to_string(), - digest: drv_path.digest().to_vec().into(), - }), - ca: drv.fod_digest().map( - |fod_digest| -> tvix_store::proto::nar_info::Ca { - (&CAHash::Nar(nix_compat::nixhash::NixHash::Sha256(fod_digest))) - .into() - }, - ), - }), - }; - - self.path_info_service - .put(path_info) - .await - .map_err(|e| std::io::Error::new(io::ErrorKind::Other, e))?; - } + self.path_info_service + .put(path_info) + .await + .map_err(|e| std::io::Error::new(io::ErrorKind::Other, e))?; + } - // find the output for the store path requested - build_result - .outputs - .into_iter() - .find(|output_node| { - output_node.node.as_ref().expect("invalid node").get_name() - == store_path.to_string().as_bytes() - }) - .expect("build didn't produce the store path") - .node - .expect("invalid node") + // find the output for the store path requested + build_result + .outputs + .into_iter() + .find(|output_node| { + output_node.node.as_ref().expect("invalid node").get_name() + == store_path.to_string().as_bytes() + }) + .expect("build didn't produce the store path") + .node + .expect("invalid node") + } + } } }; // now with the root_node and sub_path, descend to the node requested. + // We convert sub_path to the castore model here. + let sub_path = tvix_castore::PathBuf::from_host_path(sub_path, true)?; + directoryservice::descend_to(&self.directory_service, root_node, sub_path) .await .map_err(|e| std::io::Error::new(io::ErrorKind::Other, e)) } - fn store_path_to_node_sync( - &self, - store_path: &StorePath, - sub_path: &Path, - ) -> io::Result<Option<Node>> { - self.tokio_handle - .block_on(async { self.store_path_to_node(store_path, sub_path).await }) - } - - /// This forwards the ingestion to the [`tvix_castore::import::ingest_entries`] - /// with a [`tokio::runtime::Handle::block_on`] call for synchronicity. - pub(crate) fn ingest_entries_sync<S>(&self, entries_stream: S) -> io::Result<Node> - where - S: Stream<Item = DirEntry> + Unpin, - { - self.tokio_handle.block_on(async move { - tvix_castore::import::ingest_entries( - &self.blob_service, - &self.directory_service, - entries_stream, - ) - .await - .map_err(|err| std::io::Error::new(io::ErrorKind::Other, err)) - }) - } - pub(crate) async fn node_to_path_info( &self, name: &str, path: &Path, + ca: CAHash, root_node: Node, - ) -> io::Result<(PathInfo, StorePath)> { + ) -> io::Result<(PathInfo, NixHash, StorePath)> { // Ask the PathInfoService for the NAR size and sha256 + // We always need it no matter what is the actual hash mode + // because the path info construct a narinfo which *always* + // require a SHA256 of the NAR representation and the NAR size. let (nar_size, nar_sha256) = self .path_info_service .as_ref() @@ -313,46 +329,61 @@ impl TvixStoreIO { .await?; // Calculate the output path. This might still fail, as some names are illegal. - let output_path = nix_compat::store_path::build_nar_based_store_path(&nar_sha256, name) - .map_err(|_| { - std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("invalid name: {}", name), - ) - })?; + let output_path = + nix_compat::store_path::build_ca_path(name, &ca, Vec::<String>::new(), false).map_err( + |_| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("invalid name: {}", name), + ) + }, + )?; // assemble a new root_node with a name that is derived from the nar hash. let root_node = root_node.rename(output_path.to_string().into_bytes().into()); tvix_store::import::log_node(&root_node, path); let path_info = - tvix_store::import::derive_nar_ca_path_info(nar_size, nar_sha256, root_node); + tvix_store::import::derive_nar_ca_path_info(nar_size, nar_sha256, Some(ca), root_node); - Ok((path_info, output_path.to_owned())) + Ok(( + path_info, + NixHash::Sha256(nar_sha256), + output_path.to_owned(), + )) } pub(crate) async fn register_node_in_path_info_service( &self, name: &str, path: &Path, + ca: CAHash, root_node: Node, ) -> io::Result<StorePath> { - let (path_info, output_path) = self.node_to_path_info(name, path, root_node).await?; + let (path_info, _, output_path) = self.node_to_path_info(name, path, ca, root_node).await?; let _path_info = self.path_info_service.as_ref().put(path_info).await?; Ok(output_path) } - pub(crate) fn register_node_in_path_info_service_sync( - &self, - name: &str, - path: &Path, - root_node: Node, - ) -> io::Result<StorePath> { - self.tokio_handle.block_on(async { - self.register_node_in_path_info_service(name, path, root_node) - .await - }) + /// Transforms a BLAKE-3 digest into a SHA256 digest + /// by re-hashing the whole file. + pub(crate) async fn blob_to_sha256_hash(&self, blob_digest: B3Digest) -> io::Result<[u8; 32]> { + let mut reader = self + .blob_service + .open_read(&blob_digest) + .await? + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + format!("blob represented by digest: '{}' not found", blob_digest), + ) + })?; + // It is fine to use `AsyncIoBridge` here because hashing is not actually I/O. + let mut hasher = AsyncIoBridge(Sha256::new()); + + tokio::io::copy(&mut reader, &mut hasher).await?; + Ok(hasher.0.finalize().into()) } pub async fn store_path_exists<'a>(&'a self, store_path: StorePathRef<'a>) -> io::Result<bool> { @@ -363,88 +394,6 @@ impl TvixStoreIO { .await? .is_some()) } - - pub async fn fetch_url( - &self, - url: &str, - name: &str, - hash: Option<&NixHash>, - ) -> Result<StorePath, ErrorKind> { - let resp = self - .http_client - .get(url) - .send() - .await - .map_err(FetcherError::from)?; - let mut sha = Sha256::new(); - let mut data = tokio_util::io::StreamReader::new( - resp.bytes_stream() - .inspect_ok(|data| { - sha.update(data); - }) - .map_err(|e| { - let e = e.without_url(); - warn!(%e, "failed to get response body"); - io::Error::new(io::ErrorKind::BrokenPipe, e.to_string()) - }), - ); - - let mut blob = self.blob_service.open_write().await; - let size = tokio::io::copy(&mut data, blob.as_mut()).await?; - let blob_digest = blob.close().await?; - let got = NixHash::Sha256(sha.finalize().into()); - - let hash = CAHash::Flat(if let Some(wanted) = hash { - if *wanted != got { - return Err(FetcherError::HashMismatch { - url: url.to_owned(), - wanted: wanted.clone(), - got, - } - .into()); - } - wanted.clone() - } else { - got - }); - - let path = build_ca_path(name, &hash, Vec::<String>::new(), false) - .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; - let node = Node::File(FileNode { - name: path.to_string().into(), - digest: blob_digest.into(), - size, - executable: false, - }); - - let (nar_size, nar_sha256) = self - .path_info_service - .calculate_nar(&node) - .await - .map_err(|e| ErrorKind::TvixError(Rc::new(e)))?; - - let path_info = PathInfo { - node: Some(tvix_castore::proto::Node { - node: Some(node.clone()), - }), - references: vec![], - narinfo: Some(tvix_store::proto::NarInfo { - nar_size, - nar_sha256: nar_sha256.to_vec().into(), - signatures: vec![], - reference_names: vec![], - deriver: None, /* ? */ - ca: Some((&hash).into()), - }), - }; - - self.path_info_service - .put(path_info) - .await - .map_err(|e| std::io::Error::new(io::ErrorKind::Other, e))?; - - Ok(path.to_owned()) - } } impl EvalIO for TvixStoreIO { @@ -454,7 +403,8 @@ impl EvalIO for TvixStoreIO { StorePath::from_absolute_path_full(&path.to_string_lossy()) { if self - .store_path_to_node_sync(&store_path, &sub_path)? + .tokio_handle + .block_on(async { self.store_path_to_node(&store_path, &sub_path).await })? .is_some() { Ok(true) @@ -470,18 +420,21 @@ impl EvalIO for TvixStoreIO { } #[instrument(skip(self), err)] - fn read_to_end(&self, path: &Path) -> io::Result<Vec<u8>> { + fn open(&self, path: &Path) -> io::Result<Box<dyn io::Read>> { if let Ok((store_path, sub_path)) = StorePath::from_absolute_path_full(&path.to_string_lossy()) { - if let Some(node) = self.store_path_to_node_sync(&store_path, &sub_path)? { - // depending on the node type, treat read_to_string differently + if let Some(node) = self + .tokio_handle + .block_on(async { self.store_path_to_node(&store_path, &sub_path).await })? + { + // depending on the node type, treat open differently match node { Node::Directory(_) => { // This would normally be a io::ErrorKind::IsADirectory (still unstable) Err(io::Error::new( io::ErrorKind::Unsupported, - format!("tried to read directory at {:?} to string", path), + format!("tried to open directory at {:?}", path), )) } Node::File(file_node) => { @@ -498,42 +451,39 @@ impl EvalIO for TvixStoreIO { })?; self.tokio_handle.block_on(async { - let mut reader = { - let resp = self.blob_service.as_ref().open_read(&digest).await?; - match resp { - Some(blob_reader) => blob_reader, - None => { - error!( - blob.digest = %digest, - "blob not found", - ); - Err(io::Error::new( - io::ErrorKind::NotFound, - format!("blob {} not found", &digest), - ))? - } + let resp = self.blob_service.as_ref().open_read(&digest).await?; + match resp { + Some(blob_reader) => { + // The VM Response needs a sync [std::io::Reader]. + Ok(Box::new(SyncIoBridge::new(blob_reader)) + as Box<dyn io::Read>) } - }; - - let mut buf = Vec::new(); - - reader.read_to_end(&mut buf).await?; - Ok(buf) + None => { + error!( + blob.digest = %digest, + "blob not found", + ); + Err(io::Error::new( + io::ErrorKind::NotFound, + format!("blob {} not found", &digest), + )) + } + } }) } Node::Symlink(_symlink_node) => Err(io::Error::new( io::ErrorKind::Unsupported, - "read_to_string for symlinks is unsupported", + "open for symlinks is unsupported", ))?, } } else { // As tvix-store doesn't manage /nix/store on the filesystem, // we still need to also ask self.std_io here. - self.std_io.read_to_end(path) + self.std_io.open(path) } } else { // The store path is no store path, so do regular StdIO. - self.std_io.read_to_end(path) + self.std_io.open(path) } } @@ -542,7 +492,10 @@ impl EvalIO for TvixStoreIO { if let Ok((store_path, sub_path)) = StorePath::from_absolute_path_full(&path.to_string_lossy()) { - if let Some(node) = self.store_path_to_node_sync(&store_path, &sub_path)? { + if let Some(node) = self + .tokio_handle + .block_on(async { self.store_path_to_node(&store_path, &sub_path).await })? + { match node { Node::Directory(directory_node) => { // fetch the Directory itself. diff --git a/tvix/nar-bridge/pkg/http/narinfo_get.go b/tvix/nar-bridge/pkg/http/narinfo_get.go index 98d85744d8..d43cb58078 100644 --- a/tvix/nar-bridge/pkg/http/narinfo_get.go +++ b/tvix/nar-bridge/pkg/http/narinfo_get.go @@ -96,37 +96,42 @@ func renderNarinfo( } func registerNarinfoGet(s *Server) { - // GET $outHash.narinfo looks up the PathInfo from the tvix-store, - // and then render a .narinfo file to the client. - // It will keep the PathInfo in the lookup map, - // so a subsequent GET /nar/ $narhash.nar request can find it. - s.handler.Get("/{outputhash:^["+nixbase32.Alphabet+"]{32}}.narinfo", func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - - ctx := r.Context() - log := log.WithField("outputhash", chi.URLParamFromCtx(ctx, "outputhash")) - - // parse the output hash sent in the request URL - outputHash, err := nixbase32.DecodeString(chi.URLParamFromCtx(ctx, "outputhash")) - if err != nil { - log.WithError(err).Error("unable to decode output hash from url") - w.WriteHeader(http.StatusBadRequest) - _, err := w.Write([]byte("unable to decode output hash from url")) + // GET/HEAD $outHash.narinfo looks up the PathInfo from the tvix-store, + // and, if it's a GET request, render a .narinfo file to the client. + // In both cases it will keep the PathInfo in the lookup map, + // so a subsequent GET/HEAD /nar/ $narhash.nar request can find it. + genNarinfoHandler := func(isHead bool) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + + ctx := r.Context() + log := log.WithField("outputhash", chi.URLParamFromCtx(ctx, "outputhash")) + + // parse the output hash sent in the request URL + outputHash, err := nixbase32.DecodeString(chi.URLParamFromCtx(ctx, "outputhash")) if err != nil { - log.WithError(err).Errorf("unable to write error message to client") + log.WithError(err).Error("unable to decode output hash from url") + w.WriteHeader(http.StatusBadRequest) + _, err := w.Write([]byte("unable to decode output hash from url")) + if err != nil { + log.WithError(err).Errorf("unable to write error message to client") + } + + return } - return - } - - err = renderNarinfo(ctx, log, s.pathInfoServiceClient, &s.narDbMu, s.narDb, outputHash, w, false) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - w.WriteHeader(http.StatusNotFound) - } else { - log.WithError(err).Warn("unable to render narinfo") - w.WriteHeader(http.StatusInternalServerError) + err = renderNarinfo(ctx, log, s.pathInfoServiceClient, &s.narDbMu, s.narDb, outputHash, w, isHead) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + w.WriteHeader(http.StatusNotFound) + } else { + log.WithError(err).Warn("unable to render narinfo") + w.WriteHeader(http.StatusInternalServerError) + } } } - }) + } + + s.handler.Get("/{outputhash:^["+nixbase32.Alphabet+"]{32}}.narinfo", genNarinfoHandler(false)) + s.handler.Head("/{outputhash:^["+nixbase32.Alphabet+"]{32}}.narinfo", genNarinfoHandler(true)) } diff --git a/tvix/nix-compat/Cargo.toml b/tvix/nix-compat/Cargo.toml index 6e3df0485d..876ac3ecad 100644 --- a/tvix/nix-compat/Cargo.toml +++ b/tvix/nix-compat/Cargo.toml @@ -3,10 +3,14 @@ name = "nix-compat" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [features] -async = ["futures-util", "tokio"] +# async NAR writer +async = ["tokio"] +# code emitting low-level packets used in the daemon protocol. +wire = ["tokio", "pin-project-lite"] + +# Enable all features by default. +default = ["async", "wire"] [dependencies] bitflags = "2.4.1" @@ -15,7 +19,6 @@ data-encoding = "2.3.3" ed25519 = "2.2.3" ed25519-dalek = "2.1.0" enum-primitive-derive = "0.3.0" -futures-util = { version = "0.3.30", features = ["io"], optional = true } glob = "0.3.0" nom = "7.1.3" num-traits = "0.2.18" @@ -29,23 +32,21 @@ optional = true version = "1.32.0" features = ["io-util", "macros"] +[dependencies.pin-project-lite] +optional = true +version = "0.2.13" + [dev-dependencies] -futures = { version = "0.3.30", default-features = false, features = ["executor"] } -lazy_static = "1.4.0" -serde_json = "1.0" -test-case = "3.3.1" criterion = { version = "0.5", features = ["html_reports"] } +futures = { version = "0.3.30", default-features = false, features = ["executor"] } hex-literal = "0.4.1" +lazy_static = "1.4.0" pretty_assertions = "1.4.0" +rstest = "0.19.0" +serde_json = "1.0" tokio-test = "0.4.3" zstd = "^0.13.0" -[dev-dependencies.test-generator] -# This fork of test-generator adds support for cargo workspaces, see -# also https://github.com/frehberg/test-generator/pull/14 -git = "https://github.com/JamesGuthrie/test-generator.git" -rev = "82e799979980962aec1aa324ec6e0e4cad781f41" - [[bench]] name = "derivation_parse_aterm" harness = false diff --git a/tvix/nix-compat/default.nix b/tvix/nix-compat/default.nix index d6169f1339..9df76e12fc 100644 --- a/tvix/nix-compat/default.nix +++ b/tvix/nix-compat/default.nix @@ -3,5 +3,5 @@ depot.tvix.crates.workspaceMembers.nix-compat.build.override { runTests = true; # make sure we also enable async here, so run the tests behind that feature flag. - features = [ "default" "async" ]; + features = [ "default" "async" "wire" ]; } diff --git a/tvix/nix-compat/src/aterm/escape.rs b/tvix/nix-compat/src/aterm/escape.rs index 06b550bbf0..80a85d2103 100644 --- a/tvix/nix-compat/src/aterm/escape.rs +++ b/tvix/nix-compat/src/aterm/escape.rs @@ -16,12 +16,13 @@ pub fn escape_bytes<P: AsRef<[u8]>>(s: P) -> Vec<u8> { #[cfg(test)] mod tests { use super::escape_bytes; - use test_case::test_case; + use rstest::rstest; - #[test_case(b"", b""; "empty")] - #[test_case(b"\"", b"\\\""; "doublequote")] - #[test_case(b":", b":"; "colon")] - fn escape(input: &[u8], expected: &[u8]) { + #[rstest] + #[case::empty(b"", b"")] + #[case::doublequote(b"\"", b"\\\"")] + #[case::colon(b":", b":")] + fn escape(#[case] input: &[u8], #[case] expected: &[u8]) { assert_eq!(expected, escape_bytes(input)) } } diff --git a/tvix/nix-compat/src/aterm/parser.rs b/tvix/nix-compat/src/aterm/parser.rs index 72648d5ef5..a30cb40ab0 100644 --- a/tvix/nix-compat/src/aterm/parser.rs +++ b/tvix/nix-compat/src/aterm/parser.rs @@ -76,25 +76,31 @@ pub(crate) fn parse_str_list(i: &[u8]) -> IResult<&[u8], Vec<String>> { #[cfg(test)] mod tests { - use test_case::test_case; + use rstest::rstest; - #[test_case(br#""""#, b"", b""; "empty")] - #[test_case(br#""Hello World""#, b"Hello World", b""; "hello world")] - #[test_case(br#""\"""#, br#"""#, b""; "doublequote")] - #[test_case(br#"":""#, b":", b""; "colon")] - #[test_case(br#""\""Rest"#, br#"""#, b"Rest"; "doublequote rest")] - fn parse_bstr_field(input: &[u8], expected: &[u8], exp_rest: &[u8]) { + #[rstest] + #[case::empty(br#""""#, b"", b"")] + #[case::hello_world(br#""Hello World""#, b"Hello World", b"")] + #[case::doublequote(br#""\"""#, br#"""#, b"")] + #[case::colon(br#"":""#, b":", b"")] + #[case::doublequote_rest(br#""\""Rest"#, br#"""#, b"Rest")] + fn test_parse_bstr_field( + #[case] input: &[u8], + #[case] expected: &[u8], + #[case] exp_rest: &[u8], + ) { let (rest, parsed) = super::parse_bstr_field(input).expect("must parse"); assert_eq!(exp_rest, rest, "expected remainder"); assert_eq!(expected, parsed); } - #[test_case(br#""""#, "", b""; "empty")] - #[test_case(br#""Hello World""#, "Hello World", b""; "hello world")] - #[test_case(br#""\"""#, r#"""#, b""; "doublequote")] - #[test_case(br#"":""#, ":", b""; "colon")] - #[test_case(br#""\""Rest"#, r#"""#, b"Rest"; "doublequote rest")] - fn parse_string_field(input: &[u8], expected: &str, exp_rest: &[u8]) { + #[rstest] + #[case::empty(br#""""#, "", b"")] + #[case::hello_world(br#""Hello World""#, "Hello World", b"")] + #[case::doublequote(br#""\"""#, r#"""#, b"")] + #[case::colon(br#"":""#, ":", b"")] + #[case::doublequote_rest(br#""\""Rest"#, r#"""#, b"Rest")] + fn parse_string_field(#[case] input: &[u8], #[case] expected: &str, #[case] exp_rest: &[u8]) { let (rest, parsed) = super::parse_string_field(input).expect("must parse"); assert_eq!(exp_rest, rest, "expected remainder"); assert_eq!(expected, &parsed); @@ -107,10 +113,11 @@ mod tests { super::parse_string_field(&input).expect_err("must fail"); } - #[test_case(br#"["foo"]"#, vec!["foo".to_string()], b""; "single foo")] - #[test_case(b"[]", vec![], b""; "empty list")] - #[test_case(b"[]blub", vec![], b"blub"; "empty list with rest")] - fn parse_list(input: &[u8], expected: Vec<String>, exp_rest: &[u8]) { + #[rstest] + #[case::single_foo(br#"["foo"]"#, vec!["foo".to_string()], b"")] + #[case::empty_list(b"[]", vec![], b"")] + #[case::empty_list_with_rest(b"[]blub", vec![], b"blub")] + fn parse_list(#[case] input: &[u8], #[case] expected: Vec<String>, #[case] exp_rest: &[u8]) { let (rest, parsed) = super::parse_str_list(input).expect("must parse"); assert_eq!(exp_rest, rest, "expected remainder"); assert_eq!(expected, parsed); diff --git a/tvix/nix-compat/src/derivation/mod.rs b/tvix/nix-compat/src/derivation/mod.rs index 07da127ed0..6e12e3ea86 100644 --- a/tvix/nix-compat/src/derivation/mod.rs +++ b/tvix/nix-compat/src/derivation/mod.rs @@ -188,11 +188,12 @@ impl Derivation { /// `fixed:out:${algo}:${digest}:${fodPath}` string is hashed instead of /// the A-Term. /// - /// If the derivation is not a fixed derivation, it's up to the caller of - /// this function to provide a lookup function to lookup these calculation - /// results of parent derivations at `fn_get_derivation_or_fod_hash` (by - /// drv path). - pub fn derivation_or_fod_hash<F>(&self, fn_get_derivation_or_fod_hash: F) -> [u8; 32] + /// It's up to the caller of this function to provide a (infallible) lookup + /// function to query [hash_derivation_modulo] of direct input derivations, + /// by their [StorePathRef]. + /// It will only be called in case the derivation is not a fixed-output + /// derivation. + pub fn hash_derivation_modulo<F>(&self, fn_lookup_hash_derivation_modulo: F) -> [u8; 32] where F: Fn(&StorePathRef) -> [u8; 32], { @@ -200,16 +201,16 @@ impl Derivation { // Non-Fixed-output derivations return the sha256 digest of the ATerm // notation, but with all input_derivation paths replaced by a recursive // call to this function. - // We use fn_get_derivation_or_fod_hash here, so callers can precompute this. + // We call [fn_lookup_hash_derivation_modulo] rather than recursing + // ourselves, so callers can precompute this. self.fod_digest().unwrap_or({ - // For each input_derivation, look up the - // derivation_or_fod_hash, and replace the derivation path with - // it's HEXLOWER digest. + // For each input_derivation, look up the hash derivation modulo, + // and replace the derivation path in the aterm with it's HEXLOWER digest. let aterm_bytes = self.to_aterm_bytes_with_replacements(&BTreeMap::from_iter( self.input_derivations .iter() .map(|(drv_path, output_names)| { - let hash = fn_get_derivation_or_fod_hash(&drv_path.into()); + let hash = fn_lookup_hash_derivation_modulo(&drv_path.into()); (hash, output_names.to_owned()) }), @@ -226,20 +227,22 @@ impl Derivation { /// and self.environment[$outputName] needs to be an empty string. /// /// Output path calculation requires knowledge of the - /// derivation_or_fod_hash [NixHash], which (in case of non-fixed-output - /// derivations) also requires knowledge of other hash_derivation_modulo - /// [NixHash]es. + /// [hash_derivation_modulo], which (in case of non-fixed-output + /// derivations) also requires knowledge of the [hash_derivation_modulo] of + /// input derivations (recursively). /// - /// We solve this by asking the caller of this function to provide the - /// hash_derivation_modulo of the current Derivation. + /// To avoid recursing and doing unnecessary calculation, we simply + /// ask the caller of this function to provide the result of the + /// [hash_derivation_modulo] call of the current [Derivation], + /// and leave it up to them to calculate it when needed. /// - /// On completion, self.environment[$outputName] and - /// self.outputs[$outputName].path are set to the calculated output path for all + /// On completion, `self.environment[$outputName]` and + /// `self.outputs[$outputName].path` are set to the calculated output path for all /// outputs. pub fn calculate_output_paths( &mut self, name: &str, - derivation_or_fod_hash: &[u8; 32], + hash_derivation_modulo: &[u8; 32], ) -> Result<(), DerivationError> { // The fingerprint and hash differs per output for (output_name, output) in self.outputs.iter_mut() { @@ -250,14 +253,14 @@ impl Derivation { let path_name = output_path_name(name, output_name); - // For fixed output derivation we use the per-output info, otherwise we use the - // derivation hash. + // For fixed output derivation we use [build_ca_path], otherwise we + // use [build_output_path] with [hash_derivation_modulo]. let abs_store_path = if let Some(ref hwm) = output.ca_hash { build_ca_path(&path_name, hwm, Vec::<String>::new(), false).map_err(|e| { DerivationError::InvalidOutputDerivationPath(output_name.to_string(), e) })? } else { - build_output_path(derivation_or_fod_hash, output_name, &path_name).map_err(|e| { + build_output_path(hash_derivation_modulo, output_name, &path_name).map_err(|e| { DerivationError::InvalidOutputDerivationPath( output_name.to_string(), store_path::BuildStorePathError::InvalidStorePath(e), diff --git a/tvix/nix-compat/src/derivation/parser.rs b/tvix/nix-compat/src/derivation/parser.rs index 3f291c2040..2775294960 100644 --- a/tvix/nix-compat/src/derivation/parser.rs +++ b/tvix/nix-compat/src/derivation/parser.rs @@ -48,7 +48,7 @@ pub(crate) fn parse(i: &[u8]) -> Result<Derivation, Error<&[u8]>> { fn from_algo_and_mode_and_digest<B: AsRef<[u8]>>( algo_and_mode: &str, digest: B, -) -> crate::nixhash::Result<CAHash> { +) -> crate::nixhash::NixHashResult<CAHash> { Ok(match algo_and_mode.strip_prefix("r:") { Some(algo) => nixhash::CAHash::Nar(nixhash::from_algo_and_digest( algo.try_into()?, @@ -339,7 +339,8 @@ mod tests { use bstr::{BString, ByteSlice}; use hex_literal::hex; use lazy_static::lazy_static; - use test_case::test_case; + use rstest::rstest; + const DIGEST_SHA256: [u8; 32] = hex!("a5ce9c155ed09397614646c9717fc7cd94b1023d7b76b618d409e4fefd6e9d39"); @@ -418,9 +419,14 @@ mod tests { } /// Ensure parsing KVs works - #[test_case(b"[]", &BTreeMap::new(), b""; "empty")] - #[test_case(b"[(\"a\",\"1\"),(\"b\",\"2\")]", &EXP_AB_MAP, b""; "simple")] - fn parse_kv(input: &'static [u8], expected: &BTreeMap<String, BString>, exp_rest: &[u8]) { + #[rstest] + #[case::empty(b"[]", &BTreeMap::new(), b"")] + #[case::simple(b"[(\"a\",\"1\"),(\"b\",\"2\")]", &EXP_AB_MAP, b"")] + fn parse_kv( + #[case] input: &'static [u8], + #[case] expected: &BTreeMap<String, BString>, + #[case] exp_rest: &[u8], + ) { let (rest, parsed) = super::parse_kv::<BString, _>(crate::aterm::parse_bstr_field)(input) .expect("must parse"); assert_eq!(exp_rest, rest, "expected remainder"); @@ -443,11 +449,12 @@ mod tests { } /// Ensure parsing input derivations works. - #[test_case(b"[]", &BTreeMap::new(); "empty")] - #[test_case(EXP_INPUT_DERIVATIONS_SIMPLE_ATERM.as_bytes(), &EXP_INPUT_DERIVATIONS_SIMPLE; "simple")] + #[rstest] + #[case::empty(b"[]", &BTreeMap::new())] + #[case::simple(EXP_INPUT_DERIVATIONS_SIMPLE_ATERM.as_bytes(), &EXP_INPUT_DERIVATIONS_SIMPLE)] fn parse_input_derivations( - input: &'static [u8], - expected: &BTreeMap<StorePath, BTreeSet<String>>, + #[case] input: &'static [u8], + #[case] expected: &BTreeMap<StorePath, BTreeSet<String>>, ) { let (rest, parsed) = super::parse_input_derivations(input).expect("must parse"); @@ -480,9 +487,10 @@ mod tests { } /// Ensure parsing input sources works - #[test_case(b"[]", &BTreeSet::new(); "empty")] - #[test_case(b"[\"/nix/store/55lwldka5nyxa08wnvlizyqw02ihy8ic-has-multi-out\",\"/nix/store/2vixb94v0hy2xc6p7mbnxxcyc095yyia-has-multi-out-lib\"]", &EXP_INPUT_SOURCES_SIMPLE; "simple")] - fn parse_input_sources(input: &'static [u8], expected: &BTreeSet<String>) { + #[rstest] + #[case::empty(b"[]", &BTreeSet::new())] + #[case::simple(b"[\"/nix/store/55lwldka5nyxa08wnvlizyqw02ihy8ic-has-multi-out\",\"/nix/store/2vixb94v0hy2xc6p7mbnxxcyc095yyia-has-multi-out-lib\"]", &EXP_INPUT_SOURCES_SIMPLE)] + fn parse_input_sources(#[case] input: &'static [u8], #[case] expected: &BTreeSet<String>) { let (rest, parsed) = super::parse_input_sources(input).expect("must parse"); assert_eq!( @@ -519,15 +527,16 @@ mod tests { } } - #[test_case( + #[rstest] + #[case::simple( br#"("out","/nix/store/5vyvcwah9l9kf07d52rcgdk70g2f4y13-foo","","")"#, ("out".to_string(), Output { path: Some( StorePathRef::from_absolute_path("/nix/store/5vyvcwah9l9kf07d52rcgdk70g2f4y13-foo".as_bytes()).unwrap().to_owned()), ca_hash: None - }); "simple" + }) )] - #[test_case( + #[case::fod( br#"("out","/nix/store/4q0pg5zpfmznxscq3avycvf9xdvx50n3-bar","r:sha256","08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba")"#, ("out".to_string(), Output { path: Some( @@ -535,28 +544,33 @@ mod tests { "/nix/store/4q0pg5zpfmznxscq3avycvf9xdvx50n3-bar".as_bytes()).unwrap().to_owned()), ca_hash: Some(from_algo_and_mode_and_digest("r:sha256", data_encoding::HEXLOWER.decode(b"08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba").unwrap() ).unwrap()), - }); "fod" + }) )] - fn parse_output(input: &[u8], expected: (String, Output)) { + fn parse_output(#[case] input: &[u8], #[case] expected: (String, Output)) { let (rest, parsed) = super::parse_output(input).expect("must parse"); assert!(rest.is_empty()); assert_eq!(expected, parsed); } - #[test_case( + #[rstest] + #[case::multi_out( br#"[("lib","/nix/store/2vixb94v0hy2xc6p7mbnxxcyc095yyia-has-multi-out-lib","",""),("out","/nix/store/55lwldka5nyxa08wnvlizyqw02ihy8ic-has-multi-out","","")]"#, - &EXP_MULTI_OUTPUTS; - "multi-out" + &EXP_MULTI_OUTPUTS )] - fn parse_outputs(input: &[u8], expected: &BTreeMap<String, Output>) { + fn parse_outputs(#[case] input: &[u8], #[case] expected: &BTreeMap<String, Output>) { let (rest, parsed) = super::parse_outputs(input).expect("must parse"); assert!(rest.is_empty()); assert_eq!(*expected, parsed); } - #[test_case("sha256", &DIGEST_SHA256, CAHash::Flat(NIXHASH_SHA256.clone()); "sha256 flat")] - #[test_case("r:sha256", &DIGEST_SHA256, CAHash::Nar(NIXHASH_SHA256.clone()); "sha256 recursive")] - fn test_from_algo_and_mode_and_digest(algo_and_mode: &str, digest: &[u8], expected: CAHash) { + #[rstest] + #[case::sha256_flat("sha256", &DIGEST_SHA256, CAHash::Flat(NIXHASH_SHA256.clone()))] + #[case::sha256_recursive("r:sha256", &DIGEST_SHA256, CAHash::Nar(NIXHASH_SHA256.clone()))] + fn test_from_algo_and_mode_and_digest( + #[case] algo_and_mode: &str, + #[case] digest: &[u8], + #[case] expected: CAHash, + ) { assert_eq!( expected, from_algo_and_mode_and_digest(algo_and_mode, digest).unwrap() diff --git a/tvix/nix-compat/src/derivation/tests/mod.rs b/tvix/nix-compat/src/derivation/tests/mod.rs index 56bad7869d..48d4e8926a 100644 --- a/tvix/nix-compat/src/derivation/tests/mod.rs +++ b/tvix/nix-compat/src/derivation/tests/mod.rs @@ -6,51 +6,41 @@ use crate::derivation::Derivation; use crate::store_path::StorePath; use bstr::{BStr, BString}; use hex_literal::hex; +use rstest::rstest; use std::collections::BTreeSet; -use std::fs::File; -use std::io::Read; -use std::path::Path; +use std::fs; +use std::path::{Path, PathBuf}; use std::str::FromStr; -use test_case::test_case; -use test_generator::test_resources; const RESOURCES_PATHS: &str = "src/derivation/tests/derivation_tests"; -fn read_file(path: &str) -> BString { - let path = Path::new(path); - let mut file = File::open(path).unwrap(); - let mut file_contents = Vec::new(); - - file.read_to_end(&mut file_contents).unwrap(); - - file_contents.into() -} - -#[test_resources("src/derivation/tests/derivation_tests/ok/*.drv")] -fn check_serialization(path_to_drv_file: &str) { - // skip JSON files known to fail parsing - if path_to_drv_file.ends_with("cp1252.drv") || path_to_drv_file.ends_with("latin1.drv") { - return; - } - let json_bytes = read_file(&format!("{}.json", path_to_drv_file)); +#[rstest] +fn check_serialization( + #[files("src/derivation/tests/derivation_tests/ok/*.drv")] + #[exclude("(cp1252)|(latin1)")] // skip JSON files known to fail parsing + path_to_drv_file: PathBuf, +) { + let json_bytes = + fs::read(path_to_drv_file.with_extension("drv.json")).expect("unable to read JSON"); let derivation: Derivation = serde_json::from_slice(&json_bytes).expect("JSON was not well-formatted"); let mut serialized_derivation = Vec::new(); derivation.serialize(&mut serialized_derivation).unwrap(); - let expected = read_file(path_to_drv_file); + let expected = fs::read(&path_to_drv_file).expect("unable to read .drv"); assert_eq!(expected, BStr::new(&serialized_derivation)); } -#[test_resources("src/derivation/tests/derivation_tests/ok/*.drv")] -fn validate(path_to_drv_file: &str) { - // skip JSON files known to fail parsing - if path_to_drv_file.ends_with("cp1252.drv") || path_to_drv_file.ends_with("latin1.drv") { - return; - } - let json_bytes = read_file(&format!("{}.json", path_to_drv_file)); +#[rstest] +fn validate( + #[files("src/derivation/tests/derivation_tests/ok/*.drv")] + #[exclude("(cp1252)|(latin1)")] // skip JSON files known to fail parsing + path_to_drv_file: PathBuf, +) { + let json_bytes = + fs::read(path_to_drv_file.with_extension("drv.json")).expect("unable to read JSON"); let derivation: Derivation = serde_json::from_slice(&json_bytes).expect("JSON was not well-formatted"); @@ -59,17 +49,18 @@ fn validate(path_to_drv_file: &str) { .expect("derivation failed to validate") } -#[test_resources("src/derivation/tests/derivation_tests/ok/*.drv")] -fn check_to_aterm_bytes(path_to_drv_file: &str) { - // skip JSON files known to fail parsing - if path_to_drv_file.ends_with("cp1252.drv") || path_to_drv_file.ends_with("latin1.drv") { - return; - } - let json_bytes = read_file(&format!("{}.json", path_to_drv_file)); +#[rstest] +fn check_to_aterm_bytes( + #[files("src/derivation/tests/derivation_tests/ok/*.drv")] + #[exclude("(cp1252)|(latin1)")] // skip JSON files known to fail parsing + path_to_drv_file: PathBuf, +) { + let json_bytes = + fs::read(path_to_drv_file.with_extension("drv.json")).expect("unable to read JSON"); let derivation: Derivation = serde_json::from_slice(&json_bytes).expect("JSON was not well-formatted"); - let expected = read_file(path_to_drv_file); + let expected = fs::read(&path_to_drv_file).expect("unable to read .drv"); assert_eq!(expected, BStr::new(&derivation.to_aterm_bytes())); } @@ -77,22 +68,28 @@ fn check_to_aterm_bytes(path_to_drv_file: &str) { /// Reads in derivations in ATerm representation, parses with that parser, /// then compares the structs with the ones obtained by parsing the JSON /// representations. -#[test_resources("src/derivation/tests/derivation_tests/ok/*.drv")] -fn from_aterm_bytes(path_to_drv_file: &str) { +#[rstest] +fn from_aterm_bytes( + #[files("src/derivation/tests/derivation_tests/ok/*.drv")] path_to_drv_file: PathBuf, +) { // Read in ATerm representation. - let aterm_bytes = read_file(path_to_drv_file); + let aterm_bytes = fs::read(&path_to_drv_file).expect("unable to read .drv"); let parsed_drv = Derivation::from_aterm_bytes(&aterm_bytes).expect("must succeed"); // For where we're able to load JSON fixtures, parse them and compare the structs. // For where we're not, compare the bytes manually. - if path_to_drv_file.ends_with("cp1252.drv") || path_to_drv_file.ends_with("latin1.drv") { + if path_to_drv_file.file_name().is_some_and(|s| { + s.as_encoded_bytes().ends_with(b"cp1252.drv") + || s.as_encoded_bytes().ends_with(b"latin1.drv") + }) { assert_eq!( &[0xc5, 0xc4, 0xd6][..], parsed_drv.environment.get("chars").unwrap(), "expected bytes to match", ); } else { - let json_bytes = read_file(&format!("{}.json", path_to_drv_file)); + let json_bytes = + fs::read(path_to_drv_file.with_extension("drv.json")).expect("unable to read JSON"); let fixture_derivation: Derivation = serde_json::from_slice(&json_bytes).expect("JSON was not well-formatted"); @@ -112,7 +109,8 @@ fn from_aterm_bytes(path_to_drv_file: &str) { #[test] fn from_aterm_bytes_duplicate_map_key() { - let buf: Vec<u8> = read_file(&format!("{}/{}", RESOURCES_PATHS, "duplicate.drv")).into(); + let buf: Vec<u8> = + fs::read(format!("{}/{}", RESOURCES_PATHS, "duplicate.drv")).expect("unable to read .drv"); let err = Derivation::from_aterm_bytes(&buf).expect_err("must fail"); @@ -130,26 +128,31 @@ fn from_aterm_bytes_duplicate_map_key() { /// Ensure the parser detects and fails in this case. #[test] fn from_aterm_bytes_trailer() { - let mut buf: Vec<u8> = read_file(&format!( + let mut buf: Vec<u8> = fs::read(format!( "{}/ok/{}", RESOURCES_PATHS, "0hm2f1psjpcwg8fijsmr4wwxrx59s092-bar.drv" )) - .into(); + .expect("unable to read .drv"); buf.push(0x00); Derivation::from_aterm_bytes(&buf).expect_err("must fail"); } -#[test_case("bar","0hm2f1psjpcwg8fijsmr4wwxrx59s092-bar.drv"; "fixed_sha256")] -#[test_case("foo", "4wvvbi4jwn0prsdxb7vs673qa5h9gr7x-foo.drv"; "simple-sha256")] -#[test_case("bar", "ss2p4wmxijn652haqyd7dckxwl4c7hxx-bar.drv"; "fixed-sha1")] -#[test_case("foo", "ch49594n9avinrf8ip0aslidkc4lxkqv-foo.drv"; "simple-sha1")] -#[test_case("has-multi-out", "h32dahq0bx5rp1krcdx3a53asj21jvhk-has-multi-out.drv"; "multiple-outputs")] -#[test_case("structured-attrs", "9lj1lkjm2ag622mh4h9rpy6j607an8g2-structured-attrs.drv"; "structured-attrs")] -#[test_case("unicode", "52a9id8hx688hvlnz4d1n25ml1jdykz0-unicode.drv"; "unicode")] -fn derivation_path(name: &str, expected_path: &str) { - let json_bytes = read_file(&format!("{}/ok/{}.json", RESOURCES_PATHS, expected_path)); +#[rstest] +#[case::fixed_sha256("bar", "0hm2f1psjpcwg8fijsmr4wwxrx59s092-bar.drv")] +#[case::simple_sha256("foo", "4wvvbi4jwn0prsdxb7vs673qa5h9gr7x-foo.drv")] +#[case::fixed_sha1("bar", "ss2p4wmxijn652haqyd7dckxwl4c7hxx-bar.drv")] +#[case::simple_sha1("foo", "ch49594n9avinrf8ip0aslidkc4lxkqv-foo.drv")] +#[case::multiple_outputs("has-multi-out", "h32dahq0bx5rp1krcdx3a53asj21jvhk-has-multi-out.drv")] +#[case::structured_attrs( + "structured-attrs", + "9lj1lkjm2ag622mh4h9rpy6j607an8g2-structured-attrs.drv" +)] +#[case::unicode("unicode", "52a9id8hx688hvlnz4d1n25ml1jdykz0-unicode.drv")] +fn derivation_path(#[case] name: &str, #[case] expected_path: &str) { + let json_bytes = fs::read(format!("{}/ok/{}.json", RESOURCES_PATHS, expected_path)) + .expect("unable to read JSON"); let derivation: Derivation = serde_json::from_slice(&json_bytes).expect("JSON was not well-formatted"); @@ -161,7 +164,7 @@ fn derivation_path(name: &str, expected_path: &str) { /// This trims all output paths from a Derivation struct, /// by setting outputs[$outputName].path and environment[$outputName] to the empty string. -fn derivation_with_trimmed_output_paths(derivation: &Derivation) -> Derivation { +fn derivation_without_output_paths(derivation: &Derivation) -> Derivation { let mut trimmed_env = derivation.environment.clone(); let mut trimmed_outputs = derivation.outputs.clone(); @@ -185,43 +188,49 @@ fn derivation_with_trimmed_output_paths(derivation: &Derivation) -> Derivation { } } -#[test_case("0hm2f1psjpcwg8fijsmr4wwxrx59s092-bar.drv", hex!("724f3e3634fce4cbbbd3483287b8798588e80280660b9a63fd13a1bc90485b33"); "fixed_sha256")] -#[test_case("ss2p4wmxijn652haqyd7dckxwl4c7hxx-bar.drv", hex!("c79aebd0ce3269393d4a1fde2cbd1d975d879b40f0bf40a48f550edc107fd5df");"fixed-sha1")] -fn derivation_or_fod_hash(drv_path: &str, expected_digest: [u8; 32]) { +#[rstest] +#[case::fixed_sha256("0hm2f1psjpcwg8fijsmr4wwxrx59s092-bar.drv", hex!("724f3e3634fce4cbbbd3483287b8798588e80280660b9a63fd13a1bc90485b33"))] +#[case::fixed_sha1("ss2p4wmxijn652haqyd7dckxwl4c7hxx-bar.drv", hex!("c79aebd0ce3269393d4a1fde2cbd1d975d879b40f0bf40a48f550edc107fd5df"))] +fn hash_derivation_modulo_fixed(#[case] drv_path: &str, #[case] expected_digest: [u8; 32]) { // read in the fixture - let json_bytes = read_file(&format!("{}/ok/{}.json", RESOURCES_PATHS, drv_path)); + let json_bytes = + fs::read(format!("{}/ok/{}.json", RESOURCES_PATHS, drv_path)).expect("unable to read JSON"); let drv: Derivation = serde_json::from_slice(&json_bytes).expect("must deserialize"); - let actual = drv.derivation_or_fod_hash(|_| panic!("must not be called")); + let actual = drv.hash_derivation_modulo(|_| panic!("must not be called")); assert_eq!(expected_digest, actual); } /// This reads a Derivation (in A-Term), trims out all fields containing /// calculated output paths, then triggers the output path calculation and /// compares the struct to match what was originally read in. -#[test_case("bar","0hm2f1psjpcwg8fijsmr4wwxrx59s092-bar.drv"; "fixed_sha256")] -#[test_case("foo", "4wvvbi4jwn0prsdxb7vs673qa5h9gr7x-foo.drv"; "simple-sha256")] -#[test_case("bar", "ss2p4wmxijn652haqyd7dckxwl4c7hxx-bar.drv"; "fixed-sha1")] -#[test_case("foo", "ch49594n9avinrf8ip0aslidkc4lxkqv-foo.drv"; "simple-sha1")] -#[test_case("has-multi-out", "h32dahq0bx5rp1krcdx3a53asj21jvhk-has-multi-out.drv"; "multiple-outputs")] -#[test_case("structured-attrs", "9lj1lkjm2ag622mh4h9rpy6j607an8g2-structured-attrs.drv"; "structured-attrs")] -#[test_case("unicode", "52a9id8hx688hvlnz4d1n25ml1jdykz0-unicode.drv"; "unicode")] -#[test_case("cp1252", "m1vfixn8iprlf0v9abmlrz7mjw1xj8kp-cp1252.drv"; "cp1252")] -#[test_case("latin1", "x6p0hg79i3wg0kkv7699935f7rrj9jf3-latin1.drv"; "latin1")] -fn output_paths(name: &str, drv_path_str: &str) { +#[rstest] +#[case::fixed_sha256("bar", "0hm2f1psjpcwg8fijsmr4wwxrx59s092-bar.drv")] +#[case::simple_sha256("foo", "4wvvbi4jwn0prsdxb7vs673qa5h9gr7x-foo.drv")] +#[case::fixed_sha1("bar", "ss2p4wmxijn652haqyd7dckxwl4c7hxx-bar.drv")] +#[case::simple_sha1("foo", "ch49594n9avinrf8ip0aslidkc4lxkqv-foo.drv")] +#[case::multiple_outputs("has-multi-out", "h32dahq0bx5rp1krcdx3a53asj21jvhk-has-multi-out.drv")] +#[case::structured_attrs( + "structured-attrs", + "9lj1lkjm2ag622mh4h9rpy6j607an8g2-structured-attrs.drv" +)] +#[case::unicode("unicode", "52a9id8hx688hvlnz4d1n25ml1jdykz0-unicode.drv")] +#[case::cp1252("cp1252", "m1vfixn8iprlf0v9abmlrz7mjw1xj8kp-cp1252.drv")] +#[case::latin1("latin1", "x6p0hg79i3wg0kkv7699935f7rrj9jf3-latin1.drv")] +fn output_paths(#[case] name: &str, #[case] drv_path_str: &str) { // read in the derivation let expected_derivation = Derivation::from_aterm_bytes( - read_file(&format!("{}/ok/{}", RESOURCES_PATHS, drv_path_str)).as_ref(), + &fs::read(format!("{}/ok/{}", RESOURCES_PATHS, drv_path_str)).expect("unable to read .drv"), ) .expect("must succeed"); - // create a version with trimmed output paths, simulating we constructed - // the struct. - let mut derivation = derivation_with_trimmed_output_paths(&expected_derivation); + // create a version without output paths, simulating we constructed the + // struct. + let mut derivation = derivation_without_output_paths(&expected_derivation); - // calculate the derivation_or_fod_hash of derivation + // calculate the hash_derivation_modulo of Derivation // We don't expect the lookup function to be called for most derivations. - let calculated_derivation_or_fod_hash = derivation.derivation_or_fod_hash(|parent_drv_path| { + let actual_hash_derivation_modulo = derivation.hash_derivation_modulo(|parent_drv_path| { // 4wvvbi4jwn0prsdxb7vs673qa5h9gr7x-foo.drv may lookup /nix/store/0hm2f1psjpcwg8fijsmr4wwxrx59s092-bar.drv // ch49594n9avinrf8ip0aslidkc4lxkqv-foo.drv may lookup /nix/store/ss2p4wmxijn652haqyd7dckxwl4c7hxx-bar.drv if name == "foo" @@ -234,20 +243,21 @@ fn output_paths(name: &str, drv_path_str: &str) { // drv_name, and calculating its drv replacement (on the non-stripped version) // In a real-world scenario you would have already done this during construction. - let json_bytes = read_file(&format!( + let json_bytes = fs::read(format!( "{}/ok/{}.json", RESOURCES_PATHS, Path::new(&parent_drv_path.to_string()) .file_name() .unwrap() .to_string_lossy() - )); + )) + .expect("unable to read JSON"); let drv: Derivation = serde_json::from_slice(&json_bytes).expect("must deserialize"); - // calculate derivation_or_fod_hash for each parent. + // calculate hash_derivation_modulo for each parent. // This may not trigger subsequent requests, as both parents are FOD. - drv.derivation_or_fod_hash(|_| panic!("must not lookup")) + drv.hash_derivation_modulo(|_| panic!("must not lookup")) } else { // we only expect this to be called in the "foo" testcase, for the "bar derivations" panic!("may only be called for foo testcase on bar derivations"); @@ -255,7 +265,7 @@ fn output_paths(name: &str, drv_path_str: &str) { }); derivation - .calculate_output_paths(name, &calculated_derivation_or_fod_hash) + .calculate_output_paths(name, &actual_hash_derivation_modulo) .unwrap(); // The derivation should now look like it was before @@ -333,15 +343,16 @@ fn output_path_construction() { // calculate bar output paths let bar_calc_result = bar_drv.calculate_output_paths( "bar", - &bar_drv.derivation_or_fod_hash(|_| panic!("is FOD, should not lookup")), + &bar_drv.hash_derivation_modulo(|_| panic!("is FOD, should not lookup")), ); assert!(bar_calc_result.is_ok()); // ensure it matches our bar fixture - let bar_json_bytes = read_file(&format!( + let bar_json_bytes = fs::read(format!( "{}/ok/{}.json", RESOURCES_PATHS, "0hm2f1psjpcwg8fijsmr4wwxrx59s092-bar.drv" - )); + )) + .expect("unable to read JSON"); let bar_drv_expected: Derivation = serde_json::from_slice(&bar_json_bytes).expect("must deserialize"); assert_eq!(bar_drv_expected, bar_drv); @@ -349,8 +360,8 @@ fn output_path_construction() { // now construct foo, which requires bar_drv // Note how we refer to the output path, drv name and replacement_str (with calculated output paths) of bar. let bar_output_path = &bar_drv.outputs.get("out").expect("must exist").path; - let bar_drv_derivation_or_fod_hash = - bar_drv.derivation_or_fod_hash(|_| panic!("is FOD, should not lookup")); + let bar_drv_hash_derivation_modulo = + bar_drv.hash_derivation_modulo(|_| panic!("is FOD, should not lookup")); let bar_drv_path = bar_drv .calculate_derivation_path("bar") @@ -397,20 +408,21 @@ fn output_path_construction() { // calculate foo output paths let foo_calc_result = foo_drv.calculate_output_paths( "foo", - &foo_drv.derivation_or_fod_hash(|drv_path| { + &foo_drv.hash_derivation_modulo(|drv_path| { if drv_path.to_string() != "0hm2f1psjpcwg8fijsmr4wwxrx59s092-bar.drv" { panic!("lookup called with unexpected drv_path: {}", drv_path); } - bar_drv_derivation_or_fod_hash + bar_drv_hash_derivation_modulo }), ); assert!(foo_calc_result.is_ok()); // ensure it matches our foo fixture - let foo_json_bytes = read_file(&format!( + let foo_json_bytes = fs::read(format!( "{}/ok/{}.json", RESOURCES_PATHS, "4wvvbi4jwn0prsdxb7vs673qa5h9gr7x-foo.drv", - )); + )) + .expect("unable to read JSON"); let foo_drv_expected: Derivation = serde_json::from_slice(&foo_json_bytes).expect("must deserialize"); assert_eq!(foo_drv_expected, foo_drv); diff --git a/tvix/nix-compat/src/lib.rs b/tvix/nix-compat/src/lib.rs index d2849cbeae..a71ede3eec 100644 --- a/tvix/nix-compat/src/lib.rs +++ b/tvix/nix-compat/src/lib.rs @@ -4,5 +4,15 @@ pub mod nar; pub mod narinfo; pub mod nixbase32; pub mod nixhash; +pub mod path_info; pub mod store_path; + +#[cfg(feature = "wire")] pub mod wire; + +#[cfg(feature = "wire")] +mod nix_daemon; +#[cfg(feature = "wire")] +pub use nix_daemon::worker_protocol; +#[cfg(feature = "wire")] +pub use nix_daemon::ProtocolVersion; diff --git a/tvix/nix-compat/src/nar/mod.rs b/tvix/nix-compat/src/nar/mod.rs index 058977f4fc..c678d26ffb 100644 --- a/tvix/nix-compat/src/nar/mod.rs +++ b/tvix/nix-compat/src/nar/mod.rs @@ -1,4 +1,4 @@ -mod wire; +pub(crate) mod wire; pub mod reader; pub mod writer; diff --git a/tvix/nix-compat/src/nar/reader/async/mod.rs b/tvix/nix-compat/src/nar/reader/async/mod.rs new file mode 100644 index 0000000000..0808fba38c --- /dev/null +++ b/tvix/nix-compat/src/nar/reader/async/mod.rs @@ -0,0 +1,173 @@ +use std::{ + mem::MaybeUninit, + pin::Pin, + task::{self, Poll}, +}; + +use tokio::io::{self, AsyncBufRead, AsyncRead, ErrorKind::InvalidData}; + +// Required reading for understanding this module. +use crate::{ + nar::{self, wire::PadPar}, + wire::{self, BytesReader}, +}; + +mod read; +#[cfg(test)] +mod test; + +pub type Reader<'a> = dyn AsyncBufRead + Unpin + Send + 'a; + +/// Start reading a NAR file from `reader`. +pub async fn open<'a, 'r>(reader: &'a mut Reader<'r>) -> io::Result<Node<'a, 'r>> { + read::token(reader, &nar::wire::TOK_NAR).await?; + Node::new(reader).await +} + +pub enum Node<'a, 'r: 'a> { + Symlink { + target: Vec<u8>, + }, + File { + executable: bool, + reader: FileReader<'a, 'r>, + }, + Directory(DirReader<'a, 'r>), +} + +impl<'a, 'r: 'a> Node<'a, 'r> { + /// Start reading a [Node], matching the next [wire::Node]. + /// + /// Reading the terminating [wire::TOK_PAR] is done immediately for [Node::Symlink], + /// but is otherwise left to [DirReader] or [BytesReader]. + async fn new(reader: &'a mut Reader<'r>) -> io::Result<Self> { + Ok(match read::tag(reader).await? { + nar::wire::Node::Sym => { + let target = wire::read_bytes(reader, 1..=nar::wire::MAX_TARGET_LEN).await?; + + if target.contains(&0) { + return Err(InvalidData.into()); + } + + read::token(reader, &nar::wire::TOK_PAR).await?; + + Node::Symlink { target } + } + tag @ (nar::wire::Node::Reg | nar::wire::Node::Exe) => Node::File { + executable: tag == nar::wire::Node::Exe, + reader: FileReader { + inner: BytesReader::new_internal(reader, ..).await?, + }, + }, + nar::wire::Node::Dir => Node::Directory(DirReader::new(reader)), + }) + } +} + +/// File contents, readable through the [AsyncRead] trait. +/// +/// It comes with some caveats: +/// * You must always read the entire file, unless you intend to abandon the entire archive reader. +/// * You must abandon the entire archive reader upon the first error. +/// +/// It's fine to read exactly `reader.len()` bytes without ever seeing an explicit EOF. +pub struct FileReader<'a, 'r> { + inner: BytesReader<&'a mut Reader<'r>, PadPar>, +} + +impl<'a, 'r> FileReader<'a, 'r> { + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn len(&self) -> u64 { + self.inner.len() + } +} + +impl<'a, 'r> AsyncRead for FileReader<'a, 'r> { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut task::Context, + buf: &mut io::ReadBuf, + ) -> Poll<io::Result<()>> { + Pin::new(&mut self.get_mut().inner).poll_read(cx, buf) + } +} + +impl<'a, 'r> AsyncBufRead for FileReader<'a, 'r> { + fn poll_fill_buf(self: Pin<&mut Self>, cx: &mut task::Context) -> Poll<io::Result<&[u8]>> { + Pin::new(&mut self.get_mut().inner).poll_fill_buf(cx) + } + + fn consume(self: Pin<&mut Self>, amt: usize) { + Pin::new(&mut self.get_mut().inner).consume(amt) + } +} + +/// A directory iterator, yielding a sequence of [Node]s. +/// It must be fully consumed before reading further from the [DirReader] that produced it, if any. +pub struct DirReader<'a, 'r> { + reader: &'a mut Reader<'r>, + /// Previous directory entry name. + /// We have to hang onto this to enforce name monotonicity. + prev_name: Vec<u8>, +} + +pub struct Entry<'a, 'r> { + pub name: &'a [u8], + pub node: Node<'a, 'r>, +} + +impl<'a, 'r> DirReader<'a, 'r> { + fn new(reader: &'a mut Reader<'r>) -> Self { + Self { + reader, + prev_name: vec![], + } + } + + /// Read the next [Entry] from the directory. + /// + /// We explicitly don't implement [Iterator], since treating this as + /// a regular Rust iterator will surely lead you astray. + /// + /// * You must always consume the entire iterator, unless you abandon the entire archive reader. + /// * You must abandon the entire archive reader on the first error. + /// * You must abandon the directory reader upon the first [None]. + /// * Even if you know the amount of elements up front, you must keep reading until you encounter [None]. + pub async fn next(&mut self) -> io::Result<Option<Entry<'_, 'r>>> { + // COME FROM the previous iteration: if we've already read an entry, + // read its terminating TOK_PAR here. + if !self.prev_name.is_empty() { + read::token(self.reader, &nar::wire::TOK_PAR).await?; + } + + if let nar::wire::Entry::None = read::tag(self.reader).await? { + return Ok(None); + } + + let mut name = [MaybeUninit::uninit(); nar::wire::MAX_NAME_LEN + 1]; + let name = + wire::read_bytes_buf(self.reader, &mut name, 1..=nar::wire::MAX_NAME_LEN).await?; + + if name.contains(&0) || name.contains(&b'/') || name == b"." || name == b".." { + return Err(InvalidData.into()); + } + + // Enforce strict monotonicity of directory entry names. + if &self.prev_name[..] >= name { + return Err(InvalidData.into()); + } + + self.prev_name.clear(); + self.prev_name.extend_from_slice(name); + + read::token(self.reader, &nar::wire::TOK_NOD).await?; + + Ok(Some(Entry { + name: &self.prev_name, + node: Node::new(self.reader).await?, + })) + } +} diff --git a/tvix/nix-compat/src/nar/reader/async/read.rs b/tvix/nix-compat/src/nar/reader/async/read.rs new file mode 100644 index 0000000000..2adf894922 --- /dev/null +++ b/tvix/nix-compat/src/nar/reader/async/read.rs @@ -0,0 +1,69 @@ +use tokio::io::{ + self, AsyncReadExt, + ErrorKind::{InvalidData, UnexpectedEof}, +}; + +use crate::nar::wire::Tag; + +use super::Reader; + +/// Consume a known token from the reader. +pub async fn token<const N: usize>(reader: &mut Reader<'_>, token: &[u8; N]) -> io::Result<()> { + let mut buf = [0u8; N]; + + // This implements something similar to [AsyncReadExt::read_exact], but verifies that + // the input data matches the token while we read it. These two slices respectively + // represent the remaining token to be verified, and the remaining input buffer. + let mut token = &token[..]; + let mut buf = &mut buf[..]; + + while !token.is_empty() { + match reader.read(buf).await? { + 0 => { + return Err(UnexpectedEof.into()); + } + n => { + let (t, b); + (t, token) = token.split_at(n); + (b, buf) = buf.split_at_mut(n); + + if t != b { + return Err(InvalidData.into()); + } + } + } + } + + Ok(()) +} + +/// Consume a [Tag] from the reader. +pub async fn tag<T: Tag>(reader: &mut Reader<'_>) -> io::Result<T> { + let mut buf = T::make_buf(); + let buf = buf.as_mut(); + + // first read the known minimum lengthโฆ + reader.read_exact(&mut buf[..T::MIN]).await?; + + // then decide which tag we're expecting + let tag = T::from_u8(buf[T::OFF]).ok_or(InvalidData)?; + let (head, tail) = tag.as_bytes().split_at(T::MIN); + + // make sure what we've read so far is valid + if buf[..T::MIN] != *head { + return Err(InvalidData.into()); + } + + // โฆthen read the rest, if any + if !tail.is_empty() { + let rest = tail.len(); + reader.read_exact(&mut buf[..rest]).await?; + + // and make sure it's what we expect + if buf[..rest] != *tail { + return Err(InvalidData.into()); + } + } + + Ok(tag) +} diff --git a/tvix/nix-compat/src/nar/reader/async/test.rs b/tvix/nix-compat/src/nar/reader/async/test.rs new file mode 100644 index 0000000000..7bc1f8942f --- /dev/null +++ b/tvix/nix-compat/src/nar/reader/async/test.rs @@ -0,0 +1,310 @@ +use tokio::io::AsyncReadExt; + +mod nar { + pub use crate::nar::reader::r#async as reader; +} + +#[tokio::test] +async fn symlink() { + let mut f = std::io::Cursor::new(include_bytes!("../../tests/symlink.nar")); + let node = nar::reader::open(&mut f).await.unwrap(); + + match node { + nar::reader::Node::Symlink { target } => { + assert_eq!( + &b"/nix/store/somewhereelse"[..], + &target, + "target must match" + ); + } + _ => panic!("unexpected type"), + } +} + +#[tokio::test] +async fn file() { + let mut f = std::io::Cursor::new(include_bytes!("../../tests/helloworld.nar")); + let node = nar::reader::open(&mut f).await.unwrap(); + + match node { + nar::reader::Node::File { + executable, + mut reader, + } => { + assert!(!executable); + let mut buf = vec![]; + reader + .read_to_end(&mut buf) + .await + .expect("read must succeed"); + assert_eq!(&b"Hello World!"[..], &buf); + } + _ => panic!("unexpected type"), + } +} + +#[tokio::test] +async fn complicated() { + let mut f = std::io::Cursor::new(include_bytes!("../../tests/complicated.nar")); + let node = nar::reader::open(&mut f).await.unwrap(); + + match node { + nar::reader::Node::Directory(mut dir_reader) => { + // first entry is .keep, an empty regular file. + must_read_file( + ".keep", + dir_reader + .next() + .await + .expect("next must succeed") + .expect("must be some"), + ) + .await; + + // second entry is aa, a symlink to /nix/store/somewhereelse + must_be_symlink( + "aa", + "/nix/store/somewhereelse", + dir_reader + .next() + .await + .expect("next must be some") + .expect("must be some"), + ); + + { + // third entry is a directory called "keep" + let entry = dir_reader + .next() + .await + .expect("next must be some") + .expect("must be some"); + + assert_eq!(b"keep", entry.name); + + match entry.node { + nar::reader::Node::Directory(mut subdir_reader) => { + { + // first entry is .keep, an empty regular file. + let entry = subdir_reader + .next() + .await + .expect("next must succeed") + .expect("must be some"); + + must_read_file(".keep", entry).await; + } + + // we must read the None + assert!( + subdir_reader + .next() + .await + .expect("next must succeed") + .is_none(), + "keep directory contains only .keep" + ); + } + _ => panic!("unexpected type for keep/.keep"), + } + }; + + // reading more entries yields None (and we actually must read until this) + assert!(dir_reader.next().await.expect("must succeed").is_none()); + } + _ => panic!("unexpected type"), + } +} + +#[tokio::test] +#[should_panic] +#[ignore = "TODO: async poisoning"] +async fn file_read_abandoned() { + let mut f = std::io::Cursor::new(include_bytes!("../../tests/complicated.nar")); + let node = nar::reader::open(&mut f).await.unwrap(); + + match node { + nar::reader::Node::Directory(mut dir_reader) => { + // first entry is .keep, an empty regular file. + { + let entry = dir_reader + .next() + .await + .expect("next must succeed") + .expect("must be some"); + + assert_eq!(b".keep", entry.name); + // don't bother to finish reading it. + }; + + // this should panic (not return an error), because we are meant to abandon the archive reader now. + assert!(dir_reader.next().await.expect("must succeed").is_none()); + } + _ => panic!("unexpected type"), + } +} + +#[tokio::test] +#[should_panic] +#[ignore = "TODO: async poisoning"] +async fn dir_read_abandoned() { + let mut f = std::io::Cursor::new(include_bytes!("../../tests/complicated.nar")); + let node = nar::reader::open(&mut f).await.unwrap(); + + match node { + nar::reader::Node::Directory(mut dir_reader) => { + // first entry is .keep, an empty regular file. + must_read_file( + ".keep", + dir_reader + .next() + .await + .expect("next must succeed") + .expect("must be some"), + ) + .await; + + // second entry is aa, a symlink to /nix/store/somewhereelse + must_be_symlink( + "aa", + "/nix/store/somewhereelse", + dir_reader + .next() + .await + .expect("next must be some") + .expect("must be some"), + ); + + { + // third entry is a directory called "keep" + let entry = dir_reader + .next() + .await + .expect("next must be some") + .expect("must be some"); + + assert_eq!(b"keep", entry.name); + + match entry.node { + nar::reader::Node::Directory(_) => { + // don't finish using it, which poisons the archive reader + } + _ => panic!("unexpected type for keep/.keep"), + } + }; + + // this should panic, because we didn't finish reading the child subdirectory + assert!(dir_reader.next().await.expect("must succeed").is_none()); + } + _ => panic!("unexpected type"), + } +} + +#[tokio::test] +#[should_panic] +#[ignore = "TODO: async poisoning"] +async fn dir_read_after_none() { + let mut f = std::io::Cursor::new(include_bytes!("../../tests/complicated.nar")); + let node = nar::reader::open(&mut f).await.unwrap(); + + match node { + nar::reader::Node::Directory(mut dir_reader) => { + // first entry is .keep, an empty regular file. + must_read_file( + ".keep", + dir_reader + .next() + .await + .expect("next must succeed") + .expect("must be some"), + ) + .await; + + // second entry is aa, a symlink to /nix/store/somewhereelse + must_be_symlink( + "aa", + "/nix/store/somewhereelse", + dir_reader + .next() + .await + .expect("next must be some") + .expect("must be some"), + ); + + { + // third entry is a directory called "keep" + let entry = dir_reader + .next() + .await + .expect("next must be some") + .expect("must be some"); + + assert_eq!(b"keep", entry.name); + + match entry.node { + nar::reader::Node::Directory(mut subdir_reader) => { + // first entry is .keep, an empty regular file. + must_read_file( + ".keep", + subdir_reader + .next() + .await + .expect("next must succeed") + .expect("must be some"), + ) + .await; + + // we must read the None + assert!( + subdir_reader + .next() + .await + .expect("next must succeed") + .is_none(), + "keep directory contains only .keep" + ); + } + _ => panic!("unexpected type for keep/.keep"), + } + }; + + // reading more entries yields None (and we actually must read until this) + assert!(dir_reader.next().await.expect("must succeed").is_none()); + + // this should panic, because we already got a none so we're meant to stop. + dir_reader.next().await.unwrap(); + unreachable!() + } + _ => panic!("unexpected type"), + } +} + +async fn must_read_file(name: &'static str, entry: nar::reader::Entry<'_, '_>) { + assert_eq!(name.as_bytes(), entry.name); + + match entry.node { + nar::reader::Node::File { + executable, + mut reader, + } => { + assert!(!executable); + assert_eq!(reader.read(&mut [0]).await.unwrap(), 0); + } + _ => panic!("unexpected type for {}", name), + } +} + +fn must_be_symlink( + name: &'static str, + exp_target: &'static str, + entry: nar::reader::Entry<'_, '_>, +) { + assert_eq!(name.as_bytes(), entry.name); + + match entry.node { + nar::reader::Node::Symlink { target } => { + assert_eq!(exp_target.as_bytes(), &target); + } + _ => panic!("unexpected type for {}", name), + } +} diff --git a/tvix/nix-compat/src/nar/reader/mod.rs b/tvix/nix-compat/src/nar/reader/mod.rs index fa7ddc77f9..9e9237ead3 100644 --- a/tvix/nix-compat/src/nar/reader/mod.rs +++ b/tvix/nix-compat/src/nar/reader/mod.rs @@ -10,19 +10,50 @@ use std::io::{ Read, Write, }; +#[cfg(not(debug_assertions))] +use std::marker::PhantomData; + // Required reading for understanding this module. use crate::nar::wire; +#[cfg(feature = "async")] +pub mod r#async; + mod read; #[cfg(test)] mod test; pub type Reader<'a> = dyn BufRead + Send + 'a; +struct ArchiveReader<'a, 'r> { + inner: &'a mut Reader<'r>, + + /// In debug mode, also track when we need to abandon this archive reader. + /// The archive reader must be abandoned when: + /// * An error is encountered at any point + /// * A file or directory reader is dropped before being read entirely. + /// All of these checks vanish in release mode. + status: ArchiveReaderStatus<'a>, +} + +macro_rules! try_or_poison { + ($it:expr, $ex:expr) => { + match $ex { + Ok(x) => x, + Err(e) => { + $it.status.poison(); + return Err(e.into()); + } + } + }; +} /// Start reading a NAR file from `reader`. pub fn open<'a, 'r>(reader: &'a mut Reader<'r>) -> io::Result<Node<'a, 'r>> { read::token(reader, &wire::TOK_NAR)?; - Node::new(reader) + Node::new(ArchiveReader { + inner: reader, + status: ArchiveReaderStatus::top(), + }) } pub enum Node<'a, 'r> { @@ -41,21 +72,24 @@ impl<'a, 'r> Node<'a, 'r> { /// /// Reading the terminating [wire::TOK_PAR] is done immediately for [Node::Symlink], /// but is otherwise left to [DirReader] or [FileReader]. - fn new(reader: &'a mut Reader<'r>) -> io::Result<Self> { - Ok(match read::tag(reader)? { + fn new(mut reader: ArchiveReader<'a, 'r>) -> io::Result<Self> { + Ok(match read::tag(reader.inner)? { wire::Node::Sym => { - let target = read::bytes(reader, wire::MAX_TARGET_LEN)?; + let target = + try_or_poison!(reader, read::bytes(reader.inner, wire::MAX_TARGET_LEN)); if target.is_empty() || target.contains(&0) { + reader.status.poison(); return Err(InvalidData.into()); } - read::token(reader, &wire::TOK_PAR)?; + try_or_poison!(reader, read::token(reader.inner, &wire::TOK_PAR)); + reader.status.ready_parent(); // Immediately allow reading from parent again Node::Symlink { target } } tag @ (wire::Node::Reg | wire::Node::Exe) => { - let len = read::u64(reader)?; + let len = try_or_poison!(&mut reader, read::u64(reader.inner)); Node::File { executable: tag == wire::Node::Exe, @@ -74,10 +108,8 @@ impl<'a, 'r> Node<'a, 'r> { /// * You must abandon the entire archive reader upon the first error. /// /// It's fine to read exactly `reader.len()` bytes without ever seeing an explicit EOF. -/// -/// TODO(edef): enforce these in `#[cfg(debug_assertions)]` pub struct FileReader<'a, 'r> { - reader: &'a mut Reader<'r>, + reader: ArchiveReader<'a, 'r>, len: u64, /// Truncated original file length for padding computation. /// We only care about the 3 least significant bits; semantically, this is a u3. @@ -87,12 +119,13 @@ pub struct FileReader<'a, 'r> { impl<'a, 'r> FileReader<'a, 'r> { /// Instantiate a new reader, starting after [wire::TOK_REG] or [wire::TOK_EXE]. /// We handle the terminating [wire::TOK_PAR] on semantic EOF. - fn new(reader: &'a mut Reader<'r>, len: u64) -> io::Result<Self> { + fn new(mut reader: ArchiveReader<'a, 'r>, len: u64) -> io::Result<Self> { // For zero-length files, we have to read the terminating TOK_PAR // immediately, since FileReader::read may never be called; we've // already reached semantic EOF by definition. if len == 0 { - read::token(reader, &wire::TOK_PAR)?; + read::token(reader.inner, &wire::TOK_PAR)?; + reader.status.ready_parent(); } Ok(Self { @@ -121,9 +154,12 @@ impl FileReader<'_, '_> { return Ok(&[]); } - let mut buf = self.reader.fill_buf()?; + self.reader.check_correct(); + + let mut buf = try_or_poison!(self.reader, self.reader.inner.fill_buf()); if buf.is_empty() { + self.reader.status.poison(); return Err(UnexpectedEof.into()); } @@ -141,12 +177,14 @@ impl FileReader<'_, '_> { return Ok(()); } + self.reader.check_correct(); + self.len = self .len .checked_sub(n as u64) .expect("consumed bytes past EOF"); - self.reader.consume(n); + self.reader.inner.consume(n); if self.is_empty() { self.finish()?; @@ -159,7 +197,7 @@ impl FileReader<'_, '_> { pub fn copy(&mut self, mut dst: impl Write) -> io::Result<()> { while !self.is_empty() { let buf = self.fill_buf()?; - let n = dst.write(buf)?; + let n = try_or_poison!(self.reader, dst.write(buf)); self.consume(n)?; } @@ -173,14 +211,17 @@ impl Read for FileReader<'_, '_> { return Ok(0); } + self.reader.check_correct(); + if buf.len() as u64 > self.len { buf = &mut buf[..self.len as usize]; } - let n = self.reader.read(buf)?; + let n = try_or_poison!(self.reader, self.reader.inner.read(buf)); self.len -= n as u64; if n == 0 { + self.reader.status.poison(); return Err(UnexpectedEof.into()); } @@ -200,36 +241,42 @@ impl FileReader<'_, '_> { if pad != 0 { let mut buf = [0; 8]; - self.reader.read_exact(&mut buf[pad..])?; + try_or_poison!(self.reader, self.reader.inner.read_exact(&mut buf[pad..])); if buf != [0; 8] { + self.reader.status.poison(); return Err(InvalidData.into()); } } - read::token(self.reader, &wire::TOK_PAR) + try_or_poison!(self.reader, read::token(self.reader.inner, &wire::TOK_PAR)); + + // Done with reading this file, allow going back up the chain of readers + self.reader.status.ready_parent(); + + Ok(()) } } /// A directory iterator, yielding a sequence of [Node]s. /// It must be fully consumed before reading further from the [DirReader] that produced it, if any. pub struct DirReader<'a, 'r> { - reader: &'a mut Reader<'r>, + reader: ArchiveReader<'a, 'r>, /// Previous directory entry name. /// We have to hang onto this to enforce name monotonicity. - prev_name: Option<Vec<u8>>, + prev_name: Vec<u8>, } pub struct Entry<'a, 'r> { - pub name: Vec<u8>, + pub name: &'a [u8], pub node: Node<'a, 'r>, } impl<'a, 'r> DirReader<'a, 'r> { - fn new(reader: &'a mut Reader<'r>) -> Self { + fn new(reader: ArchiveReader<'a, 'r>) -> Self { Self { reader, - prev_name: None, + prev_name: vec![], } } @@ -242,23 +289,28 @@ impl<'a, 'r> DirReader<'a, 'r> { /// * You must abandon the entire archive reader on the first error. /// * You must abandon the directory reader upon the first [None]. /// * Even if you know the amount of elements up front, you must keep reading until you encounter [None]. - /// - /// TODO(edef): enforce these in `#[cfg(debug_assertions)]` #[allow(clippy::should_implement_trait)] - pub fn next(&mut self) -> io::Result<Option<Entry>> { + pub fn next(&mut self) -> io::Result<Option<Entry<'_, 'r>>> { + self.reader.check_correct(); + // COME FROM the previous iteration: if we've already read an entry, // read its terminating TOK_PAR here. - if self.prev_name.is_some() { - read::token(self.reader, &wire::TOK_PAR)?; + if !self.prev_name.is_empty() { + try_or_poison!(self.reader, read::token(self.reader.inner, &wire::TOK_PAR)); } // Determine if there are more entries to follow - if let wire::Entry::None = read::tag(self.reader)? { + if let wire::Entry::None = try_or_poison!(self.reader, read::tag(self.reader.inner)) { // We've reached the end of this directory. + self.reader.status.ready_parent(); return Ok(None); } - let name = read::bytes(self.reader, wire::MAX_NAME_LEN)?; + let mut name = [0; wire::MAX_NAME_LEN + 1]; + let name = try_or_poison!( + self.reader, + read::bytes_buf(self.reader.inner, &mut name, wire::MAX_NAME_LEN) + ); if name.is_empty() || name.contains(&0) @@ -266,28 +318,160 @@ impl<'a, 'r> DirReader<'a, 'r> { || name == b"." || name == b".." { + self.reader.status.poison(); return Err(InvalidData.into()); } // Enforce strict monotonicity of directory entry names. - match &mut self.prev_name { - None => { - self.prev_name = Some(name.clone()); + if &self.prev_name[..] >= name { + self.reader.status.poison(); + return Err(InvalidData.into()); + } + + self.prev_name.clear(); + self.prev_name.extend_from_slice(name); + + try_or_poison!(self.reader, read::token(self.reader.inner, &wire::TOK_NOD)); + + Ok(Some(Entry { + name: &self.prev_name, + // Don't need to worry about poisoning here: Node::new will do it for us if needed + node: Node::new(self.reader.child())?, + })) + } +} + +/// We use a stack of statuses to: +/// * Share poisoned state across all objects from the same underlying reader, +/// so we can check they are abandoned when an error occurs +/// * Make sure only the most recently created object is read from, and is fully exhausted +/// before anything it was created from is used again. +enum ArchiveReaderStatus<'a> { + #[cfg(not(debug_assertions))] + None(PhantomData<&'a ()>), + #[cfg(debug_assertions)] + StackTop { poisoned: bool, ready: bool }, + #[cfg(debug_assertions)] + StackChild { + poisoned: &'a mut bool, + parent_ready: &'a mut bool, + ready: bool, + }, +} + +impl ArchiveReaderStatus<'_> { + fn top() -> Self { + #[cfg(debug_assertions)] + { + ArchiveReaderStatus::StackTop { + poisoned: false, + ready: true, } - Some(prev_name) => { - if *prev_name >= name { - return Err(InvalidData.into()); - } + } - name[..].clone_into(prev_name); + #[cfg(not(debug_assertions))] + ArchiveReaderStatus::None(PhantomData) + } + + /// Poison all the objects sharing the same reader, to be used when an error occurs + fn poison(&mut self) { + match self { + #[cfg(not(debug_assertions))] + ArchiveReaderStatus::None(_) => {} + #[cfg(debug_assertions)] + ArchiveReaderStatus::StackTop { poisoned: x, .. } => *x = true, + #[cfg(debug_assertions)] + ArchiveReaderStatus::StackChild { poisoned: x, .. } => **x = true, + } + } + + /// Mark the parent as ready, allowing it to be used again and preventing this reference to the reader being used again. + fn ready_parent(&mut self) { + match self { + #[cfg(not(debug_assertions))] + ArchiveReaderStatus::None(_) => {} + #[cfg(debug_assertions)] + ArchiveReaderStatus::StackTop { ready, .. } => { + *ready = false; + } + #[cfg(debug_assertions)] + ArchiveReaderStatus::StackChild { + ready, + parent_ready, + .. + } => { + *ready = false; + **parent_ready = true; } + }; + } + + fn poisoned(&self) -> bool { + match self { + #[cfg(not(debug_assertions))] + ArchiveReaderStatus::None(_) => false, + #[cfg(debug_assertions)] + ArchiveReaderStatus::StackTop { poisoned, .. } => *poisoned, + #[cfg(debug_assertions)] + ArchiveReaderStatus::StackChild { poisoned, .. } => **poisoned, } + } - read::token(self.reader, &wire::TOK_NOD)?; + fn ready(&self) -> bool { + match self { + #[cfg(not(debug_assertions))] + ArchiveReaderStatus::None(_) => true, + #[cfg(debug_assertions)] + ArchiveReaderStatus::StackTop { ready, .. } => *ready, + #[cfg(debug_assertions)] + ArchiveReaderStatus::StackChild { ready, .. } => *ready, + } + } +} - Ok(Some(Entry { - name, - node: Node::new(&mut self.reader)?, - })) +impl<'a, 'r> ArchiveReader<'a, 'r> { + /// Create a new child reader from this one. + /// In debug mode, this reader will panic if called before the new child is exhausted / calls `ready_parent` + fn child(&mut self) -> ArchiveReader<'_, 'r> { + ArchiveReader { + inner: self.inner, + #[cfg(not(debug_assertions))] + status: ArchiveReaderStatus::None(PhantomData), + #[cfg(debug_assertions)] + status: match &mut self.status { + ArchiveReaderStatus::StackTop { poisoned, ready } => { + *ready = false; + ArchiveReaderStatus::StackChild { + poisoned, + parent_ready: ready, + ready: true, + } + } + ArchiveReaderStatus::StackChild { + poisoned, ready, .. + } => { + *ready = false; + ArchiveReaderStatus::StackChild { + poisoned, + parent_ready: ready, + ready: true, + } + } + }, + } + } + + /// Check the reader is in the correct status. + /// Only does anything when debug assertions are on. + #[inline(always)] + fn check_correct(&self) { + assert!( + !self.status.poisoned(), + "Archive reader used after it was meant to be abandoned!" + ); + assert!( + self.status.ready(), + "Non-ready archive reader used! (Should've been reading from something else)" + ); } } diff --git a/tvix/nix-compat/src/nar/reader/read.rs b/tvix/nix-compat/src/nar/reader/read.rs index 1ce1613764..9938581f2a 100644 --- a/tvix/nix-compat/src/nar/reader/read.rs +++ b/tvix/nix-compat/src/nar/reader/read.rs @@ -15,6 +15,38 @@ pub fn u64(reader: &mut Reader) -> io::Result<u64> { Ok(u64::from_le_bytes(buf)) } +/// Consume a byte string from the reader into a provided buffer, +/// returning the data bytes. +pub fn bytes_buf<'a, const N: usize>( + reader: &mut Reader, + buf: &'a mut [u8; N], + max_len: usize, +) -> io::Result<&'a [u8]> { + assert_eq!(N % 8, 0); + assert!(max_len <= N); + + // read the length, and reject excessively large values + let len = self::u64(reader)?; + if len > max_len as u64 { + return Err(InvalidData.into()); + } + // we know the length fits in a usize now + let len = len as usize; + + // read the data and padding into a buffer + let buf_len = (len + 7) & !7; + reader.read_exact(&mut buf[..buf_len])?; + + // verify that the padding is all zeroes + for &b in &buf[len..buf_len] { + if b != 0 { + return Err(InvalidData.into()); + } + } + + Ok(&buf[..len]) +} + /// Consume a byte string of up to `max_len` bytes from the reader. pub fn bytes(reader: &mut Reader, max_len: usize) -> io::Result<Vec<u8>> { assert!(max_len <= isize::MAX as usize); diff --git a/tvix/nix-compat/src/nar/reader/test.rs b/tvix/nix-compat/src/nar/reader/test.rs index fd0d6a9f5a..63e4fb289f 100644 --- a/tvix/nix-compat/src/nar/reader/test.rs +++ b/tvix/nix-compat/src/nar/reader/test.rs @@ -46,75 +46,233 @@ fn complicated() { match node { nar::reader::Node::Directory(mut dir_reader) => { // first entry is .keep, an empty regular file. - let entry = dir_reader - .next() - .expect("next must succeed") - .expect("must be some"); - - assert_eq!(&b".keep"[..], &entry.name); - - match entry.node { - nar::reader::Node::File { - executable, - mut reader, - } => { - assert!(!executable); - assert_eq!(reader.read(&mut [0]).unwrap(), 0); + must_read_file( + ".keep", + dir_reader + .next() + .expect("next must succeed") + .expect("must be some"), + ); + + // second entry is aa, a symlink to /nix/store/somewhereelse + must_be_symlink( + "aa", + "/nix/store/somewhereelse", + dir_reader + .next() + .expect("next must be some") + .expect("must be some"), + ); + + { + // third entry is a directory called "keep" + let entry = dir_reader + .next() + .expect("next must be some") + .expect("must be some"); + + assert_eq!(b"keep", entry.name); + + match entry.node { + nar::reader::Node::Directory(mut subdir_reader) => { + { + // first entry is .keep, an empty regular file. + let entry = subdir_reader + .next() + .expect("next must succeed") + .expect("must be some"); + + must_read_file(".keep", entry); + } + + // we must read the None + assert!( + subdir_reader.next().expect("next must succeed").is_none(), + "keep directory contains only .keep" + ); + } + _ => panic!("unexpected type for keep/.keep"), } - _ => panic!("unexpected type for .keep"), - } + }; + + // reading more entries yields None (and we actually must read until this) + assert!(dir_reader.next().expect("must succeed").is_none()); + } + _ => panic!("unexpected type"), + } +} + +#[test] +#[should_panic] +fn file_read_abandoned() { + let mut f = std::io::Cursor::new(include_bytes!("../tests/complicated.nar")); + let node = nar::reader::open(&mut f).unwrap(); + + match node { + nar::reader::Node::Directory(mut dir_reader) => { + // first entry is .keep, an empty regular file. + { + let entry = dir_reader + .next() + .expect("next must succeed") + .expect("must be some"); + + assert_eq!(b".keep", entry.name); + // don't bother to finish reading it. + }; + + // this should panic (not return an error), because we are meant to abandon the archive reader now. + assert!(dir_reader.next().expect("must succeed").is_none()); + } + _ => panic!("unexpected type"), + } +} + +#[test] +#[should_panic] +fn dir_read_abandoned() { + let mut f = std::io::Cursor::new(include_bytes!("../tests/complicated.nar")); + let node = nar::reader::open(&mut f).unwrap(); + + match node { + nar::reader::Node::Directory(mut dir_reader) => { + // first entry is .keep, an empty regular file. + must_read_file( + ".keep", + dir_reader + .next() + .expect("next must succeed") + .expect("must be some"), + ); // second entry is aa, a symlink to /nix/store/somewhereelse - let entry = dir_reader - .next() - .expect("next must be some") - .expect("must be some"); + must_be_symlink( + "aa", + "/nix/store/somewhereelse", + dir_reader + .next() + .expect("next must be some") + .expect("must be some"), + ); - assert_eq!(&b"aa"[..], &entry.name); + { + // third entry is a directory called "keep" + let entry = dir_reader + .next() + .expect("next must be some") + .expect("must be some"); - match entry.node { - nar::reader::Node::Symlink { target } => { - assert_eq!(&b"/nix/store/somewhereelse"[..], &target); + assert_eq!(b"keep", entry.name); + + match entry.node { + nar::reader::Node::Directory(_) => { + // don't finish using it, which poisons the archive reader + } + _ => panic!("unexpected type for keep/.keep"), } - _ => panic!("unexpected type for aa"), - } - - // third entry is a directory called "keep" - let entry = dir_reader - .next() - .expect("next must be some") - .expect("must be some"); - - assert_eq!(&b"keep"[..], &entry.name); - - match entry.node { - nar::reader::Node::Directory(mut subdir_reader) => { - // first entry is .keep, an empty regular file. - let entry = subdir_reader - .next() - .expect("next must succeed") - .expect("must be some"); - - // โฆ it contains a single .keep, an empty regular file. - assert_eq!(&b".keep"[..], &entry.name); - - match entry.node { - nar::reader::Node::File { - executable, - mut reader, - } => { - assert!(!executable); - assert_eq!(reader.read(&mut [0]).unwrap(), 0); - } - _ => panic!("unexpected type for keep/.keep"), + }; + + // this should panic, because we didn't finish reading the child subdirectory + assert!(dir_reader.next().expect("must succeed").is_none()); + } + _ => panic!("unexpected type"), + } +} + +#[test] +#[should_panic] +fn dir_read_after_none() { + let mut f = std::io::Cursor::new(include_bytes!("../tests/complicated.nar")); + let node = nar::reader::open(&mut f).unwrap(); + + match node { + nar::reader::Node::Directory(mut dir_reader) => { + // first entry is .keep, an empty regular file. + must_read_file( + ".keep", + dir_reader + .next() + .expect("next must succeed") + .expect("must be some"), + ); + + // second entry is aa, a symlink to /nix/store/somewhereelse + must_be_symlink( + "aa", + "/nix/store/somewhereelse", + dir_reader + .next() + .expect("next must be some") + .expect("must be some"), + ); + + { + // third entry is a directory called "keep" + let entry = dir_reader + .next() + .expect("next must be some") + .expect("must be some"); + + assert_eq!(b"keep", entry.name); + + match entry.node { + nar::reader::Node::Directory(mut subdir_reader) => { + // first entry is .keep, an empty regular file. + must_read_file( + ".keep", + subdir_reader + .next() + .expect("next must succeed") + .expect("must be some"), + ); + + // we must read the None + assert!( + subdir_reader.next().expect("next must succeed").is_none(), + "keep directory contains only .keep" + ); } + _ => panic!("unexpected type for keep/.keep"), } - _ => panic!("unexpected type for keep/.keep"), - } + }; // reading more entries yields None (and we actually must read until this) assert!(dir_reader.next().expect("must succeed").is_none()); + + // this should panic, because we already got a none so we're meant to stop. + dir_reader.next().unwrap(); + unreachable!() } _ => panic!("unexpected type"), } } + +fn must_read_file(name: &'static str, entry: nar::reader::Entry<'_, '_>) { + assert_eq!(name.as_bytes(), entry.name); + + match entry.node { + nar::reader::Node::File { + executable, + mut reader, + } => { + assert!(!executable); + assert_eq!(reader.read(&mut [0]).unwrap(), 0); + } + _ => panic!("unexpected type for {}", name), + } +} + +fn must_be_symlink( + name: &'static str, + exp_target: &'static str, + entry: nar::reader::Entry<'_, '_>, +) { + assert_eq!(name.as_bytes(), entry.name); + + match entry.node { + nar::reader::Node::Symlink { target } => { + assert_eq!(exp_target.as_bytes(), &target); + } + _ => panic!("unexpected type for {}", name), + } +} diff --git a/tvix/nix-compat/src/nar/wire/mod.rs b/tvix/nix-compat/src/nar/wire/mod.rs index b9e0212495..9e99b530ce 100644 --- a/tvix/nix-compat/src/nar/wire/mod.rs +++ b/tvix/nix-compat/src/nar/wire/mod.rs @@ -90,6 +90,23 @@ pub const TOK_DIR: [u8; 24] = *b"\x09\0\0\0\0\0\0\0directory\0\0\0\0\0\0\0"; pub const TOK_ENT: [u8; 48] = *b"\x05\0\0\0\0\0\0\0entry\0\0\0\x01\0\0\0\0\0\0\0(\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0name\0\0\0\0"; pub const TOK_NOD: [u8; 48] = *b"\x04\0\0\0\0\0\0\0node\0\0\0\0\x01\0\0\0\0\0\0\0(\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0type\0\0\0\0"; pub const TOK_PAR: [u8; 16] = *b"\x01\0\0\0\0\0\0\0)\0\0\0\0\0\0\0"; +#[cfg(feature = "async")] +const TOK_PAD_PAR: [u8; 24] = *b"\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0)\0\0\0\0\0\0\0"; + +#[cfg(feature = "async")] +#[derive(Debug)] +pub(crate) enum PadPar {} + +#[cfg(feature = "async")] +impl crate::wire::reader::Tag for PadPar { + const PATTERN: &'static [u8] = &TOK_PAD_PAR; + + type Buf = [u8; 24]; + + fn make_buf() -> Self::Buf { + [0; 24] + } +} #[test] fn tokens() { diff --git a/tvix/nix-compat/src/nar/wire/tag.rs b/tvix/nix-compat/src/nar/wire/tag.rs index 55b93f9985..4982a0d707 100644 --- a/tvix/nix-compat/src/nar/wire/tag.rs +++ b/tvix/nix-compat/src/nar/wire/tag.rs @@ -10,6 +10,7 @@ pub trait Tag: Sized { const MIN: usize; /// Minimal suitably sized buffer for reading the wire representation + /// /// HACK: This is a workaround for const generics limitations. type Buf: AsMut<[u8]> + Send; diff --git a/tvix/nix-compat/src/nar/writer/async.rs b/tvix/nix-compat/src/nar/writer/async.rs index 11aefab9cb..a2ce68fc3c 100644 --- a/tvix/nix-compat/src/nar/writer/async.rs +++ b/tvix/nix-compat/src/nar/writer/async.rs @@ -10,7 +10,7 @@ //! //! ```rust //! # futures::executor::block_on(async { -//! # use futures::io::BufReader; +//! # use tokio::io::BufReader; //! # let some_file: Vec<u8> = vec![0, 1, 2, 3, 4]; //! //! // Output location to write the NAR to. @@ -31,7 +31,6 @@ //! ``` use crate::nar::wire; -use futures_util::{AsyncBufRead, AsyncBufReadExt, AsyncWrite, AsyncWriteExt}; use std::{ io::{ self, @@ -39,6 +38,7 @@ use std::{ }, pin::Pin, }; +use tokio::io::{AsyncBufRead, AsyncBufReadExt, AsyncWrite, AsyncWriteExt}; /// Convenience type alias for types implementing [`AsyncWrite`]. pub type Writer<'a> = dyn AsyncWrite + Unpin + Send + 'a; diff --git a/tvix/nix-compat/src/nar/writer/test.rs b/tvix/nix-compat/src/nar/writer/test.rs index 7b1dd1a2a9..d7f18a49af 100644 --- a/tvix/nix-compat/src/nar/writer/test.rs +++ b/tvix/nix-compat/src/nar/writer/test.rs @@ -11,16 +11,14 @@ fn symlink() { } #[cfg(feature = "async")] -#[test] -fn symlink_async() { +#[tokio::test] +async fn symlink_async() { let mut buf = vec![]; - futures::executor::block_on(async { - let node = nar::writer::r#async::open(&mut buf).await.unwrap(); - node.symlink("/nix/store/somewhereelse".as_bytes()) - .await - .unwrap(); - }); + let node = nar::writer::r#async::open(&mut buf).await.unwrap(); + node.symlink("/nix/store/somewhereelse".as_bytes()) + .await + .unwrap(); assert_eq!(include_bytes!("../tests/symlink.nar"), buf.as_slice()); } @@ -42,22 +40,22 @@ fn file() { } #[cfg(feature = "async")] -#[test] -fn file_async() { +#[tokio::test] +async fn file_async() { + use std::io::Cursor; + let mut buf = vec![]; - futures::executor::block_on(async { - let node = nar::writer::r#async::open(&mut buf).await.unwrap(); + let node = nar::writer::r#async::open(&mut buf).await.unwrap(); - let file_contents = "Hello World!".to_string(); - node.file( - false, - file_contents.len() as u64, - &mut futures::io::Cursor::new(file_contents), - ) - .await - .unwrap(); - }); + let file_contents = "Hello World!".to_string(); + node.file( + false, + file_contents.len() as u64, + &mut Cursor::new(file_contents), + ) + .await + .unwrap(); assert_eq!(include_bytes!("../tests/helloworld.nar"), buf.as_slice()); } @@ -93,41 +91,38 @@ fn complicated() { } #[cfg(feature = "async")] -#[test] -fn complicated_async() { +#[tokio::test] +async fn complicated_async() { + use std::io::Cursor; + let mut buf = vec![]; - futures::executor::block_on(async { - let node = nar::writer::r#async::open(&mut buf).await.unwrap(); - - let mut dir_node = node.directory().await.unwrap(); - - let e = dir_node.entry(".keep".as_bytes()).await.unwrap(); - e.file(false, 0, &mut futures::io::Cursor::new([])) - .await - .expect("read .keep must succeed"); - - let e = dir_node.entry("aa".as_bytes()).await.unwrap(); - e.symlink("/nix/store/somewhereelse".as_bytes()) - .await - .expect("symlink must succeed"); - - let e = dir_node.entry("keep".as_bytes()).await.unwrap(); - let mut subdir_node = e.directory().await.expect("directory must succeed"); - - let e_sub = subdir_node - .entry(".keep".as_bytes()) - .await - .expect("subdir entry must succeed"); - e_sub - .file(false, 0, &mut futures::io::Cursor::new([])) - .await - .unwrap(); - - // close the subdir, and then the dir, which is required. - subdir_node.close().await.unwrap(); - dir_node.close().await.unwrap(); - }); + let node = nar::writer::r#async::open(&mut buf).await.unwrap(); + + let mut dir_node = node.directory().await.unwrap(); + + let e = dir_node.entry(".keep".as_bytes()).await.unwrap(); + e.file(false, 0, &mut Cursor::new([])) + .await + .expect("read .keep must succeed"); + + let e = dir_node.entry("aa".as_bytes()).await.unwrap(); + e.symlink("/nix/store/somewhereelse".as_bytes()) + .await + .expect("symlink must succeed"); + + let e = dir_node.entry("keep".as_bytes()).await.unwrap(); + let mut subdir_node = e.directory().await.expect("directory must succeed"); + + let e_sub = subdir_node + .entry(".keep".as_bytes()) + .await + .expect("subdir entry must succeed"); + e_sub.file(false, 0, &mut Cursor::new([])).await.unwrap(); + + // close the subdir, and then the dir, which is required. + subdir_node.close().await.unwrap(); + dir_node.close().await.unwrap(); assert_eq!(include_bytes!("../tests/complicated.nar"), buf.as_slice()); } diff --git a/tvix/nix-compat/src/narinfo/public_keys.rs b/tvix/nix-compat/src/narinfo/public_keys.rs index ced05cadb1..27dd90e096 100644 --- a/tvix/nix-compat/src/narinfo/public_keys.rs +++ b/tvix/nix-compat/src/narinfo/public_keys.rs @@ -106,40 +106,43 @@ impl Display for PubKey { mod test { use data_encoding::BASE64; use ed25519_dalek::PUBLIC_KEY_LENGTH; - use test_case::test_case; + use rstest::rstest; use crate::narinfo::Signature; use super::PubKey; const FINGERPRINT: &str = "1;/nix/store/syd87l2rxw8cbsxmxl853h0r6pdwhwjr-curl-7.82.0-bin;sha256:1b4sb93wp679q4zx9k1ignby1yna3z7c4c2ri3wphylbc2dwsys0;196040;/nix/store/0jqd0rlxzra1rs38rdxl43yh6rxchgc6-curl-7.82.0,/nix/store/6w8g7njm4mck5dmjxws0z1xnrxvl81xa-glibc-2.34-115,/nix/store/j5jxw3iy7bbz4a57fh9g2xm2gxmyal8h-zlib-1.2.12,/nix/store/yxvjs9drzsphm9pcf42a4byzj1kb9m7k-openssl-1.1.1n"; - #[test_case("cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=", "cache.nixos.org-1", BASE64.decode(b"6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=").unwrap()[..].try_into().unwrap(); "cache.nixos.org")] - #[test_case("cheesecake:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=", "cheesecake", BASE64.decode(b"6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=").unwrap()[..].try_into().unwrap(); "cache.nixos.org different name")] - #[test_case("test1:tLAEn+EeaBUJYqEpTd2yeerr7Ic6+0vWe+aXL/vYUpE=", "test1", BASE64.decode(b"tLAEn+EeaBUJYqEpTd2yeerr7Ic6+0vWe+aXL/vYUpE=").unwrap()[..].try_into().unwrap(); "test-1")] + #[rstest] + #[case::cache_nixos_org("cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=", "cache.nixos.org-1", &BASE64.decode(b"6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=").unwrap()[..].try_into().unwrap())] + #[case::cache_nixos_org_different_name("cheesecake:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=", "cheesecake", &BASE64.decode(b"6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=").unwrap()[..].try_into().unwrap())] + #[case::test_1("test1:tLAEn+EeaBUJYqEpTd2yeerr7Ic6+0vWe+aXL/vYUpE=", "test1", &BASE64.decode(b"tLAEn+EeaBUJYqEpTd2yeerr7Ic6+0vWe+aXL/vYUpE=").unwrap()[..].try_into().unwrap())] fn parse( - input: &'static str, - exp_name: &'static str, - exp_verifying_key_bytes: &[u8; PUBLIC_KEY_LENGTH], + #[case] input: &'static str, + #[case] exp_name: &'static str, + #[case] exp_verifying_key_bytes: &[u8; PUBLIC_KEY_LENGTH], ) { let pubkey = PubKey::parse(input).expect("must parse"); assert_eq!(exp_name, pubkey.name()); assert_eq!(exp_verifying_key_bytes, pubkey.verifying_key.as_bytes()); } - #[test_case("6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; "empty name")] - #[test_case("cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY"; "missing padding")] - #[test_case("cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDS"; "wrong length")] - fn parse_fail(input: &'static str) { + #[rstest] + #[case::empty_name("6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=")] + #[case::missing_padding("cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY")] + #[case::wrong_length("cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDS")] + fn parse_fail(#[case] input: &'static str) { PubKey::parse(input).expect_err("must fail"); } - #[test_case("cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=", FINGERPRINT, "cache.nixos.org-1:TsTTb3WGTZKphvYdBHXwo6weVILmTytUjLB+vcX89fOjjRicCHmKA4RCPMVLkj6TMJ4GMX3HPVWRdD1hkeKZBQ==", true; "correct cache.nixos.org")] - #[test_case("cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=", FINGERPRINT, "cache.nixos.org:TsTTb3WGTZKphvYdBHXwo6weVILmTytUjLB+vcX89fOjjRicCHmKA4RCPMVLkj6TMJ4GMX3HPVWRdD1hkeKZBQ==", false; "wrong name mismatch")] + #[rstest] + #[case::correct_cache_nixos_org("cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=", FINGERPRINT, "cache.nixos.org-1:TsTTb3WGTZKphvYdBHXwo6weVILmTytUjLB+vcX89fOjjRicCHmKA4RCPMVLkj6TMJ4GMX3HPVWRdD1hkeKZBQ==", true)] + #[case::wrong_name_mismatch("cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=", FINGERPRINT, "cache.nixos.org:TsTTb3WGTZKphvYdBHXwo6weVILmTytUjLB+vcX89fOjjRicCHmKA4RCPMVLkj6TMJ4GMX3HPVWRdD1hkeKZBQ==", false)] fn verify( - pubkey_str: &'static str, - fingerprint: &'static str, - signature_str: &'static str, - expected: bool, + #[case] pubkey_str: &'static str, + #[case] fingerprint: &'static str, + #[case] signature_str: &'static str, + #[case] expected: bool, ) { let pubkey = PubKey::parse(pubkey_str).expect("must parse"); let signature = Signature::parse(signature_str).expect("must parse"); diff --git a/tvix/nix-compat/src/narinfo/signature.rs b/tvix/nix-compat/src/narinfo/signature.rs index 1f4a90c26c..fd197e771d 100644 --- a/tvix/nix-compat/src/narinfo/signature.rs +++ b/tvix/nix-compat/src/narinfo/signature.rs @@ -2,8 +2,9 @@ use std::fmt::{self, Display}; use data_encoding::BASE64; use ed25519_dalek::SIGNATURE_LENGTH; +use serde::{Deserialize, Serialize}; -#[derive(Debug)] +#[derive(Clone, Debug, Eq, PartialEq)] pub struct Signature<'a> { name: &'a str, bytes: [u8; SIGNATURE_LENGTH], @@ -57,6 +58,29 @@ impl<'a> Signature<'a> { } } +impl<'de: 'a, 'a> Deserialize<'de> for Signature<'a> { + fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> + where + D: serde::Deserializer<'de>, + { + let str: &'de str = Deserialize::deserialize(deserializer)?; + Self::parse(str).map_err(|_| { + serde::de::Error::invalid_value(serde::de::Unexpected::Str(str), &"Signature") + }) + } +} + +impl<'a> Serialize for Signature<'a> { + fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> + where + S: serde::Serializer, + { + let string: String = self.to_string(); + + string.serialize(serializer) + } +} + #[derive(Debug, thiserror::Error)] pub enum Error { #[error("Invalid name: {0}")] @@ -79,10 +103,11 @@ impl Display for Signature<'_> { mod test { use data_encoding::BASE64; use ed25519_dalek::VerifyingKey; + use hex_literal::hex; use lazy_static::lazy_static; use super::Signature; - use test_case::test_case; + use rstest::rstest; const FINGERPRINT: &str = "1;/nix/store/syd87l2rxw8cbsxmxl853h0r6pdwhwjr-curl-7.82.0-bin;sha256:1b4sb93wp679q4zx9k1ignby1yna3z7c4c2ri3wphylbc2dwsys0;196040;/nix/store/0jqd0rlxzra1rs38rdxl43yh6rxchgc6-curl-7.82.0,/nix/store/6w8g7njm4mck5dmjxws0z1xnrxvl81xa-glibc-2.34-115,/nix/store/j5jxw3iy7bbz4a57fh9g2xm2gxmyal8h-zlib-1.2.12,/nix/store/yxvjs9drzsphm9pcf42a4byzj1kb9m7k-openssl-1.1.1n"; @@ -106,27 +131,54 @@ mod test { .unwrap(); } - #[test_case(&PUB_CACHE_NIXOS_ORG_1, &"cache.nixos.org-1:TsTTb3WGTZKphvYdBHXwo6weVILmTytUjLB+vcX89fOjjRicCHmKA4RCPMVLkj6TMJ4GMX3HPVWRdD1hkeKZBQ==", FINGERPRINT, true; "valid cache.nixos.org-1")] - #[test_case(&PUB_CACHE_NIXOS_ORG_1, &"cache.nixos.org-1:TsTTb3WGTZKphvYdBHXwo6weVILmTytUjLB+vcX89fOjjRicCHmKA4RCPMVLkj6TMJ4GMX3HPVWRdD1hkeKZBQ==", FINGERPRINT, true; "valid test1")] - #[test_case(&PUB_CACHE_NIXOS_ORG_1, &"cache.nixos.org-2:TsTTb3WGTZKphvYdBHXwo6weVILmTytUjLB+vcX89fOjjRicCHmKA4RCPMVLkj6TMJ4GMX3HPVWRdD1hkeKZBQ==", FINGERPRINT, true; "valid cache.nixos.org different name")] - #[test_case(&PUB_CACHE_NIXOS_ORG_1, &"cache.nixos.org-1:TsTTb000000000000000000000000ytUjLB+vcX89fOjjRicCHmKA4RCPMVLkj6TMJ4GMX3HPVWRdD1hkeKZBQ==", FINGERPRINT, false; "fail invalid cache.nixos.org-1 signature")] - #[test_case(&PUB_CACHE_NIXOS_ORG_1, &"cache.nixos.org-1:TsTTb3WGTZKphvYdBHXwo6weVILmTytUjLB+vcX89fOjjRicCHmKA4RCPMVLkj6TMJ4GMX3HPVWRdD1hkeKZBQ==", &FINGERPRINT[0..5], false; "fail valid sig but wrong fp cache.nixos.org-1")] + #[rstest] + #[case::valid_cache_nixos_org_1(&PUB_CACHE_NIXOS_ORG_1, &"cache.nixos.org-1:TsTTb3WGTZKphvYdBHXwo6weVILmTytUjLB+vcX89fOjjRicCHmKA4RCPMVLkj6TMJ4GMX3HPVWRdD1hkeKZBQ==", FINGERPRINT, true)] + #[case::valid_test1(&PUB_CACHE_NIXOS_ORG_1, &"cache.nixos.org-1:TsTTb3WGTZKphvYdBHXwo6weVILmTytUjLB+vcX89fOjjRicCHmKA4RCPMVLkj6TMJ4GMX3HPVWRdD1hkeKZBQ==", FINGERPRINT, true)] + #[case::valid_cache_nixos_org_different_name(&PUB_CACHE_NIXOS_ORG_1, &"cache.nixos.org-2:TsTTb3WGTZKphvYdBHXwo6weVILmTytUjLB+vcX89fOjjRicCHmKA4RCPMVLkj6TMJ4GMX3HPVWRdD1hkeKZBQ==", FINGERPRINT, true)] + #[case::fail_invalid_cache_nixos_org_1_signature(&PUB_CACHE_NIXOS_ORG_1, &"cache.nixos.org-1:TsTTb000000000000000000000000ytUjLB+vcX89fOjjRicCHmKA4RCPMVLkj6TMJ4GMX3HPVWRdD1hkeKZBQ==", FINGERPRINT, false)] + #[case::fail_valid_sig_but_wrong_fp_cache_nixos_org_1(&PUB_CACHE_NIXOS_ORG_1, &"cache.nixos.org-1:TsTTb3WGTZKphvYdBHXwo6weVILmTytUjLB+vcX89fOjjRicCHmKA4RCPMVLkj6TMJ4GMX3HPVWRdD1hkeKZBQ==", &FINGERPRINT[0..5], false)] fn verify_sigs( - verifying_key: &VerifyingKey, - sig_str: &'static str, - fp: &str, - expect_valid: bool, + #[case] verifying_key: &VerifyingKey, + #[case] sig_str: &'static str, + #[case] fp: &str, + #[case] expect_valid: bool, ) { let sig = Signature::parse(sig_str).expect("must parse"); assert_eq!(expect_valid, sig.verify(fp.as_bytes(), verifying_key)); } - #[test_case("cache.nixos.org-1:o1DTsjCz0PofLJ216P2RBuSulI8BAb6zHxWE4N+tzlcELk5Uk/GO2SCxWTRN5wJutLZZ+cHTMdWqOHF8"; "wrong length")] - #[test_case("test\n:u01BybwQhyI5H1bW1EIWXssMDhDDIvXOG5uh8Qzgdyjz6U1qg6DHhMAvXZOUStIj6X5t4/ufFgR8i3fjf0bMAw=="; "wrong name newline")] - #[test_case("test :u01BybwQhyI5H1bW1EIWXssMDhDDIvXOG5uh8Qzgdyjz6U1qg6DHhMAvXZOUStIj6X5t4/ufFgR8i3fjf0bMAw=="; "wrong name space")] - #[test_case(":u01BybwQhyI5H1bW1EIWXssMDhDDIvXOG5uh8Qzgdyjz6U1qg6DHhMAvXZOUStIj6X5t4/ufFgR8i3fjf0bMAw=="; "empty name")] - #[test_case("u01BybwQhyI5H1bW1EIWXssMDhDDIvXOG5uh8Qzgdyjz6U1qg6DHhMAvXZOUStIj6X5t4/ufFgR8i3fjf0bMAw=="; "b64 only")] - fn parse_fail(input: &'static str) { + #[rstest] + #[case::wrong_length("cache.nixos.org-1:o1DTsjCz0PofLJ216P2RBuSulI8BAb6zHxWE4N+tzlcELk5Uk/GO2SCxWTRN5wJutLZZ+cHTMdWqOHF8")] + #[case::wrong_name_newline("test\n:u01BybwQhyI5H1bW1EIWXssMDhDDIvXOG5uh8Qzgdyjz6U1qg6DHhMAvXZOUStIj6X5t4/ufFgR8i3fjf0bMAw==")] + #[case::wrong_name_space("test :u01BybwQhyI5H1bW1EIWXssMDhDDIvXOG5uh8Qzgdyjz6U1qg6DHhMAvXZOUStIj6X5t4/ufFgR8i3fjf0bMAw==")] + #[case::empty_name( + ":u01BybwQhyI5H1bW1EIWXssMDhDDIvXOG5uh8Qzgdyjz6U1qg6DHhMAvXZOUStIj6X5t4/ufFgR8i3fjf0bMAw==" + )] + #[case::b64_only( + "u01BybwQhyI5H1bW1EIWXssMDhDDIvXOG5uh8Qzgdyjz6U1qg6DHhMAvXZOUStIj6X5t4/ufFgR8i3fjf0bMAw==" + )] + fn parse_fail(#[case] input: &'static str) { Signature::parse(input).expect_err("must fail"); } + + #[test] + fn serialize_deserialize() { + let signature_actual = Signature { + name: "cache.nixos.org-1", + bytes: hex!( + r#"4e c4 d3 6f 75 86 4d 92 a9 86 f6 1d 04 75 f0 a3 + ac 1e 54 82 e6 4f 2b 54 8c b0 7e bd c5 fc f5 f3 + a3 8d 18 9c 08 79 8a 03 84 42 3c c5 4b 92 3e 93 + 30 9e 06 31 7d c7 3d 55 91 74 3d 61 91 e2 99 05"# + ), + }; + let signature_str_json = "\"cache.nixos.org-1:TsTTb3WGTZKphvYdBHXwo6weVILmTytUjLB+vcX89fOjjRicCHmKA4RCPMVLkj6TMJ4GMX3HPVWRdD1hkeKZBQ==\""; + + let serialized = serde_json::to_string(&signature_actual).expect("must serialize"); + assert_eq!(signature_str_json, &serialized); + + let deserialized: Signature<'_> = + serde_json::from_str(signature_str_json).expect("must deserialize"); + assert_eq!(&signature_actual, &deserialized); + } } diff --git a/tvix/nix-compat/src/nix_daemon/mod.rs b/tvix/nix-compat/src/nix_daemon/mod.rs new file mode 100644 index 0000000000..fe652377d1 --- /dev/null +++ b/tvix/nix-compat/src/nix_daemon/mod.rs @@ -0,0 +1,4 @@ +pub mod worker_protocol; + +mod protocol_version; +pub use protocol_version::ProtocolVersion; diff --git a/tvix/nix-compat/src/nix_daemon/protocol_version.rs b/tvix/nix-compat/src/nix_daemon/protocol_version.rs new file mode 100644 index 0000000000..8fd2b085c9 --- /dev/null +++ b/tvix/nix-compat/src/nix_daemon/protocol_version.rs @@ -0,0 +1,123 @@ +/// Protocol versions are represented as a u16. +/// The upper 8 bits are the major version, the lower bits the minor. +/// This is not aware of any endianness, use [crate::wire::read_u64] to get an +/// u64 first, and the try_from() impl from here if you're receiving over the +/// Nix Worker protocol. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ProtocolVersion(u16); + +impl ProtocolVersion { + pub const fn from_parts(major: u8, minor: u8) -> Self { + Self(((major as u16) << 8) | minor as u16) + } + + pub fn major(&self) -> u8 { + ((self.0 & 0xff00) >> 8) as u8 + } + + pub fn minor(&self) -> u8 { + (self.0 & 0x00ff) as u8 + } +} + +impl PartialOrd for ProtocolVersion { + fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { + Some(self.cmp(other)) + } +} + +impl Ord for ProtocolVersion { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + match self.major().cmp(&other.major()) { + std::cmp::Ordering::Less => std::cmp::Ordering::Less, + std::cmp::Ordering::Greater => std::cmp::Ordering::Greater, + std::cmp::Ordering::Equal => { + // same major, compare minor + self.minor().cmp(&other.minor()) + } + } + } +} + +impl From<u16> for ProtocolVersion { + fn from(value: u16) -> Self { + Self::from_parts(((value & 0xff00) >> 8) as u8, (value & 0x00ff) as u8) + } +} + +impl TryFrom<u64> for ProtocolVersion { + type Error = &'static str; + + fn try_from(value: u64) -> Result<Self, Self::Error> { + if value & !0xffff != 0 { + return Err("only two least significant bits might be populated"); + } + + Ok((value as u16).into()) + } +} + +impl From<ProtocolVersion> for u16 { + fn from(value: ProtocolVersion) -> Self { + value.0 + } +} + +impl From<ProtocolVersion> for u64 { + fn from(value: ProtocolVersion) -> Self { + value.0 as u64 + } +} + +impl std::fmt::Display for ProtocolVersion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}.{}", self.major(), self.minor()) + } +} + +#[cfg(test)] +mod tests { + use super::ProtocolVersion; + + #[test] + fn from_parts() { + let version = ProtocolVersion::from_parts(1, 37); + assert_eq!(version.major(), 1, "correct major"); + assert_eq!(version.minor(), 37, "correct minor"); + assert_eq!("1.37", &version.to_string(), "to_string"); + + assert_eq!(0x0125, Into::<u16>::into(version)); + assert_eq!(0x0125, Into::<u64>::into(version)); + } + + #[test] + fn from_u16() { + let version = ProtocolVersion::from(0x0125_u16); + assert_eq!("1.37", &version.to_string()); + } + + #[test] + fn from_u64() { + let version = ProtocolVersion::try_from(0x0125_u64).expect("must succeed"); + assert_eq!("1.37", &version.to_string()); + } + + /// This contains data in higher bits, which should fail. + #[test] + fn from_u64_fail() { + ProtocolVersion::try_from(0xaa0125_u64).expect_err("must fail"); + } + + #[test] + fn ord() { + let v0_37 = ProtocolVersion::from_parts(0, 37); + let v1_37 = ProtocolVersion::from_parts(1, 37); + let v1_40 = ProtocolVersion::from_parts(1, 40); + + assert!(v0_37 < v1_37); + assert!(v1_37 > v0_37); + assert!(v1_37 < v1_40); + assert!(v1_40 > v1_37); + assert!(v1_40 <= v1_40); + } +} diff --git a/tvix/nix-compat/src/nix_daemon/worker_protocol.rs b/tvix/nix-compat/src/nix_daemon/worker_protocol.rs new file mode 100644 index 0000000000..7e3adc0db2 --- /dev/null +++ b/tvix/nix-compat/src/nix_daemon/worker_protocol.rs @@ -0,0 +1,434 @@ +use std::{ + collections::HashMap, + io::{Error, ErrorKind}, +}; + +use enum_primitive_derive::Primitive; +use num_traits::{FromPrimitive, ToPrimitive}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +use crate::wire; + +use super::ProtocolVersion; + +static WORKER_MAGIC_1: u64 = 0x6e697863; // "nixc" +static WORKER_MAGIC_2: u64 = 0x6478696f; // "dxio" +pub static STDERR_LAST: u64 = 0x616c7473; // "alts" + +/// | Nix version | Protocol | +/// |-----------------|----------| +/// | 0.11 | 1.02 | +/// | 0.12 | 1.04 | +/// | 0.13 | 1.05 | +/// | 0.14 | 1.05 | +/// | 0.15 | 1.05 | +/// | 0.16 | 1.06 | +/// | 1.0 | 1.10 | +/// | 1.1 | 1.11 | +/// | 1.2 | 1.12 | +/// | 1.3 - 1.5.3 | 1.13 | +/// | 1.6 - 1.10 | 1.14 | +/// | 1.11 - 1.11.16 | 1.15 | +/// | 2.0 - 2.0.4 | 1.20 | +/// | 2.1 - 2.3.18 | 1.21 | +/// | 2.4 - 2.6.1 | 1.32 | +/// | 2.7.0 | 1.33 | +/// | 2.8.0 - 2.14.1 | 1.34 | +/// | 2.15.0 - 2.19.4 | 1.35 | +/// | 2.20.0 - 2.22.0 | 1.37 | +static PROTOCOL_VERSION: ProtocolVersion = ProtocolVersion::from_parts(1, 37); + +/// Max length of a Nix setting name/value. In bytes. +/// +/// This value has been arbitrarily choosen after looking the nix.conf +/// manpage. Don't hesitate to increase it if it's too limiting. +pub static MAX_SETTING_SIZE: usize = 1024; + +/// Worker Operation +/// +/// These operations are encoded as unsigned 64 bits before being sent +/// to the wire. See the [read_op] and +/// [write_op] operations to serialize/deserialize the +/// operation on the wire. +/// +/// Note: for now, we're using the Nix 2.20 operation description. The +/// operations marked as obsolete are obsolete for Nix 2.20, not +/// necessarily for Nix 2.3. We'll revisit this later on. +#[derive(Debug, PartialEq, Primitive)] +pub enum Operation { + IsValidPath = 1, + HasSubstitutes = 3, + QueryPathHash = 4, // obsolete + QueryReferences = 5, // obsolete + QueryReferrers = 6, + AddToStore = 7, + AddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use WorkerProto::Op::AddToStore + BuildPaths = 9, + EnsurePath = 10, + AddTempRoot = 11, + AddIndirectRoot = 12, + SyncWithGC = 13, + FindRoots = 14, + ExportPath = 16, // obsolete + QueryDeriver = 18, // obsolete + SetOptions = 19, + CollectGarbage = 20, + QuerySubstitutablePathInfo = 21, + QueryDerivationOutputs = 22, // obsolete + QueryAllValidPaths = 23, + QueryFailedPaths = 24, + ClearFailedPaths = 25, + QueryPathInfo = 26, + ImportPaths = 27, // obsolete + QueryDerivationOutputNames = 28, // obsolete + QueryPathFromHashPart = 29, + QuerySubstitutablePathInfos = 30, + QueryValidPaths = 31, + QuerySubstitutablePaths = 32, + QueryValidDerivers = 33, + OptimiseStore = 34, + VerifyStore = 35, + BuildDerivation = 36, + AddSignatures = 37, + NarFromPath = 38, + AddToStoreNar = 39, + QueryMissing = 40, + QueryDerivationOutputMap = 41, + RegisterDrvOutput = 42, + QueryRealisation = 43, + AddMultipleToStore = 44, + AddBuildLog = 45, + BuildPathsWithResults = 46, + AddPermRoot = 47, +} + +/// Log verbosity. In the Nix wire protocol, the client requests a +/// verbosity level to the daemon, which in turns does not produce any +/// log below this verbosity. +#[derive(Debug, PartialEq, Primitive)] +pub enum Verbosity { + LvlError = 0, + LvlWarn = 1, + LvlNotice = 2, + LvlInfo = 3, + LvlTalkative = 4, + LvlChatty = 5, + LvlDebug = 6, + LvlVomit = 7, +} + +/// Settings requested by the client. These settings are applied to a +/// connection to between the daemon and a client. +#[derive(Debug, PartialEq)] +pub struct ClientSettings { + pub keep_failed: bool, + pub keep_going: bool, + pub try_fallback: bool, + pub verbosity: Verbosity, + pub max_build_jobs: u64, + pub max_silent_time: u64, + pub verbose_build: bool, + pub build_cores: u64, + pub use_substitutes: bool, + /// Key/Value dictionary in charge of overriding the settings set + /// by the Nix config file. + /// + /// Some settings can be safely overidden, + /// some other require the user running the Nix client to be part + /// of the trusted users group. + pub overrides: HashMap<String, String>, +} + +/// Reads the client settings from the wire. +/// +/// Note: this function **only** reads the settings. It does not +/// manage the log state with the daemon. You'll have to do that on +/// your own. A minimal log implementation will consist in sending +/// back [STDERR_LAST] to the client after reading the client +/// settings. +/// +/// FUTUREWORK: write serialization. +pub async fn read_client_settings<R: AsyncReadExt + Unpin>( + r: &mut R, + client_version: ProtocolVersion, +) -> std::io::Result<ClientSettings> { + let keep_failed = r.read_u64_le().await? != 0; + let keep_going = r.read_u64_le().await? != 0; + let try_fallback = r.read_u64_le().await? != 0; + let verbosity_uint = r.read_u64_le().await?; + let verbosity = Verbosity::from_u64(verbosity_uint).ok_or_else(|| { + Error::new( + ErrorKind::InvalidData, + format!("Can't convert integer {} to verbosity", verbosity_uint), + ) + })?; + let max_build_jobs = r.read_u64_le().await?; + let max_silent_time = r.read_u64_le().await?; + _ = r.read_u64_le().await?; // obsolete useBuildHook + let verbose_build = r.read_u64_le().await? != 0; + _ = r.read_u64_le().await?; // obsolete logType + _ = r.read_u64_le().await?; // obsolete printBuildTrace + let build_cores = r.read_u64_le().await?; + let use_substitutes = r.read_u64_le().await? != 0; + let mut overrides = HashMap::new(); + if client_version.minor() >= 12 { + let num_overrides = r.read_u64_le().await?; + for _ in 0..num_overrides { + let name = wire::read_string(r, 0..=MAX_SETTING_SIZE).await?; + let value = wire::read_string(r, 0..=MAX_SETTING_SIZE).await?; + overrides.insert(name, value); + } + } + Ok(ClientSettings { + keep_failed, + keep_going, + try_fallback, + verbosity, + max_build_jobs, + max_silent_time, + verbose_build, + build_cores, + use_substitutes, + overrides, + }) +} + +/// Performs the initial handshake the server is sending to a connecting client. +/// +/// During the handshake, the client first send a magic u64, to which +/// the daemon needs to respond with another magic u64. +/// Then, the daemon retrieves the client version, and discards a bunch of now +/// obsolete data. +/// +/// # Arguments +/// +/// * conn: connection with the Nix client. +/// * nix_version: semantic version of the Nix daemon. "2.18.2" for +/// instance. +/// * trusted: trust level of the Nix client. +/// +/// # Return +/// +/// The protocol version of the client. +pub async fn server_handshake_client<'a, RW: 'a>( + mut conn: &'a mut RW, + nix_version: &str, + trusted: Trust, +) -> std::io::Result<ProtocolVersion> +where + &'a mut RW: AsyncReadExt + AsyncWriteExt + Unpin, +{ + let worker_magic_1 = conn.read_u64_le().await?; + if worker_magic_1 != WORKER_MAGIC_1 { + Err(std::io::Error::new( + ErrorKind::InvalidData, + format!("Incorrect worker magic number received: {}", worker_magic_1), + )) + } else { + conn.write_u64_le(WORKER_MAGIC_2).await?; + conn.write_u64_le(PROTOCOL_VERSION.into()).await?; + conn.flush().await?; + let client_version = conn.read_u64_le().await?; + // Parse into ProtocolVersion. + let client_version: ProtocolVersion = client_version + .try_into() + .map_err(|e| Error::new(ErrorKind::Unsupported, e))?; + if client_version < ProtocolVersion::from_parts(1, 10) { + return Err(Error::new( + ErrorKind::Unsupported, + format!("The nix client version {} is too old", client_version), + )); + } + if client_version.minor() >= 14 { + // Obsolete CPU affinity. + let read_affinity = conn.read_u64_le().await?; + if read_affinity != 0 { + let _cpu_affinity = conn.read_u64_le().await?; + }; + } + if client_version.minor() >= 11 { + // Obsolete reserveSpace + let _reserve_space = conn.read_u64_le().await?; + } + if client_version.minor() >= 33 { + // Nix version. We're plain lying, we're not Nix, but ehโฆ + // Setting it to the 2.3 lineage. Not 100% sure this is a + // good idea. + wire::write_bytes(&mut conn, nix_version).await?; + conn.flush().await?; + } + if client_version.minor() >= 35 { + write_worker_trust_level(&mut conn, trusted).await?; + } + Ok(client_version) + } +} + +/// Read a worker [Operation] from the wire. +pub async fn read_op<R: AsyncReadExt + Unpin>(r: &mut R) -> std::io::Result<Operation> { + let op_number = r.read_u64_le().await?; + Operation::from_u64(op_number).ok_or(Error::new( + ErrorKind::InvalidData, + format!("Invalid OP number {}", op_number), + )) +} + +/// Write a worker [Operation] to the wire. +pub async fn write_op<W: AsyncWriteExt + Unpin>(w: &mut W, op: &Operation) -> std::io::Result<()> { + let op = Operation::to_u64(op).ok_or(Error::new( + ErrorKind::Other, + format!("Can't convert the OP {:?} to u64", op), + ))?; + w.write_u64(op).await +} + +#[derive(Debug, PartialEq)] +pub enum Trust { + Trusted, + NotTrusted, +} + +/// Write the worker [Trust] level to the wire. +/// +/// Cpp Nix has a legacy third option: u8 0. This option is meant to +/// be used as a backward compatible measure. Since we're not +/// targetting protocol versions pre-dating the trust notion, we +/// decided not to implement it here. +pub async fn write_worker_trust_level<W>(conn: &mut W, t: Trust) -> std::io::Result<()> +where + W: AsyncReadExt + AsyncWriteExt + Unpin, +{ + match t { + Trust::Trusted => conn.write_u64_le(1).await, + Trust::NotTrusted => conn.write_u64_le(2).await, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use hex_literal::hex; + use tokio_test::io::Builder; + + #[tokio::test] + async fn test_init_hanshake() { + let mut test_conn = tokio_test::io::Builder::new() + .read(&WORKER_MAGIC_1.to_le_bytes()) + .write(&WORKER_MAGIC_2.to_le_bytes()) + .write(&[37, 1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) + // Let's say the client is in sync with the daemon + // protocol-wise + .read(&[37, 1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) + // cpu affinity + .read(&[0; 8]) + // reservespace + .read(&[0; 8]) + // version (size) + .write(&[0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) + // version (data == 2.18.2 + padding) + .write(&[50, 46, 49, 56, 46, 50, 0, 0]) + // Trusted (1 == client trusted + .write(&[1, 0, 0, 0, 0, 0, 0, 0]) + .build(); + let client_version = server_handshake_client(&mut test_conn, "2.18.2", Trust::Trusted) + .await + .unwrap(); + + assert_eq!(client_version, PROTOCOL_VERSION) + } + + #[tokio::test] + async fn test_read_client_settings_without_overrides() { + // Client settings bits captured from a Nix 2.3.17 run w/ sockdump (protocol version 21). + let wire_bits = hex!( + "00 00 00 00 00 00 00 00 \ + 00 00 00 00 00 00 00 00 \ + 00 00 00 00 00 00 00 00 \ + 02 00 00 00 00 00 00 00 \ + 10 00 00 00 00 00 00 00 \ + 00 00 00 00 00 00 00 00 \ + 01 00 00 00 00 00 00 00 \ + 00 00 00 00 00 00 00 00 \ + 00 00 00 00 00 00 00 00 \ + 00 00 00 00 00 00 00 00 \ + 00 00 00 00 00 00 00 00 \ + 01 00 00 00 00 00 00 00 \ + 00 00 00 00 00 00 00 00" + ); + let mut mock = Builder::new().read(&wire_bits).build(); + let settings = read_client_settings(&mut mock, ProtocolVersion::from_parts(1, 21)) + .await + .expect("should parse"); + let expected = ClientSettings { + keep_failed: false, + keep_going: false, + try_fallback: false, + verbosity: Verbosity::LvlNotice, + max_build_jobs: 16, + max_silent_time: 0, + verbose_build: false, + build_cores: 0, + use_substitutes: true, + overrides: HashMap::new(), + }; + assert_eq!(settings, expected); + } + + #[tokio::test] + async fn test_read_client_settings_with_overrides() { + // Client settings bits captured from a Nix 2.3.17 run w/ sockdump (protocol version 21). + let wire_bits = hex!( + "00 00 00 00 00 00 00 00 \ + 00 00 00 00 00 00 00 00 \ + 00 00 00 00 00 00 00 00 \ + 02 00 00 00 00 00 00 00 \ + 10 00 00 00 00 00 00 00 \ + 00 00 00 00 00 00 00 00 \ + 01 00 00 00 00 00 00 00 \ + 00 00 00 00 00 00 00 00 \ + 00 00 00 00 00 00 00 00 \ + 00 00 00 00 00 00 00 00 \ + 00 00 00 00 00 00 00 00 \ + 01 00 00 00 00 00 00 00 \ + 02 00 00 00 00 00 00 00 \ + 0c 00 00 00 00 00 00 00 \ + 61 6c 6c 6f 77 65 64 2d \ + 75 72 69 73 00 00 00 00 \ + 1e 00 00 00 00 00 00 00 \ + 68 74 74 70 73 3a 2f 2f \ + 62 6f 72 64 65 61 75 78 \ + 2e 67 75 69 78 2e 67 6e \ + 75 2e 6f 72 67 2f 00 00 \ + 0d 00 00 00 00 00 00 00 \ + 61 6c 6c 6f 77 65 64 2d \ + 75 73 65 72 73 00 00 00 \ + 0b 00 00 00 00 00 00 00 \ + 6a 65 61 6e 20 70 69 65 \ + 72 72 65 00 00 00 00 00" + ); + let mut mock = Builder::new().read(&wire_bits).build(); + let settings = read_client_settings(&mut mock, ProtocolVersion::from_parts(1, 21)) + .await + .expect("should parse"); + let overrides = HashMap::from([ + ( + String::from("allowed-uris"), + String::from("https://bordeaux.guix.gnu.org/"), + ), + (String::from("allowed-users"), String::from("jean pierre")), + ]); + let expected = ClientSettings { + keep_failed: false, + keep_going: false, + try_fallback: false, + verbosity: Verbosity::LvlNotice, + max_build_jobs: 16, + max_silent_time: 0, + verbose_build: false, + build_cores: 0, + use_substitutes: true, + overrides, + }; + assert_eq!(settings, expected); + } +} diff --git a/tvix/nix-compat/src/nixbase32.rs b/tvix/nix-compat/src/nixbase32.rs index c4c2f2b967..b7ffc1dc2b 100644 --- a/tvix/nix-compat/src/nixbase32.rs +++ b/tvix/nix-compat/src/nixbase32.rs @@ -141,27 +141,30 @@ pub const fn encode_len(len: usize) -> usize { #[cfg(test)] mod tests { use hex_literal::hex; - use test_case::test_case; + use rstest::rstest; - #[test_case("", &[]; "empty bytes")] - #[test_case("0z", &hex!("1f"); "one byte")] - #[test_case("00bgd045z0d4icpbc2yyz4gx48ak44la", &hex!("8a12321522fd91efbd60ebb2481af88580f61600"); "store path")] - #[test_case("0c5b8vw40dy178xlpddw65q9gf1h2186jcc3p4swinwggbllv8mk", &hex!("b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30"); "sha256")] - fn encode(enc: &str, dec: &[u8]) { + #[rstest] + #[case::empty_bytes("", &[])] + #[case::one_byte("0z", &hex!("1f"))] + #[case::store_path("00bgd045z0d4icpbc2yyz4gx48ak44la", &hex!("8a12321522fd91efbd60ebb2481af88580f61600"))] + #[case::sha256("0c5b8vw40dy178xlpddw65q9gf1h2186jcc3p4swinwggbllv8mk", &hex!("b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30"))] + #[test] + fn encode(#[case] enc: &str, #[case] dec: &[u8]) { assert_eq!(enc, super::encode(dec)); } - #[test_case("", Some(&[]) ; "empty bytes")] - #[test_case("0z", Some(&hex!("1f")); "one byte")] - #[test_case("00bgd045z0d4icpbc2yyz4gx48ak44la", Some(&hex!("8a12321522fd91efbd60ebb2481af88580f61600")); "store path")] - #[test_case("0c5b8vw40dy178xlpddw65q9gf1h2186jcc3p4swinwggbllv8mk", Some(&hex!("b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30")); "sha256")] + #[rstest] + #[case::empty_bytes("", Some(&[][..]) )] + #[case::one_byte("0z", Some(&hex!("1f")[..]))] + #[case::store_path("00bgd045z0d4icpbc2yyz4gx48ak44la", Some(&hex!("8a12321522fd91efbd60ebb2481af88580f61600")[..]))] + #[case::sha256("0c5b8vw40dy178xlpddw65q9gf1h2186jcc3p4swinwggbllv8mk", Some(&hex!("b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30")[..]))] // this is invalid encoding, because it encodes 10 1-bits, so the carry // would be 2 1-bits - #[test_case("zz", None; "invalid encoding-1")] + #[case::invalid_encoding_1("zz", None)] // this is an even more specific example - it'd decode as 00000000 11 - #[test_case("c0", None; "invalid encoding-2")] - - fn decode(enc: &str, dec: Option<&[u8]>) { + #[case::invalid_encoding_2("c0", None)] + #[test] + fn decode(#[case] enc: &str, #[case] dec: Option<&[u8]>) { match dec { Some(dec) => { // The decode needs to match what's passed in dec diff --git a/tvix/nix-compat/src/nixhash/mod.rs b/tvix/nix-compat/src/nixhash/mod.rs index 97b40aa0b9..d86cb8b79f 100644 --- a/tvix/nix-compat/src/nixhash/mod.rs +++ b/tvix/nix-compat/src/nixhash/mod.rs @@ -1,6 +1,8 @@ use crate::nixbase32; use bstr::ByteSlice; use data_encoding::{BASE64, BASE64_NOPAD, HEXLOWER}; +use serde::Deserialize; +use serde::Serialize; use std::cmp::Ordering; use std::fmt::Display; use thiserror; @@ -50,7 +52,7 @@ impl Display for NixHash { } /// convenience Result type for all nixhash parsing Results. -pub type Result<V> = std::result::Result<V, Error>; +pub type NixHashResult<V> = std::result::Result<V, Error>; impl NixHash { /// returns the algo as [HashAlgo]. @@ -118,16 +120,39 @@ impl TryFrom<(HashAlgo, &[u8])> for NixHash { /// Constructs a new [NixHash] by specifying [HashAlgo] and digest. /// It can fail if the passed digest length doesn't match what's expected for /// the passed algo. - fn try_from(value: (HashAlgo, &[u8])) -> Result<Self> { + fn try_from(value: (HashAlgo, &[u8])) -> NixHashResult<Self> { let (algo, digest) = value; from_algo_and_digest(algo, digest) } } +impl<'de> Deserialize<'de> for NixHash { + fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> + where + D: serde::Deserializer<'de>, + { + let str: &'de str = Deserialize::deserialize(deserializer)?; + from_str(str, None).map_err(|_| { + serde::de::Error::invalid_value(serde::de::Unexpected::Str(str), &"NixHash") + }) + } +} + +impl Serialize for NixHash { + fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> + where + S: serde::Serializer, + { + // encode as SRI + let string = format!("{}-{}", self.algo(), BASE64.encode(self.digest_as_bytes())); + string.serialize(serializer) + } +} + /// Constructs a new [NixHash] by specifying [HashAlgo] and digest. /// It can fail if the passed digest length doesn't match what's expected for /// the passed algo. -pub fn from_algo_and_digest(algo: HashAlgo, digest: &[u8]) -> Result<NixHash> { +pub fn from_algo_and_digest(algo: HashAlgo, digest: &[u8]) -> NixHashResult<NixHash> { if digest.len() != algo.digest_length() { return Err(Error::InvalidEncodedDigestLength(digest.len(), algo)); } @@ -180,7 +205,7 @@ pub enum Error { /// The hash is communicated out-of-band, but might also be in-band (in the /// case of a nix hash string or SRI), in which it needs to be consistent with the /// one communicated out-of-band. -pub fn from_str(s: &str, algo_str: Option<&str>) -> Result<NixHash> { +pub fn from_str(s: &str, algo_str: Option<&str>) -> NixHashResult<NixHash> { // if algo_str is some, parse or bail out let algo: Option<HashAlgo> = if let Some(algo_str) = algo_str { Some(algo_str.try_into()?) @@ -230,7 +255,7 @@ pub fn from_str(s: &str, algo_str: Option<&str>) -> Result<NixHash> { } /// Parses a Nix hash string ($algo:$digest) to a NixHash. -pub fn from_nix_str(s: &str) -> Result<NixHash> { +pub fn from_nix_str(s: &str) -> NixHashResult<NixHash> { if let Some(rest) = s.strip_prefix("sha1:") { decode_digest(rest.as_bytes(), HashAlgo::Sha1) } else if let Some(rest) = s.strip_prefix("sha256:") { @@ -250,7 +275,7 @@ pub fn from_nix_str(s: &str) -> Result<NixHash> { /// It instead simply cuts everything off after the expected length for the /// specified algo, and tries to parse the rest in permissive base64 (allowing /// missing padding). -pub fn from_sri_str(s: &str) -> Result<NixHash> { +pub fn from_sri_str(s: &str) -> NixHashResult<NixHash> { // split at the first occurence of "-" let (algo_str, digest_str) = s .split_once('-') @@ -294,7 +319,7 @@ pub fn from_sri_str(s: &str) -> Result<NixHash> { /// Decode a plain digest depending on the hash algo specified externally. /// hexlower, nixbase32 and base64 encodings are supported - the encoding is /// inferred from the input length. -fn decode_digest(s: &[u8], algo: HashAlgo) -> Result<NixHash> { +fn decode_digest(s: &[u8], algo: HashAlgo) -> NixHashResult<NixHash> { // for the chosen hash algo, calculate the expected (decoded) digest length // (as bytes) let digest = if s.len() == HEXLOWER.encode_len(algo.digest_length()) { @@ -322,7 +347,7 @@ mod tests { }; use data_encoding::{BASE64, BASE64_NOPAD, HEXLOWER}; use hex_literal::hex; - use test_case::test_case; + use rstest::rstest; const DIGEST_SHA1: [u8; 20] = hex!("6016777997c30ab02413cf5095622cd7924283ac"); const DIGEST_SHA256: [u8; 32] = @@ -355,11 +380,12 @@ mod tests { } /// Test parsing a hash string in various formats, and also when/how the out-of-band algo is needed. - #[test_case(&NixHash::Sha1(DIGEST_SHA1); "sha1")] - #[test_case(&NixHash::Sha256(DIGEST_SHA256); "sha256")] - #[test_case(&NixHash::Sha512(Box::new(DIGEST_SHA512)); "sha512")] - #[test_case(&NixHash::Md5(DIGEST_MD5); "md5")] - fn from_str(expected_hash: &NixHash) { + #[rstest] + #[case::sha1(&NixHash::Sha1(DIGEST_SHA1))] + #[case::sha256(&NixHash::Sha256(DIGEST_SHA256))] + #[case::sha512(&NixHash::Sha512(Box::new(DIGEST_SHA512)))] + #[case::md5(&NixHash::Md5(DIGEST_MD5))] + fn from_str(#[case] expected_hash: &NixHash) { let algo = &expected_hash.algo(); let digest = expected_hash.digest_as_bytes(); // parse SRI @@ -465,12 +491,13 @@ mod tests { } /// Test parsing sha512 SRI hash with various paddings, Nix accepts all of them. - #[test_case("sha512-7g91TBvYoYQorRTqo+rYD/i5YnWvUBLnqDhPHxBJDaBW7smuPMeRp6E6JOFuVN9bzN0QnH1ToUU0u9c2CjALEQ"; "no padding")] - #[test_case("sha512-7g91TBvYoYQorRTqo+rYD/i5YnWvUBLnqDhPHxBJDaBW7smuPMeRp6E6JOFuVN9bzN0QnH1ToUU0u9c2CjALEQ="; "too little padding")] - #[test_case("sha512-7g91TBvYoYQorRTqo+rYD/i5YnWvUBLnqDhPHxBJDaBW7smuPMeRp6E6JOFuVN9bzN0QnH1ToUU0u9c2CjALEQ=="; "correct padding")] - #[test_case("sha512-7g91TBvYoYQorRTqo+rYD/i5YnWvUBLnqDhPHxBJDaBW7smuPMeRp6E6JOFuVN9bzN0QnH1ToUU0u9c2CjALEQ==="; "too much padding")] - #[test_case("sha512-7g91TBvYoYQorRTqo+rYD/i5YnWvUBLnqDhPHxBJDaBW7smuPMeRp6E6JOFuVN9bzN0QnH1ToUU0u9c2CjALEQ== cheesecake"; "additional suffix ignored")] - fn from_sri_str_sha512_paddings(sri_str: &str) { + #[rstest] + #[case::no_padding("sha512-7g91TBvYoYQorRTqo+rYD/i5YnWvUBLnqDhPHxBJDaBW7smuPMeRp6E6JOFuVN9bzN0QnH1ToUU0u9c2CjALEQ")] + #[case::too_little_padding("sha512-7g91TBvYoYQorRTqo+rYD/i5YnWvUBLnqDhPHxBJDaBW7smuPMeRp6E6JOFuVN9bzN0QnH1ToUU0u9c2CjALEQ=")] + #[case::correct_padding("sha512-7g91TBvYoYQorRTqo+rYD/i5YnWvUBLnqDhPHxBJDaBW7smuPMeRp6E6JOFuVN9bzN0QnH1ToUU0u9c2CjALEQ==")] + #[case::too_much_padding("sha512-7g91TBvYoYQorRTqo+rYD/i5YnWvUBLnqDhPHxBJDaBW7smuPMeRp6E6JOFuVN9bzN0QnH1ToUU0u9c2CjALEQ===")] + #[case::additional_suffix_ignored("sha512-7g91TBvYoYQorRTqo+rYD/i5YnWvUBLnqDhPHxBJDaBW7smuPMeRp6E6JOFuVN9bzN0QnH1ToUU0u9c2CjALEQ== cheesecake")] + fn from_sri_str_sha512_paddings(#[case] sri_str: &str) { let nix_hash = nixhash::from_sri_str(sri_str).expect("must succeed"); assert_eq!(HashAlgo::Sha512, nix_hash.algo()); @@ -556,4 +583,20 @@ mod tests { // not passing SRI, but hash algo out of band should fail nixhash::from_str(weird_base64, Some("sha256")).expect_err("must fail"); } + + #[test] + fn serialize_deserialize() { + let nixhash_actual = NixHash::Sha256(hex!( + "b3271e24c5049270430872bc786b3aad45372109fe1e741f5117c2ac3c583daf" + )); + let nixhash_str_json = "\"sha256-syceJMUEknBDCHK8eGs6rUU3IQn+HnQfURfCrDxYPa8=\""; + + let serialized = serde_json::to_string(&nixhash_actual).expect("can serialize"); + + assert_eq!(nixhash_str_json, &serialized); + + let deserialized: NixHash = + serde_json::from_str(nixhash_str_json).expect("must deserialize"); + assert_eq!(&nixhash_actual, &deserialized); + } } diff --git a/tvix/nix-compat/src/path_info.rs b/tvix/nix-compat/src/path_info.rs new file mode 100644 index 0000000000..f289ebde33 --- /dev/null +++ b/tvix/nix-compat/src/path_info.rs @@ -0,0 +1,121 @@ +use crate::{nixbase32, nixhash::NixHash, store_path::StorePathRef}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; + +/// Represents information about a Store Path that Nix provides inside the build +/// if the exportReferencesGraph feature is used. +/// This is not to be confused with the format Nix uses in its `nix path-info` command. +/// It includes some more fields, like `registrationTime`, `signatures` and `ultimate`, +/// does not include the `closureSize` and encodes `narHash` as SRI. +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +pub struct ExportedPathInfo<'a> { + #[serde(rename = "closureSize")] + pub closure_size: u64, + + #[serde( + rename = "narHash", + serialize_with = "to_nix_nixbase32_string", + deserialize_with = "from_nix_nixbase32_string" + )] + pub nar_sha256: [u8; 32], + + #[serde(rename = "narSize")] + pub nar_size: u64, + + #[serde(borrow)] + pub path: StorePathRef<'a>, + + /// The list of other Store Paths this Store Path refers to. + /// StorePathRef does Ord by the nixbase32-encoded string repr, so this is correct. + pub references: BTreeSet<StorePathRef<'a>>, + // more recent versions of Nix also have a `valid: true` field here, Nix 2.3 doesn't, + // and nothing seems to use it. +} + +/// ExportedPathInfo are ordered by their `path` field. +impl Ord for ExportedPathInfo<'_> { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.path.cmp(&other.path) + } +} + +impl PartialOrd for ExportedPathInfo<'_> { + fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { + Some(self.cmp(other)) + } +} + +fn to_nix_nixbase32_string<S>(v: &[u8; 32], serializer: S) -> Result<S::Ok, S::Error> +where + S: serde::Serializer, +{ + let string = NixHash::Sha256(*v).to_nix_nixbase32_string(); + string.serialize(serializer) +} + +/// The length of a sha256 digest, nixbase32-encoded. +const NIXBASE32_SHA256_ENCODE_LEN: usize = nixbase32::encode_len(32); + +fn from_nix_nixbase32_string<'de, D>(deserializer: D) -> Result<[u8; 32], D::Error> +where + D: serde::Deserializer<'de>, +{ + let str: &'de str = Deserialize::deserialize(deserializer)?; + + let digest_str = str.strip_prefix("sha256:").ok_or_else(|| { + serde::de::Error::invalid_value(serde::de::Unexpected::Str(str), &"sha256:โฆ") + })?; + + let digest_str: [u8; NIXBASE32_SHA256_ENCODE_LEN] = + digest_str.as_bytes().try_into().map_err(|_| { + serde::de::Error::invalid_value(serde::de::Unexpected::Str(str), &"valid digest len") + })?; + + let digest: [u8; 32] = nixbase32::decode_fixed(digest_str).map_err(|_| { + serde::de::Error::invalid_value(serde::de::Unexpected::Str(str), &"valid nixbase32") + })?; + + Ok(digest) +} + +#[cfg(test)] +mod tests { + use hex_literal::hex; + + use super::*; + + /// Ensure we can create the same JSON as the exportReferencesGraph feature + #[test] + fn serialize_deserialize() { + // JSON extracted from a build of + // stdenv.mkDerivation { name = "hello"; __structuredAttrs = true; exportReferencesGraph.blub = [ pkgs.hello ]; nativeBuildInputs = [pkgs.jq]; buildCommand = "jq -rc .blub $NIX_ATTRS_JSON_FILE > $out"; } + let pathinfos_str_json = r#"[{"closureSize":1828984,"narHash":"sha256:11vm2x1ajhzsrzw7lsyss51mmr3b6yll9wdjn51bh7liwkpc8ila","narSize":1828984,"path":"/nix/store/7n0mbqydcipkpbxm24fab066lxk68aqk-libunistring-1.1","references":["/nix/store/7n0mbqydcipkpbxm24fab066lxk68aqk-libunistring-1.1"]},{"closureSize":32696176,"narHash":"sha256:0alzbhjxdcsmr1pk7z0bdh46r2xpq3xs3k9y82bi4bx5pklcvw5x","narSize":226560,"path":"/nix/store/dbghhbq1x39yxgkv3vkgfwbxrmw9nfzi-hello-2.12.1","references":["/nix/store/dbghhbq1x39yxgkv3vkgfwbxrmw9nfzi-hello-2.12.1","/nix/store/ddwyrxif62r8n6xclvskjyy6szdhvj60-glibc-2.39-5"]},{"closureSize":32469616,"narHash":"sha256:1zw5p05fh0k836ybfxkskv8apcv2m3pm2wa6y90wqn5w5kjyj13c","narSize":30119936,"path":"/nix/store/ddwyrxif62r8n6xclvskjyy6szdhvj60-glibc-2.39-5","references":["/nix/store/ddwyrxif62r8n6xclvskjyy6szdhvj60-glibc-2.39-5","/nix/store/rxganm4ibf31qngal3j3psp20mak37yy-xgcc-13.2.0-libgcc","/nix/store/s32cldbh9pfzd9z82izi12mdlrw0yf8q-libidn2-2.3.7"]},{"closureSize":159560,"narHash":"sha256:10q8iyvfmpfck3yiisnj1j8vp6lq3km17r26sr95zpdf9mgmk69s","narSize":159560,"path":"/nix/store/rxganm4ibf31qngal3j3psp20mak37yy-xgcc-13.2.0-libgcc","references":[]},{"closureSize":2190120,"narHash":"sha256:1cv997nzxbd91jhmzwnhxa1ahlzp5ffli8m4a5npcq8zg0vb1kwg","narSize":361136,"path":"/nix/store/s32cldbh9pfzd9z82izi12mdlrw0yf8q-libidn2-2.3.7","references":["/nix/store/7n0mbqydcipkpbxm24fab066lxk68aqk-libunistring-1.1","/nix/store/s32cldbh9pfzd9z82izi12mdlrw0yf8q-libidn2-2.3.7"]}]"#; + + // We ensure it roundtrips (to check the sorting is correct) + let deserialized: BTreeSet<ExportedPathInfo> = + serde_json::from_str(pathinfos_str_json).expect("must serialize"); + + let serialized_again = serde_json::to_string(&deserialized).expect("must deserialize"); + assert_eq!(pathinfos_str_json, serialized_again); + + // Also compare one specific item to be populated as expected. + assert_eq!( + &ExportedPathInfo { + closure_size: 1828984, + nar_sha256: hex!( + "8a46c4eee4911eb842b1b2f144a9376be45a43d1da6b7af8cffa43a942177587" + ), + nar_size: 1828984, + path: StorePathRef::from_bytes( + b"7n0mbqydcipkpbxm24fab066lxk68aqk-libunistring-1.1" + ) + .expect("must parse"), + references: BTreeSet::from_iter([StorePathRef::from_bytes( + b"7n0mbqydcipkpbxm24fab066lxk68aqk-libunistring-1.1" + ) + .unwrap()]), + }, + deserialized.first().unwrap() + ); + } +} diff --git a/tvix/nix-compat/src/store_path/mod.rs b/tvix/nix-compat/src/store_path/mod.rs index 71b7bdcc74..707c41a92d 100644 --- a/tvix/nix-compat/src/store_path/mod.rs +++ b/tvix/nix-compat/src/store_path/mod.rs @@ -56,7 +56,7 @@ pub enum Error { #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct StorePath { digest: [u8; DIGEST_SIZE], - name: String, + name: Box<str>, } impl StorePath { @@ -65,7 +65,7 @@ impl StorePath { } pub fn name(&self) -> &str { - self.name.as_ref() + &self.name } pub fn as_ref(&self) -> StorePathRef<'_> { @@ -86,22 +86,7 @@ impl PartialOrd for StorePath { /// of the nixbase32-encoded string. impl Ord for StorePath { fn cmp(&self, other: &Self) -> std::cmp::Ordering { - let order = self.digest.iter().rev().cmp(other.digest.iter().rev()); - - // This order must match the order of the nixbase32 encoded digests. - #[cfg(debug_assertions)] - { - let self_hash = nixbase32::encode(&self.digest); - let other_hash = nixbase32::encode(&other.digest); - let canonical_order = self_hash.cmp(&other_hash); - assert_eq!( - order, canonical_order, - "Ordering of nixbase32 differs, {:?} instead of {:?}:\n{:?}\n{:?}", - order, canonical_order, self_hash, other_hash - ); - } - - order + self.as_ref().cmp(&other.as_ref()) } } @@ -191,10 +176,21 @@ pub struct StorePathRef<'a> { impl<'a> From<&'a StorePath> for StorePathRef<'a> { fn from(&StorePath { digest, ref name }: &'a StorePath) -> Self { - StorePathRef { - digest, - name: name.as_ref(), - } + StorePathRef { digest, name } + } +} + +impl<'a> PartialOrd for StorePathRef<'a> { + fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { + Some(self.cmp(other)) + } +} + +/// `StorePathRef`s are sorted by their reverse digest to match the sorting order +/// of the nixbase32-encoded string. +impl<'a> Ord for StorePathRef<'a> { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.digest.iter().rev().cmp(other.digest.iter().rev()) } } @@ -210,7 +206,7 @@ impl<'a> StorePathRef<'a> { pub fn to_owned(&self) -> StorePath { StorePath { digest: self.digest, - name: self.name.to_owned(), + name: self.name.into(), } } @@ -275,7 +271,7 @@ impl<'a> StorePathRef<'a> { } } -impl<'de> Deserialize<'de> for StorePathRef<'de> { +impl<'de: 'a, 'a> Deserialize<'de> for StorePathRef<'a> { fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> where D: serde::Deserializer<'de>, @@ -304,8 +300,7 @@ impl Serialize for StorePathRef<'_> { } } -/// NAME_CHARS contains `true` for bytes that are valid in store path names, -/// not accounting for '.' being permitted only past the first character. +/// NAME_CHARS contains `true` for bytes that are valid in store path names. static NAME_CHARS: [bool; 256] = { let mut tbl = [false; 256]; let mut c = 0; @@ -333,10 +328,6 @@ pub(crate) fn validate_name(s: &(impl AsRef<[u8]> + ?Sized)) -> Result<&str, Err return Err(Error::InvalidLength); } - if s[0] == b'.' { - return Err(Error::InvalidName(s.to_vec(), 0)); - } - let mut valid = true; for &c in s { valid = valid && NAME_CHARS[c as usize]; @@ -373,15 +364,23 @@ impl fmt::Display for StorePathRef<'_> { #[cfg(test)] mod tests { + use super::Error; use std::cmp::Ordering; use std::path::PathBuf; - use crate::store_path::{StorePathRef, DIGEST_SIZE}; + use crate::store_path::{StorePath, StorePathRef, DIGEST_SIZE}; use hex_literal::hex; use pretty_assertions::assert_eq; - use test_case::test_case; + use rstest::rstest; + use serde::Deserialize; - use super::{Error, StorePath}; + #[derive(Deserialize)] + /// An example struct, holding a StorePathRef. + /// Used to test deserializing StorePathRef. + struct Container<'a> { + #[serde(borrow)] + store_path: StorePathRef<'a>, + } #[test] fn happy_path() { @@ -392,7 +391,7 @@ mod tests { let expected_digest: [u8; DIGEST_SIZE] = hex!("8a12321522fd91efbd60ebb2481af88580f61600"); - assert_eq!("net-tools-1.60_p20170221182432", nixpath.name); + assert_eq!("net-tools-1.60_p20170221182432", nixpath.name()); assert_eq!(nixpath.digest, expected_digest); assert_eq!(example_nix_path_str, nixpath.to_string()) @@ -439,15 +438,18 @@ mod tests { } } - /// This is the store path rejected when `nix-store --add`'ing an + /// This is the store path *accepted* when `nix-store --add`'ing an /// empty `.gitignore` file. /// - /// Nix 2.4 accidentally dropped this behaviour, but this is considered a bug. - /// See https://github.com/NixOS/nix/pull/9095. + /// Nix 2.4 accidentally permitted this behaviour, but the revert came + /// too late to beat Hyrum's law. It is now considered permissible. + /// + /// https://github.com/NixOS/nix/pull/9095 (revert) + /// https://github.com/NixOS/nix/pull/9867 (revert-of-revert) #[test] fn starts_with_dot() { StorePath::from_bytes(b"fli4bwscgna7lpm7v5xgnjxrxh0yc7ra-.gitignore") - .expect_err("must fail"); + .expect("must succeed"); } #[test] @@ -550,6 +552,32 @@ mod tests { let store_path_str_json = "\"/nix/store/00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432\""; + let store_path: StorePathRef<'_> = + serde_json::from_str(store_path_str_json).expect("valid json"); + + assert_eq!( + "/nix/store/00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432", + store_path.to_absolute_path() + ); + } + + #[test] + fn deserialize_ref_container() { + let str_json = "{\"store_path\":\"/nix/store/00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432\"}"; + + let container: Container<'_> = serde_json::from_str(str_json).expect("must deserialize"); + + assert_eq!( + "/nix/store/00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432", + container.store_path.to_absolute_path() + ); + } + + #[test] + fn deserialize_owned() { + let store_path_str_json = + "\"/nix/store/00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432\""; + let store_path: StorePath = serde_json::from_str(store_path_str_json).expect("valid json"); assert_eq!( @@ -558,25 +586,29 @@ mod tests { ); } - #[test_case( + #[rstest] + #[case::without_prefix( "/nix/store/00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432", - (StorePath::from_bytes(b"00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432").unwrap(), PathBuf::new()) - ; "without prefix")] - #[test_case( + StorePath::from_bytes(b"00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432").unwrap(), PathBuf::new())] + #[case::without_prefix_but_trailing_slash( "/nix/store/00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432/", - (StorePath::from_bytes(b"00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432").unwrap(), PathBuf::new()) - ; "without prefix, but trailing slash")] - #[test_case( + StorePath::from_bytes(b"00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432").unwrap(), PathBuf::new())] + #[case::with_prefix( "/nix/store/00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432/bin/arp", - (StorePath::from_bytes(b"00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432").unwrap(), PathBuf::from("bin/arp")) - ; "with prefix")] - #[test_case( + StorePath::from_bytes(b"00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432").unwrap(), PathBuf::from("bin/arp"))] + #[case::with_prefix_and_trailing_slash( "/nix/store/00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432/bin/arp/", - (StorePath::from_bytes(b"00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432").unwrap(), PathBuf::from("bin/arp/")) - ; "with prefix and trailing slash")] - fn from_absolute_path_full(s: &str, expected: (StorePath, PathBuf)) { - let actual = StorePath::from_absolute_path_full(s).expect("must succeed"); - assert_eq!(expected, actual); + StorePath::from_bytes(b"00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432").unwrap(), PathBuf::from("bin/arp/"))] + fn from_absolute_path_full( + #[case] s: &str, + #[case] exp_store_path: StorePath, + #[case] exp_path: PathBuf, + ) { + let (actual_store_path, actual_path) = + StorePath::from_absolute_path_full(s).expect("must succeed"); + + assert_eq!(exp_store_path, actual_store_path); + assert_eq!(exp_path, actual_path); } #[test] diff --git a/tvix/nix-compat/src/wire/bytes.rs b/tvix/nix-compat/src/wire/bytes.rs deleted file mode 100644 index f2fe30083b..0000000000 --- a/tvix/nix-compat/src/wire/bytes.rs +++ /dev/null @@ -1,130 +0,0 @@ -use std::ops::RangeBounds; - -use tokio::io::AsyncReadExt; - -use super::primitive; - -#[allow(dead_code)] -/// Read a limited number of bytes from the AsyncRead. -/// Rejects reading more than `allowed_size` bytes of payload. -/// Internally takes care of dealing with the padding, so the returned `Vec<u8>` -/// only contains the payload. -/// This always buffers the entire contents into memory, we'll add a streaming -/// version later. -pub async fn read_bytes<R, S>(r: &mut R, allowed_size: S) -> std::io::Result<Vec<u8>> -where - R: AsyncReadExt + Unpin, - S: RangeBounds<u64>, -{ - // read the length field - let len = primitive::read_u64(r).await?; - - if !allowed_size.contains(&len) { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "signalled package size not in allowed range", - )); - } - - // calculate the total length, including padding. - // byte packets are padded to 8 byte blocks each. - let padded_len = if len % 8 == 0 { - len - } else { - len + (8 - len % 8) - }; - - let mut limited_reader = r.take(padded_len); - - let mut buf = Vec::new(); - - let s = limited_reader.read_to_end(&mut buf).await?; - - // make sure we got exactly the number of bytes, and not less. - if s as u64 != padded_len { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "got less bytes than expected", - )); - } - - let (_content, padding) = buf.split_at(len as usize); - - // ensure the padding is all zeroes. - if !padding.iter().all(|e| *e == b'\0') { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "padding is not all zeroes", - )); - } - - // return the data without the padding - buf.truncate(len as usize); - Ok(buf) -} - -#[allow(dead_code)] -/// Read an unlimited number of bytes from the AsyncRead. -/// Note this can exhaust memory. -/// Internally uses [read_bytes], which takes care of dealing with the padding, -/// so the returned `Vec<u8>` only contains the payload. -pub async fn read_bytes_unchecked<R: AsyncReadExt + Unpin>(r: &mut R) -> std::io::Result<Vec<u8>> { - read_bytes(r, 0u64..).await -} - -#[cfg(test)] -mod tests { - use tokio_test::io::Builder; - - use super::*; - use hex_literal::hex; - - #[tokio::test] - async fn test_read_8_bytes_unchecked() { - let mut mock = Builder::new() - .read(&8u64.to_le_bytes()) - .read(&12345678u64.to_le_bytes()) - .build(); - - assert_eq!( - &12345678u64.to_le_bytes(), - read_bytes_unchecked(&mut mock).await.unwrap().as_slice() - ); - } - - #[tokio::test] - async fn test_read_9_bytes_unchecked() { - let mut mock = Builder::new() - .read(&9u64.to_le_bytes()) - .read(&hex!("01020304050607080900000000000000")) - .build(); - - assert_eq!( - hex!("010203040506070809"), - read_bytes_unchecked(&mut mock).await.unwrap().as_slice() - ); - } - - #[tokio::test] - async fn test_read_0_bytes_unchecked() { - // A empty byte packet is essentially just the 0 length field. - // No data is read, and there's zero padding. - let mut mock = Builder::new().read(&0u64.to_le_bytes()).build(); - - assert_eq!( - hex!(""), - read_bytes_unchecked(&mut mock).await.unwrap().as_slice() - ); - } - - #[tokio::test] - /// Ensure we don't read any further than the size field if the length - /// doesn't match the range we want to accept. - async fn test_reject_too_large() { - let mut mock = Builder::new().read(&100u64.to_le_bytes()).build(); - - read_bytes(&mut mock, 10..10) - .await - .expect_err("expect this to fail"); - } -} diff --git a/tvix/nix-compat/src/wire/bytes/mod.rs b/tvix/nix-compat/src/wire/bytes/mod.rs new file mode 100644 index 0000000000..2ed071e379 --- /dev/null +++ b/tvix/nix-compat/src/wire/bytes/mod.rs @@ -0,0 +1,283 @@ +use std::{ + io::{Error, ErrorKind}, + mem::MaybeUninit, + ops::RangeInclusive, +}; +use tokio::io::{self, AsyncReadExt, AsyncWriteExt, ReadBuf}; + +pub(crate) mod reader; +pub use reader::BytesReader; +mod writer; +pub use writer::BytesWriter; + +/// 8 null bytes, used to write out padding. +const EMPTY_BYTES: &[u8; 8] = &[0u8; 8]; + +/// The length of the size field, in bytes is always 8. +const LEN_SIZE: usize = 8; + +/// Read a "bytes wire packet" from the AsyncRead. +/// Rejects reading more than `allowed_size` bytes of payload. +/// +/// The packet is made up of three parts: +/// - a length header, u64, LE-encoded +/// - the payload itself +/// - null bytes to the next 8 byte boundary +/// +/// Ensures the payload size fits into the `allowed_size` passed, +/// and that the padding is actual null bytes. +/// +/// On success, the returned `Vec<u8>` only contains the payload itself. +/// On failure (for example if a too large byte packet was sent), the reader +/// becomes unusable. +/// +/// This buffers the entire payload into memory, +/// a streaming version is available at [crate::wire::bytes::BytesReader]. +pub async fn read_bytes<R: ?Sized>( + r: &mut R, + allowed_size: RangeInclusive<usize>, +) -> io::Result<Vec<u8>> +where + R: AsyncReadExt + Unpin, +{ + // read the length field + let len = r.read_u64_le().await?; + let len: usize = len + .try_into() + .ok() + .filter(|len| allowed_size.contains(len)) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "signalled package size not in allowed range", + ) + })?; + + // calculate the total length, including padding. + // byte packets are padded to 8 byte blocks each. + let padded_len = padding_len(len as u64) as u64 + (len as u64); + let mut limited_reader = r.take(padded_len); + + let mut buf = Vec::new(); + + let s = limited_reader.read_to_end(&mut buf).await?; + + // make sure we got exactly the number of bytes, and not less. + if s as u64 != padded_len { + return Err(io::ErrorKind::UnexpectedEof.into()); + } + + let (_content, padding) = buf.split_at(len); + + // ensure the padding is all zeroes. + if padding.iter().any(|&b| b != 0) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "padding is not all zeroes", + )); + } + + // return the data without the padding + buf.truncate(len); + Ok(buf) +} + +pub(crate) async fn read_bytes_buf<'a, const N: usize, R: ?Sized>( + reader: &mut R, + buf: &'a mut [MaybeUninit<u8>; N], + allowed_size: RangeInclusive<usize>, +) -> io::Result<&'a [u8]> +where + R: AsyncReadExt + Unpin, +{ + assert_eq!(N % 8, 0); + assert!(*allowed_size.end() <= N); + + let len = reader.read_u64_le().await?; + let len: usize = len + .try_into() + .ok() + .filter(|len| allowed_size.contains(len)) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "signalled package size not in allowed range", + ) + })?; + + let buf_len = (len + 7) & !7; + let buf = { + let mut read_buf = ReadBuf::uninit(&mut buf[..buf_len]); + + while read_buf.filled().len() < buf_len { + reader.read_buf(&mut read_buf).await?; + } + + // ReadBuf::filled does not pass the underlying buffer's lifetime through, + // so we must make a trip to hell. + // + // SAFETY: `read_buf` is filled up to `buf_len`, and we verify that it is + // still pointing at the same underlying buffer. + unsafe { + assert_eq!(read_buf.filled().as_ptr(), buf.as_ptr() as *const u8); + assume_init_bytes(&buf[..buf_len]) + } + }; + + if buf[len..buf_len].iter().any(|&b| b != 0) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "padding is not all zeroes", + )); + } + + Ok(&buf[..len]) +} + +/// SAFETY: The bytes have to actually be initialized. +unsafe fn assume_init_bytes(slice: &[MaybeUninit<u8>]) -> &[u8] { + &*(slice as *const [MaybeUninit<u8>] as *const [u8]) +} + +/// Read a "bytes wire packet" of from the AsyncRead and tries to parse as string. +/// Internally uses [read_bytes]. +/// Rejects reading more than `allowed_size` bytes of payload. +pub async fn read_string<R>(r: &mut R, allowed_size: RangeInclusive<usize>) -> io::Result<String> +where + R: AsyncReadExt + Unpin, +{ + let bytes = read_bytes(r, allowed_size).await?; + String::from_utf8(bytes).map_err(|e| Error::new(ErrorKind::InvalidData, e)) +} + +/// Writes a "bytes wire packet" to a (hopefully buffered) [AsyncWriteExt]. +/// +/// Accepts anything implementing AsRef<[u8]> as payload. +/// +/// See [read_bytes] for a description of the format. +/// +/// Note: if performance matters to you, make sure your +/// [AsyncWriteExt] handle is buffered. This function is quite +/// write-intesive. +pub async fn write_bytes<W: AsyncWriteExt + Unpin, B: AsRef<[u8]>>( + w: &mut W, + b: B, +) -> io::Result<()> { + // write the size packet. + w.write_u64_le(b.as_ref().len() as u64).await?; + + // write the payload + w.write_all(b.as_ref()).await?; + + // write padding if needed + let padding_len = padding_len(b.as_ref().len() as u64) as usize; + if padding_len != 0 { + w.write_all(&EMPTY_BYTES[..padding_len]).await?; + } + Ok(()) +} + +/// Computes the number of bytes we should add to len (a length in +/// bytes) to be aligned on 64 bits (8 bytes). +fn padding_len(len: u64) -> u8 { + let aligned = len.wrapping_add(7) & !7; + aligned.wrapping_sub(len) as u8 +} + +#[cfg(test)] +mod tests { + use tokio_test::{assert_ok, io::Builder}; + + use super::*; + use hex_literal::hex; + + /// The maximum length of bytes packets we're willing to accept in the test + /// cases. + const MAX_LEN: usize = 1024; + + #[tokio::test] + async fn test_read_8_bytes() { + let mut mock = Builder::new() + .read(&8u64.to_le_bytes()) + .read(&12345678u64.to_le_bytes()) + .build(); + + assert_eq!( + &12345678u64.to_le_bytes(), + read_bytes(&mut mock, 0..=MAX_LEN).await.unwrap().as_slice() + ); + } + + #[tokio::test] + async fn test_read_9_bytes() { + let mut mock = Builder::new() + .read(&9u64.to_le_bytes()) + .read(&hex!("01020304050607080900000000000000")) + .build(); + + assert_eq!( + hex!("010203040506070809"), + read_bytes(&mut mock, 0..=MAX_LEN).await.unwrap().as_slice() + ); + } + + #[tokio::test] + async fn test_read_0_bytes() { + // A empty byte packet is essentially just the 0 length field. + // No data is read, and there's zero padding. + let mut mock = Builder::new().read(&0u64.to_le_bytes()).build(); + + assert_eq!( + hex!(""), + read_bytes(&mut mock, 0..=MAX_LEN).await.unwrap().as_slice() + ); + } + + #[tokio::test] + /// Ensure we don't read any further than the size field if the length + /// doesn't match the range we want to accept. + async fn test_read_reject_too_large() { + let mut mock = Builder::new().read(&100u64.to_le_bytes()).build(); + + read_bytes(&mut mock, 10..=10) + .await + .expect_err("expect this to fail"); + } + + #[tokio::test] + async fn test_write_bytes_no_padding() { + let input = hex!("6478696f34657661"); + let len = input.len() as u64; + let mut mock = Builder::new() + .write(&len.to_le_bytes()) + .write(&input) + .build(); + assert_ok!(write_bytes(&mut mock, &input).await) + } + #[tokio::test] + async fn test_write_bytes_with_padding() { + let input = hex!("322e332e3137"); + let len = input.len() as u64; + let mut mock = Builder::new() + .write(&len.to_le_bytes()) + .write(&hex!("322e332e31370000")) + .build(); + assert_ok!(write_bytes(&mut mock, &input).await) + } + + #[tokio::test] + async fn test_write_string() { + let input = "Hello, World!"; + let len = input.len() as u64; + let mut mock = Builder::new() + .write(&len.to_le_bytes()) + .write(&hex!("48656c6c6f2c20576f726c6421000000")) + .build(); + assert_ok!(write_bytes(&mut mock, &input).await) + } + + #[test] + fn padding_len_u64_max() { + assert_eq!(padding_len(u64::MAX), 1); + } +} diff --git a/tvix/nix-compat/src/wire/bytes/reader/mod.rs b/tvix/nix-compat/src/wire/bytes/reader/mod.rs new file mode 100644 index 0000000000..6bd376c06f --- /dev/null +++ b/tvix/nix-compat/src/wire/bytes/reader/mod.rs @@ -0,0 +1,684 @@ +use std::{ + future::Future, + io, + num::NonZeroU64, + ops::RangeBounds, + pin::Pin, + task::{self, ready, Poll}, +}; +use tokio::io::{AsyncBufRead, AsyncRead, AsyncReadExt, ReadBuf}; + +use trailer::{read_trailer, ReadTrailer, Trailer}; + +#[doc(hidden)] +pub use self::trailer::Pad; +pub(crate) use self::trailer::Tag; +mod trailer; + +/// Reads a "bytes wire packet" from the underlying reader. +/// The format is the same as in [crate::wire::bytes::read_bytes], +/// however this structure provides a [AsyncRead] interface, +/// allowing to not having to pass around the entire payload in memory. +/// +/// It is constructed by reading a size with [BytesReader::new], +/// and yields payload data until the end of the packet is reached. +/// +/// It will not return the final bytes before all padding has been successfully +/// consumed as well, but the full length of the reader must be consumed. +/// +/// If the data is not read all the way to the end, or an error is encountered, +/// the underlying reader is no longer usable and might return garbage. +#[derive(Debug)] +#[allow(private_bounds)] +pub struct BytesReader<R, T: Tag = Pad> { + state: State<R, T>, +} + +/// Split the `user_len` into `body_len` and `tail_len`, which are respectively +/// the non-terminal 8-byte blocks, and the โค8 bytes of user data contained in +/// the trailer block. +#[inline(always)] +fn split_user_len(user_len: NonZeroU64) -> (u64, u8) { + let n = user_len.get() - 1; + let body_len = n & !7; + let tail_len = (n & 7) as u8 + 1; + (body_len, tail_len) +} + +#[derive(Debug)] +enum State<R, T: Tag> { + /// Full 8-byte blocks are being read and released to the caller. + /// NOTE: The final 8-byte block is *always* part of the trailer. + Body { + reader: Option<R>, + consumed: u64, + /// The total length of all user data contained in both the body and trailer. + user_len: NonZeroU64, + }, + /// The trailer is in the process of being read. + ReadTrailer(ReadTrailer<R, T>), + /// The trailer has been fully read and validated, + /// and data can now be released to the caller. + ReleaseTrailer { consumed: u8, data: Trailer }, +} + +impl<R> BytesReader<R> +where + R: AsyncRead + Unpin, +{ + /// Constructs a new BytesReader, using the underlying passed reader. + pub async fn new<S: RangeBounds<u64>>(reader: R, allowed_size: S) -> io::Result<Self> { + BytesReader::new_internal(reader, allowed_size).await + } +} + +#[allow(private_bounds)] +impl<R, T: Tag> BytesReader<R, T> +where + R: AsyncRead + Unpin, +{ + /// Constructs a new BytesReader, using the underlying passed reader. + pub(crate) async fn new_internal<S: RangeBounds<u64>>( + mut reader: R, + allowed_size: S, + ) -> io::Result<Self> { + let size = reader.read_u64_le().await?; + + if !allowed_size.contains(&size) { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid size")); + } + + Ok(Self { + state: match NonZeroU64::new(size) { + Some(size) => State::Body { + reader: Some(reader), + consumed: 0, + user_len: size, + }, + None => State::ReleaseTrailer { + consumed: 0, + data: read_trailer::<R, T>(reader, 0).await?, + }, + }, + }) + } + + /// Returns whether there is any remaining data to be read. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Remaining data length, ie not including data already read. + /// + /// If the size has not been read yet, this is [None]. + pub fn len(&self) -> u64 { + match self.state { + State::Body { + consumed, user_len, .. + } => user_len.get() - consumed, + State::ReadTrailer(ref fut) => fut.len() as u64, + State::ReleaseTrailer { consumed, ref data } => data.len() as u64 - consumed as u64, + } + } +} + +#[allow(private_bounds)] +impl<R: AsyncRead + Unpin, T: Tag> AsyncRead for BytesReader<R, T> { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut task::Context, + buf: &mut ReadBuf, + ) -> Poll<io::Result<()>> { + let this = &mut self.state; + + loop { + match this { + State::Body { + reader, + consumed, + user_len, + } => { + let (body_len, tail_len) = split_user_len(*user_len); + let remaining = body_len - *consumed; + + let reader = if remaining == 0 { + let reader = reader.take().unwrap(); + *this = State::ReadTrailer(read_trailer(reader, tail_len)); + continue; + } else { + Pin::new(reader.as_mut().unwrap()) + }; + + let mut bytes_read = 0; + ready!(with_limited(buf, remaining, |buf| { + let ret = reader.poll_read(cx, buf); + bytes_read = buf.initialized().len(); + ret + }))?; + + *consumed += bytes_read as u64; + + return if bytes_read != 0 { + Ok(()) + } else { + Err(io::ErrorKind::UnexpectedEof.into()) + } + .into(); + } + State::ReadTrailer(fut) => { + *this = State::ReleaseTrailer { + consumed: 0, + data: ready!(Pin::new(fut).poll(cx))?, + }; + } + State::ReleaseTrailer { consumed, data } => { + let data = &data[*consumed as usize..]; + let data = &data[..usize::min(data.len(), buf.remaining())]; + + buf.put_slice(data); + *consumed += data.len() as u8; + + return Ok(()).into(); + } + } + } + } +} + +#[allow(private_bounds)] +impl<R: AsyncBufRead + Unpin, T: Tag> AsyncBufRead for BytesReader<R, T> { + fn poll_fill_buf(self: Pin<&mut Self>, cx: &mut task::Context) -> Poll<io::Result<&[u8]>> { + let this = &mut self.get_mut().state; + + loop { + match this { + // This state comes *after* the following case, + // but we can't keep it in logical order because + // that would lengthen the borrow lifetime. + State::Body { + reader, + consumed, + user_len, + } if { + let (body_len, _) = split_user_len(*user_len); + let remaining = body_len - *consumed; + + remaining == 0 + } => + { + let reader = reader.take().unwrap(); + let (_, tail_len) = split_user_len(*user_len); + + *this = State::ReadTrailer(read_trailer(reader, tail_len)); + } + State::Body { + reader, + consumed, + user_len, + } => { + let (body_len, _) = split_user_len(*user_len); + let remaining = body_len - *consumed; + + let reader = Pin::new(reader.as_mut().unwrap()); + + match ready!(reader.poll_fill_buf(cx))? { + &[] => { + return Err(io::ErrorKind::UnexpectedEof.into()).into(); + } + mut buf => { + if buf.len() as u64 > remaining { + buf = &buf[..remaining as usize]; + } + + return Ok(buf).into(); + } + } + } + State::ReadTrailer(fut) => { + *this = State::ReleaseTrailer { + consumed: 0, + data: ready!(Pin::new(fut).poll(cx))?, + }; + } + State::ReleaseTrailer { consumed, data } => { + return Ok(&data[*consumed as usize..]).into(); + } + } + } + } + + fn consume(mut self: Pin<&mut Self>, amt: usize) { + match &mut self.state { + State::Body { + reader, + consumed, + user_len, + } => { + let reader = Pin::new(reader.as_mut().unwrap()); + let (body_len, _) = split_user_len(*user_len); + + *consumed = consumed + .checked_add(amt as u64) + .filter(|&consumed| consumed <= body_len) + .expect("consumed out of bounds"); + + reader.consume(amt); + } + State::ReadTrailer(_) => unreachable!(), + State::ReleaseTrailer { consumed, data } => { + *consumed = amt + .checked_add(*consumed as usize) + .filter(|&consumed| consumed <= data.len()) + .expect("consumed out of bounds") as u8; + } + } + } +} + +/// Make a limited version of `buf`, consisting only of up to `n` bytes of the unfilled section, and call `f` with it. +/// After `f` returns, we propagate the filled cursor advancement back to `buf`. +fn with_limited<R>(buf: &mut ReadBuf, n: u64, f: impl FnOnce(&mut ReadBuf) -> R) -> R { + let mut nbuf = buf.take(n.try_into().unwrap_or(usize::MAX)); + let ptr = nbuf.initialized().as_ptr(); + let ret = f(&mut nbuf); + + // SAFETY: `ReadBuf::take` only returns the *unfilled* section of `buf`, + // so anything filled is new, initialized data. + // + // We verify that `nbuf` still points to the same buffer, + // so we're sure it hasn't been swapped out. + unsafe { + // ensure our buffer hasn't been swapped out + assert_eq!(nbuf.initialized().as_ptr(), ptr); + + let n = nbuf.filled().len(); + buf.assume_init(n); + buf.advance(n); + } + + ret +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use crate::wire::bytes::{padding_len, write_bytes}; + use hex_literal::hex; + use lazy_static::lazy_static; + use rstest::rstest; + use tokio::io::{AsyncReadExt, BufReader}; + use tokio_test::io::Builder; + + use super::*; + + /// The maximum length of bytes packets we're willing to accept in the test + /// cases. + const MAX_LEN: u64 = 1024; + + lazy_static! { + pub static ref LARGE_PAYLOAD: Vec<u8> = (0..255).collect::<Vec<u8>>().repeat(4 * 1024); + } + + /// Helper function, calling the (simpler) write_bytes with the payload. + /// We use this to create data we want to read from the wire. + async fn produce_packet_bytes(payload: &[u8]) -> Vec<u8> { + let mut exp = vec![]; + write_bytes(&mut exp, payload).await.unwrap(); + exp + } + + /// Read bytes packets of various length, and ensure read_to_end returns the + /// expected payload. + #[rstest] + #[case::empty(&[])] // empty bytes packet + #[case::size_1b(&[0xff])] // 1 bytes payload + #[case::size_8b(&hex!("0001020304050607"))] // 8 bytes payload (no padding) + #[case::size_9b(&hex!("000102030405060708"))] // 9 bytes payload (7 bytes padding) + #[case::size_1m(LARGE_PAYLOAD.as_slice())] // larger bytes packet + #[tokio::test] + async fn read_payload_correct(#[case] payload: &[u8]) { + let mut mock = Builder::new() + .read(&produce_packet_bytes(payload).await) + .build(); + + let mut r = BytesReader::new(&mut mock, ..=LARGE_PAYLOAD.len() as u64) + .await + .unwrap(); + let mut buf = Vec::new(); + r.read_to_end(&mut buf).await.expect("must succeed"); + + assert_eq!(payload, &buf[..]); + } + + /// Read bytes packets of various length, and ensure copy_buf reads the + /// expected payload. + #[rstest] + #[case::empty(&[])] // empty bytes packet + #[case::size_1b(&[0xff])] // 1 bytes payload + #[case::size_8b(&hex!("0001020304050607"))] // 8 bytes payload (no padding) + #[case::size_9b(&hex!("000102030405060708"))] // 9 bytes payload (7 bytes padding) + #[case::size_1m(LARGE_PAYLOAD.as_slice())] // larger bytes packet + #[tokio::test] + async fn read_payload_correct_readbuf(#[case] payload: &[u8]) { + let mut mock = BufReader::new( + Builder::new() + .read(&produce_packet_bytes(payload).await) + .build(), + ); + + let mut r = BytesReader::new(&mut mock, ..=LARGE_PAYLOAD.len() as u64) + .await + .unwrap(); + + let mut buf = Vec::new(); + tokio::io::copy_buf(&mut r, &mut buf) + .await + .expect("copy_buf must succeed"); + + assert_eq!(payload, &buf[..]); + } + + /// Fail if the bytes packet is larger than allowed + #[tokio::test] + async fn read_bigger_than_allowed_fail() { + let payload = LARGE_PAYLOAD.as_slice(); + let mut mock = Builder::new() + .read(&produce_packet_bytes(payload).await[0..8]) // We stop reading after the size packet + .build(); + + assert_eq!( + BytesReader::new(&mut mock, ..2048) + .await + .unwrap_err() + .kind(), + io::ErrorKind::InvalidData + ); + } + + /// Fail if the bytes packet is smaller than allowed + #[tokio::test] + async fn read_smaller_than_allowed_fail() { + let payload = &[0x00, 0x01, 0x02]; + let mut mock = Builder::new() + .read(&produce_packet_bytes(payload).await[0..8]) // We stop reading after the size packet + .build(); + + assert_eq!( + BytesReader::new(&mut mock, 1024..2048) + .await + .unwrap_err() + .kind(), + io::ErrorKind::InvalidData + ); + } + + /// Read the trailer immediately if there is no payload. + #[tokio::test] + async fn read_trailer_immediately() { + use crate::nar::wire::PadPar; + + let mut mock = Builder::new() + .read(&[0; 8]) + .read(&PadPar::PATTERN[8..]) + .build(); + + BytesReader::<_, PadPar>::new_internal(&mut mock, ..) + .await + .unwrap(); + + // The mock reader will panic if dropped without reading all data. + } + + /// Read the trailer even if we only read the exact payload size. + #[tokio::test] + async fn read_exact_trailer() { + use crate::nar::wire::PadPar; + + let mut mock = Builder::new() + .read(&16u64.to_le_bytes()) + .read(&[0x55; 16]) + .read(&PadPar::PATTERN[8..]) + .build(); + + let mut reader = BytesReader::<_, PadPar>::new_internal(&mut mock, ..) + .await + .unwrap(); + + let mut buf = [0; 16]; + reader.read_exact(&mut buf).await.unwrap(); + assert_eq!(buf, [0x55; 16]); + + // The mock reader will panic if dropped without reading all data. + } + + /// Fail if the padding is not all zeroes + #[tokio::test] + async fn read_fail_if_nonzero_padding() { + let payload = &[0x00, 0x01, 0x02]; + let mut packet_bytes = produce_packet_bytes(payload).await; + // Flip some bits in the padding + packet_bytes[12] = 0xff; + let mut mock = Builder::new().read(&packet_bytes).build(); // We stop reading after the faulty bit + + let mut r = BytesReader::new(&mut mock, ..MAX_LEN).await.unwrap(); + let mut buf = Vec::new(); + + r.read_to_end(&mut buf).await.expect_err("must fail"); + } + + /// Start a 9 bytes payload packet, but have the underlying reader return + /// EOF in the middle of the size packet (after 4 bytes). + /// We should get an unexpected EOF error, already when trying to read the + /// first byte (of payload) + #[tokio::test] + async fn read_9b_eof_during_size() { + let payload = &hex!("FF0102030405060708"); + let mut mock = Builder::new() + .read(&produce_packet_bytes(payload).await[..4]) + .build(); + + assert_eq!( + BytesReader::new(&mut mock, ..MAX_LEN) + .await + .expect_err("must fail") + .kind(), + io::ErrorKind::UnexpectedEof + ); + } + + /// Start a 9 bytes payload packet, but have the underlying reader return + /// EOF in the middle of the payload (4 bytes into the payload). + /// We should get an unexpected EOF error, after reading the first 4 bytes + /// (successfully). + #[tokio::test] + async fn read_9b_eof_during_payload() { + let payload = &hex!("FF0102030405060708"); + let mut mock = Builder::new() + .read(&produce_packet_bytes(payload).await[..8 + 4]) + .build(); + + let mut r = BytesReader::new(&mut mock, ..MAX_LEN).await.unwrap(); + let mut buf = [0; 9]; + + r.read_exact(&mut buf[..4]).await.expect("must succeed"); + + assert_eq!( + r.read_exact(&mut buf[4..=4]) + .await + .expect_err("must fail") + .kind(), + std::io::ErrorKind::UnexpectedEof + ); + } + + /// Start a 9 bytes payload packet, but don't supply the necessary padding. + /// This is expected to always fail before returning the final data. + #[rstest] + #[case::before_padding(8 + 9)] + #[case::during_padding(8 + 9 + 2)] + #[case::after_padding(8 + 9 + padding_len(9) as usize - 1)] + #[tokio::test] + async fn read_9b_eof_after_payload(#[case] offset: usize) { + let payload = &hex!("FF0102030405060708"); + let mut mock = Builder::new() + .read(&produce_packet_bytes(payload).await[..offset]) + .build(); + + let mut r = BytesReader::new(&mut mock, ..MAX_LEN).await.unwrap(); + + // read_exact of the payload *body* will succeed, but a subsequent read will + // return UnexpectedEof error. + assert_eq!(r.read_exact(&mut [0; 8]).await.unwrap(), 8); + assert_eq!( + r.read_exact(&mut [0]).await.unwrap_err().kind(), + std::io::ErrorKind::UnexpectedEof + ); + } + + /// Start a 9 bytes payload packet, but return an error after a certain position. + /// Ensure that error is propagated. + #[rstest] + #[case::during_size(4)] + #[case::before_payload(8)] + #[case::during_payload(8 + 4)] + #[case::before_padding(8 + 4)] + #[case::during_padding(8 + 9 + 2)] + #[tokio::test] + async fn propagate_error_from_reader(#[case] offset: usize) { + let payload = &hex!("FF0102030405060708"); + let mut mock = Builder::new() + .read(&produce_packet_bytes(payload).await[..offset]) + .read_error(std::io::Error::new(std::io::ErrorKind::Other, "foo")) + .build(); + + // Either length reading or data reading can fail, depending on which test case we're in. + let err: io::Error = async { + let mut r = BytesReader::new(&mut mock, ..MAX_LEN).await?; + let mut buf = Vec::new(); + + r.read_to_end(&mut buf).await?; + + Ok(()) + } + .await + .expect_err("must fail"); + + assert_eq!( + err.kind(), + std::io::ErrorKind::Other, + "error kind must match" + ); + + assert_eq!( + err.into_inner().unwrap().to_string(), + "foo", + "error payload must contain foo" + ); + } + + /// Start a 9 bytes payload packet, but return an error after a certain position. + /// Ensure that error is propagated (AsyncReadBuf case) + #[rstest] + #[case::during_size(4)] + #[case::before_payload(8)] + #[case::during_payload(8 + 4)] + #[case::before_padding(8 + 4)] + #[case::during_padding(8 + 9 + 2)] + #[tokio::test] + async fn propagate_error_from_reader_buffered(#[case] offset: usize) { + let payload = &hex!("FF0102030405060708"); + let mock = Builder::new() + .read(&produce_packet_bytes(payload).await[..offset]) + .read_error(std::io::Error::new(std::io::ErrorKind::Other, "foo")) + .build(); + let mut mock = BufReader::new(mock); + + // Either length reading or data reading can fail, depending on which test case we're in. + let err: io::Error = async { + let mut r = BytesReader::new(&mut mock, ..MAX_LEN).await?; + let mut buf = Vec::new(); + + tokio::io::copy_buf(&mut r, &mut buf).await?; + + Ok(()) + } + .await + .expect_err("must fail"); + + assert_eq!( + err.kind(), + std::io::ErrorKind::Other, + "error kind must match" + ); + + assert_eq!( + err.into_inner().unwrap().to_string(), + "foo", + "error payload must contain foo" + ); + } + + /// If there's an error right after the padding, we don't propagate it, as + /// we're done reading. We just return EOF. + #[tokio::test] + async fn no_error_after_eof() { + let payload = &hex!("FF0102030405060708"); + let mut mock = Builder::new() + .read(&produce_packet_bytes(payload).await) + .read_error(std::io::Error::new(std::io::ErrorKind::Other, "foo")) + .build(); + + let mut r = BytesReader::new(&mut mock, ..MAX_LEN).await.unwrap(); + let mut buf = Vec::new(); + + r.read_to_end(&mut buf).await.expect("must succeed"); + assert_eq!(buf.as_slice(), payload); + } + + /// If there's an error right after the padding, we don't propagate it, as + /// we're done reading. We just return EOF. + #[tokio::test] + async fn no_error_after_eof_buffered() { + let payload = &hex!("FF0102030405060708"); + let mock = Builder::new() + .read(&produce_packet_bytes(payload).await) + .read_error(std::io::Error::new(std::io::ErrorKind::Other, "foo")) + .build(); + let mut mock = BufReader::new(mock); + + let mut r = BytesReader::new(&mut mock, ..MAX_LEN).await.unwrap(); + let mut buf = Vec::new(); + + tokio::io::copy_buf(&mut r, &mut buf) + .await + .expect("must succeed"); + assert_eq!(buf.as_slice(), payload); + } + + /// Introduce various stalls in various places of the packet, to ensure we + /// handle these cases properly, too. + #[rstest] + #[case::beginning(0)] + #[case::before_payload(8)] + #[case::during_payload(8 + 4)] + #[case::before_padding(8 + 4)] + #[case::during_padding(8 + 9 + 2)] + #[tokio::test] + async fn read_payload_correct_pending(#[case] offset: usize) { + let payload = &hex!("FF0102030405060708"); + let mut mock = Builder::new() + .read(&produce_packet_bytes(payload).await[..offset]) + .wait(Duration::from_nanos(0)) + .read(&produce_packet_bytes(payload).await[offset..]) + .build(); + + let mut r = BytesReader::new(&mut mock, ..=LARGE_PAYLOAD.len() as u64) + .await + .unwrap(); + let mut buf = Vec::new(); + r.read_to_end(&mut buf).await.expect("must succeed"); + + assert_eq!(payload, &buf[..]); + } +} diff --git a/tvix/nix-compat/src/wire/bytes/reader/trailer.rs b/tvix/nix-compat/src/wire/bytes/reader/trailer.rs new file mode 100644 index 0000000000..3a5bb75e71 --- /dev/null +++ b/tvix/nix-compat/src/wire/bytes/reader/trailer.rs @@ -0,0 +1,197 @@ +use std::{ + fmt::Debug, + future::Future, + marker::PhantomData, + ops::Deref, + pin::Pin, + task::{self, ready, Poll}, +}; + +use tokio::io::{self, AsyncRead, ReadBuf}; + +/// Trailer represents up to 8 bytes of data read as part of the trailer block(s) +#[derive(Debug)] +pub(crate) struct Trailer { + data_len: u8, + buf: [u8; 8], +} + +impl Deref for Trailer { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + &self.buf[..self.data_len as usize] + } +} + +/// Tag defines a "trailer tag": specific, fixed bytes that must follow wire data. +pub(crate) trait Tag { + /// The expected suffix + /// + /// The first 8 bytes may be ignored, and it must be an 8-byte aligned size. + const PATTERN: &'static [u8]; + + /// Suitably sized buffer for reading [Self::PATTERN] + /// + /// HACK: This is a workaround for const generics limitations. + type Buf: AsRef<[u8]> + AsMut<[u8]> + Debug + Unpin; + + /// Make an instance of [Self::Buf] + fn make_buf() -> Self::Buf; +} + +#[derive(Debug)] +pub enum Pad {} + +impl Tag for Pad { + const PATTERN: &'static [u8] = &[0; 8]; + + type Buf = [u8; 8]; + + fn make_buf() -> Self::Buf { + [0; 8] + } +} + +#[derive(Debug)] +pub(crate) struct ReadTrailer<R, T: Tag> { + reader: R, + data_len: u8, + filled: u8, + buf: T::Buf, + _phantom: PhantomData<fn(T) -> T>, +} + +/// read_trailer returns a [Future] that reads a trailer with a given [Tag] from `reader` +pub(crate) fn read_trailer<R: AsyncRead + Unpin, T: Tag>( + reader: R, + data_len: u8, +) -> ReadTrailer<R, T> { + assert!(data_len <= 8, "payload in trailer must be <= 8 bytes"); + + let buf = T::make_buf(); + assert_eq!(buf.as_ref().len(), T::PATTERN.len()); + assert_eq!(T::PATTERN.len() % 8, 0); + + ReadTrailer { + reader, + data_len, + filled: if data_len != 0 { 0 } else { 8 }, + buf, + _phantom: PhantomData, + } +} + +impl<R, T: Tag> ReadTrailer<R, T> { + pub fn len(&self) -> u8 { + self.data_len + } +} + +impl<R: AsyncRead + Unpin, T: Tag> Future for ReadTrailer<R, T> { + type Output = io::Result<Trailer>; + + fn poll(mut self: Pin<&mut Self>, cx: &mut task::Context) -> Poll<Self::Output> { + let this = &mut *self; + + loop { + if this.filled >= this.data_len { + let check_range = || this.data_len as usize..this.filled as usize; + + if this.buf.as_ref()[check_range()] != T::PATTERN[check_range()] { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid trailer", + )) + .into(); + } + } + + if this.filled as usize == T::PATTERN.len() { + let mut buf = [0; 8]; + buf.copy_from_slice(&this.buf.as_ref()[..8]); + + return Ok(Trailer { + data_len: this.data_len, + buf, + }) + .into(); + } + + let mut buf = ReadBuf::new(this.buf.as_mut()); + buf.advance(this.filled as usize); + + ready!(Pin::new(&mut this.reader).poll_read(cx, &mut buf))?; + + this.filled = { + let filled = buf.filled().len() as u8; + + if filled == this.filled { + return Err(io::ErrorKind::UnexpectedEof.into()).into(); + } + + filled + }; + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[tokio::test] + async fn unexpected_eof() { + let reader = tokio_test::io::Builder::new() + .read(&[0xed]) + .wait(Duration::ZERO) + .read(&[0xef, 0x00]) + .build(); + + assert_eq!( + read_trailer::<_, Pad>(reader, 2).await.unwrap_err().kind(), + io::ErrorKind::UnexpectedEof + ); + } + + #[tokio::test] + async fn invalid_padding() { + let reader = tokio_test::io::Builder::new() + .read(&[0xed]) + .wait(Duration::ZERO) + .read(&[0xef, 0x01, 0x00]) + .wait(Duration::ZERO) + .build(); + + assert_eq!( + read_trailer::<_, Pad>(reader, 2).await.unwrap_err().kind(), + io::ErrorKind::InvalidData + ); + } + + #[tokio::test] + async fn success() { + let reader = tokio_test::io::Builder::new() + .read(&[0xed]) + .wait(Duration::ZERO) + .read(&[0xef, 0x00]) + .wait(Duration::ZERO) + .read(&[0x00, 0x00, 0x00, 0x00, 0x00]) + .build(); + + assert_eq!( + &*read_trailer::<_, Pad>(reader, 2).await.unwrap(), + &[0xed, 0xef] + ); + } + + #[tokio::test] + async fn no_padding() { + assert!(read_trailer::<_, Pad>(io::empty(), 0) + .await + .unwrap() + .is_empty()); + } +} diff --git a/tvix/nix-compat/src/wire/bytes/writer.rs b/tvix/nix-compat/src/wire/bytes/writer.rs new file mode 100644 index 0000000000..f5632771e9 --- /dev/null +++ b/tvix/nix-compat/src/wire/bytes/writer.rs @@ -0,0 +1,538 @@ +use pin_project_lite::pin_project; +use std::task::{ready, Poll}; + +use tokio::io::AsyncWrite; + +use super::{padding_len, EMPTY_BYTES, LEN_SIZE}; + +pin_project! { + /// Writes a "bytes wire packet" to the underlying writer. + /// The format is the same as in [crate::wire::bytes::write_bytes], + /// however this structure provides a [AsyncWrite] interface, + /// allowing to not having to pass around the entire payload in memory. + /// + /// It internally takes care of writing (non-payload) framing (size and + /// padding). + /// + /// During construction, the expected payload size needs to be provided. + /// + /// After writing the payload to it, the user MUST call flush (or shutdown), + /// which will validate the written payload size to match, and write the + /// necessary padding. + /// + /// In case flush is not called at the end, invalid data might be sent + /// silently. + /// + /// The underlying writer returning `Ok(0)` is considered an EOF situation, + /// which is stronger than the "typically means the underlying object is no + /// longer able to accept bytes" interpretation from the docs. If such a + /// situation occurs, an error is returned. + /// + /// The struct holds three fields, the underlying writer, the (expected) + /// payload length, and an enum, tracking the state. + pub struct BytesWriter<W> + where + W: AsyncWrite, + { + #[pin] + inner: W, + payload_len: u64, + state: BytesPacketPosition, + } +} + +/// Models the position inside a "bytes wire packet" that the writer is in. +/// It can be in three different stages, inside size, payload or padding fields. +/// The number tracks the number of bytes written inside the specific field. +/// There shall be no ambiguous states, at the end of a stage we immediately +/// move to the beginning of the next one: +/// - Size(LEN_SIZE) must be expressed as Payload(0) +/// - Payload(self.payload_len) must be expressed as Padding(0) +/// +/// Padding(padding_len) means we're at the end of the bytes wire packet. +#[derive(Clone, Debug, PartialEq, Eq)] +enum BytesPacketPosition { + Size(usize), + Payload(u64), + Padding(usize), +} + +impl<W> BytesWriter<W> +where + W: AsyncWrite, +{ + /// Constructs a new BytesWriter, using the underlying passed writer. + pub fn new(w: W, payload_len: u64) -> Self { + Self { + inner: w, + payload_len, + state: BytesPacketPosition::Size(0), + } + } +} + +/// Returns an error if the passed usize is 0. +#[inline] +fn ensure_nonzero_bytes_written(bytes_written: usize) -> Result<usize, std::io::Error> { + if bytes_written == 0 { + Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "underlying writer accepted 0 bytes", + )) + } else { + Ok(bytes_written) + } +} + +impl<W> AsyncWrite for BytesWriter<W> +where + W: AsyncWrite, +{ + fn poll_write( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> Poll<Result<usize, std::io::Error>> { + // Use a loop, so we can deal with (multiple) state transitions. + let mut this = self.project(); + + loop { + match *this.state { + BytesPacketPosition::Size(LEN_SIZE) => unreachable!(), + BytesPacketPosition::Size(pos) => { + let size_field = &this.payload_len.to_le_bytes(); + + let bytes_written = ensure_nonzero_bytes_written(ready!(this + .inner + .as_mut() + .poll_write(cx, &size_field[pos..]))?)?; + + let new_pos = pos + bytes_written; + if new_pos == LEN_SIZE { + *this.state = BytesPacketPosition::Payload(0); + } else { + *this.state = BytesPacketPosition::Size(new_pos); + } + } + BytesPacketPosition::Payload(pos) => { + // Ensure we still have space for more payload + if pos + (buf.len() as u64) > *this.payload_len { + return Poll::Ready(Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "tried to write excess bytes", + ))); + } + let bytes_written = ready!(this.inner.as_mut().poll_write(cx, buf))?; + ensure_nonzero_bytes_written(bytes_written)?; + let new_pos = pos + (bytes_written as u64); + if new_pos == *this.payload_len { + *this.state = BytesPacketPosition::Padding(0) + } else { + *this.state = BytesPacketPosition::Payload(new_pos) + } + + return Poll::Ready(Ok(bytes_written)); + } + // If we're already in padding state, there should be no more payload left to write! + BytesPacketPosition::Padding(_pos) => { + return Poll::Ready(Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "tried to write excess bytes", + ))) + } + } + } + } + + fn poll_flush( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll<Result<(), std::io::Error>> { + let mut this = self.project(); + + loop { + match *this.state { + BytesPacketPosition::Size(LEN_SIZE) => unreachable!(), + BytesPacketPosition::Size(pos) => { + // More bytes to write in the size field + let size_field = &this.payload_len.to_le_bytes()[..]; + let bytes_written = ensure_nonzero_bytes_written(ready!(this + .inner + .as_mut() + .poll_write(cx, &size_field[pos..]))?)?; + let new_pos = pos + bytes_written; + if new_pos == LEN_SIZE { + // Size field written, now ready to receive payload + *this.state = BytesPacketPosition::Payload(0); + } else { + *this.state = BytesPacketPosition::Size(new_pos); + } + } + BytesPacketPosition::Payload(_pos) => { + // If we're at position 0 and want to write 0 bytes of payload + // in total, we can transition to padding. + // Otherwise, break, as we're expecting more payload to + // be written. + if *this.payload_len == 0 { + *this.state = BytesPacketPosition::Padding(0); + } else { + break; + } + } + BytesPacketPosition::Padding(pos) => { + // Write remaining padding, if there is padding to write. + let total_padding_len = padding_len(*this.payload_len) as usize; + + if pos != total_padding_len { + let bytes_written = ensure_nonzero_bytes_written(ready!(this + .inner + .as_mut() + .poll_write(cx, &EMPTY_BYTES[pos..total_padding_len]))?)?; + *this.state = BytesPacketPosition::Padding(pos + bytes_written); + } else { + // everything written, break + break; + } + } + } + } + // Flush the underlying writer. + this.inner.as_mut().poll_flush(cx) + } + + fn poll_shutdown( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll<Result<(), std::io::Error>> { + // Call flush. + ready!(self.as_mut().poll_flush(cx))?; + + let this = self.project(); + + // After a flush, being inside the padding state, and at the end of the padding + // is the only way to prevent a dirty shutdown. + if let BytesPacketPosition::Padding(pos) = *this.state { + let padding_len = padding_len(*this.payload_len) as usize; + if padding_len == pos { + // Shutdown the underlying writer + return this.inner.poll_shutdown(cx); + } + } + + // Shutdown the underlying writer, bubbling up any errors. + ready!(this.inner.poll_shutdown(cx))?; + + // return an error about unclean shutdown + Poll::Ready(Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "unclean shutdown", + ))) + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use crate::wire::bytes::write_bytes; + use hex_literal::hex; + use lazy_static::lazy_static; + use tokio::io::AsyncWriteExt; + use tokio_test::{assert_err, assert_ok, io::Builder}; + + use super::*; + + lazy_static! { + pub static ref LARGE_PAYLOAD: Vec<u8> = (0..255).collect::<Vec<u8>>().repeat(4 * 1024); + } + + /// Helper function, calling the (simpler) write_bytes with the payload. + /// We use this to create data we want to see on the wire. + async fn produce_exp_bytes(payload: &[u8]) -> Vec<u8> { + let mut exp = vec![]; + write_bytes(&mut exp, payload).await.unwrap(); + exp + } + + /// Write an empty bytes packet. + #[tokio::test] + async fn write_empty() { + let payload = &[]; + let mut mock = Builder::new() + .write(&produce_exp_bytes(payload).await) + .build(); + + let mut w = BytesWriter::new(&mut mock, 0); + assert_ok!(w.write_all(&[]).await, "write all data"); + assert_ok!(w.flush().await, "flush"); + } + + /// Write an empty bytes packet, not calling write. + #[tokio::test] + async fn write_empty_only_flush() { + let payload = &[]; + let mut mock = Builder::new() + .write(&produce_exp_bytes(payload).await) + .build(); + + let mut w = BytesWriter::new(&mut mock, 0); + assert_ok!(w.flush().await, "flush"); + } + + /// Write an empty bytes packet, not calling write or flush, only shutdown. + #[tokio::test] + async fn write_empty_only_shutdown() { + let payload = &[]; + let mut mock = Builder::new() + .write(&produce_exp_bytes(payload).await) + .build(); + + let mut w = BytesWriter::new(&mut mock, 0); + assert_ok!(w.shutdown().await, "shutdown"); + } + + /// Write a 1 bytes packet + #[tokio::test] + async fn write_1b() { + let payload = &[0xff]; + + let mut mock = Builder::new() + .write(&produce_exp_bytes(payload).await) + .build(); + + let mut w = BytesWriter::new(&mut mock, payload.len() as u64); + assert_ok!(w.write_all(payload).await); + assert_ok!(w.flush().await, "flush"); + } + + /// Write a 8 bytes payload (no padding) + #[tokio::test] + async fn write_8b() { + let payload = &hex!("0001020304050607"); + + let mut mock = Builder::new() + .write(&produce_exp_bytes(payload).await) + .build(); + + let mut w = BytesWriter::new(&mut mock, payload.len() as u64); + assert_ok!(w.write_all(payload).await); + assert_ok!(w.flush().await, "flush"); + } + + /// Write a 9 bytes payload (7 bytes padding) + #[tokio::test] + async fn write_9b() { + let payload = &hex!("000102030405060708"); + + let mut mock = Builder::new() + .write(&produce_exp_bytes(payload).await) + .build(); + + let mut w = BytesWriter::new(&mut mock, payload.len() as u64); + assert_ok!(w.write_all(payload).await); + assert_ok!(w.flush().await, "flush"); + } + + /// Write a 9 bytes packet very granularly, with a lot of flushing in between, + /// and a shutdown at the end. + #[tokio::test] + async fn write_9b_flush() { + let payload = &hex!("000102030405060708"); + let exp_bytes = produce_exp_bytes(payload).await; + + let mut mock = Builder::new().write(&exp_bytes).build(); + + let mut w = BytesWriter::new(&mut mock, payload.len() as u64); + assert_ok!(w.flush().await); + + assert_ok!(w.write_all(&payload[..4]).await); + assert_ok!(w.flush().await); + + // empty write, cause why not + assert_ok!(w.write_all(&[]).await); + assert_ok!(w.flush().await); + + assert_ok!(w.write_all(&payload[4..]).await); + assert_ok!(w.flush().await); + assert_ok!(w.shutdown().await); + } + + /// Write a 9 bytes packet, but cause the sink to only accept half of the + /// padding, ensuring we correctly write (only) the rest of the padding later. + /// We write another 2 bytes of "bait", where a faulty implementation (pre + /// cl/11384) would put too many null bytes. + #[tokio::test] + async fn write_9b_write_padding_2steps() { + let payload = &hex!("000102030405060708"); + let exp_bytes = produce_exp_bytes(payload).await; + + let mut mock = Builder::new() + .write(&exp_bytes[0..8]) // size + .write(&exp_bytes[8..17]) // payload + .write(&exp_bytes[17..19]) // padding (2 of 7 bytes) + // insert a wait to prevent Mock from merging the two writes into one + .wait(Duration::from_nanos(1)) + .write(&hex!("0000000000ffff")) // padding (5 of 7 bytes, plus 2 bytes of "bait") + .build(); + + let mut w = BytesWriter::new(&mut mock, payload.len() as u64); + assert_ok!(w.write_all(&payload[..]).await); + assert_ok!(w.flush().await); + // Write bait + assert_ok!(mock.write_all(&hex!("ffff")).await); + } + + /// Write a larger bytes packet + #[tokio::test] + async fn write_1m() { + let payload = LARGE_PAYLOAD.as_slice(); + let exp_bytes = produce_exp_bytes(payload).await; + + let mut mock = Builder::new().write(&exp_bytes).build(); + let mut w = BytesWriter::new(&mut mock, payload.len() as u64); + + assert_ok!(w.write_all(payload).await); + assert_ok!(w.flush().await, "flush"); + } + + /// Not calling flush at the end, but shutdown is also ok if we wrote all + /// bytes we promised to write (as shutdown implies flush) + #[tokio::test] + async fn write_shutdown_without_flush_end() { + let payload = &[0xf0, 0xff]; + let exp_bytes = produce_exp_bytes(payload).await; + + let mut mock = Builder::new().write(&exp_bytes).build(); + let mut w = BytesWriter::new(&mut mock, payload.len() as u64); + + // call flush to write the size field + assert_ok!(w.flush().await); + + // write payload + assert_ok!(w.write_all(payload).await); + + // call shutdown + assert_ok!(w.shutdown().await); + } + + /// Writing more bytes than previously signalled should fail. + #[tokio::test] + async fn write_more_than_signalled_fail() { + let mut buf = Vec::new(); + let mut w = BytesWriter::new(&mut buf, 2); + + assert_err!(w.write_all(&hex!("000102")).await); + } + /// Writing more bytes than previously signalled, but in two parts + #[tokio::test] + async fn write_more_than_signalled_split_fail() { + let mut buf = Vec::new(); + let mut w = BytesWriter::new(&mut buf, 2); + + // write two bytes + assert_ok!(w.write_all(&hex!("0001")).await); + + // write the excess byte. + assert_err!(w.write_all(&hex!("02")).await); + } + + /// Writing more bytes than previously signalled, but flushing after the + /// signalled amount should fail. + #[tokio::test] + async fn write_more_than_signalled_flush_fail() { + let mut buf = Vec::new(); + let mut w = BytesWriter::new(&mut buf, 2); + + // write two bytes, then flush + assert_ok!(w.write_all(&hex!("0001")).await); + assert_ok!(w.flush().await); + + // write the excess byte. + assert_err!(w.write_all(&hex!("02")).await); + } + + /// Calling shutdown while not having written all bytes that were promised + /// returns an error. + /// Note there's still cases of silent corruption if the user doesn't call + /// shutdown explicitly (only drops). + #[tokio::test] + async fn premature_shutdown() { + let payload = &[0xf0, 0xff]; + let mut buf = Vec::new(); + let mut w = BytesWriter::new(&mut buf, payload.len() as u64); + + // call flush to write the size field + assert_ok!(w.flush().await); + + // write half of the payload (!) + assert_ok!(w.write_all(&payload[0..1]).await); + + // call shutdown, ensure it fails + assert_err!(w.shutdown().await); + } + + /// Write to a Writer that fails to write during the size packet (after 4 bytes). + /// Ensure this error gets propagated on the first call to write. + #[tokio::test] + async fn inner_writer_fail_during_size_firstwrite() { + let payload = &[0xf0]; + + let mut mock = Builder::new() + .write(&1u32.to_le_bytes()) + .write_error(std::io::Error::new(std::io::ErrorKind::Other, "๐ฟ")) + .build(); + let mut w = BytesWriter::new(&mut mock, payload.len() as u64); + + assert_err!(w.write_all(payload).await); + } + + /// Write to a Writer that fails to write during the size packet (after 4 bytes). + /// Ensure this error gets propagated during an initial flush + #[tokio::test] + async fn inner_writer_fail_during_size_initial_flush() { + let payload = &[0xf0]; + + let mut mock = Builder::new() + .write(&1u32.to_le_bytes()) + .write_error(std::io::Error::new(std::io::ErrorKind::Other, "๐ฟ")) + .build(); + let mut w = BytesWriter::new(&mut mock, payload.len() as u64); + + assert_err!(w.flush().await); + } + + /// Write to a writer that fails to write during the payload (after 9 bytes). + /// Ensure this error gets propagated when we're writing this byte. + #[tokio::test] + async fn inner_writer_fail_during_write() { + let payload = &hex!("f0ff"); + + let mut mock = Builder::new() + .write(&2u64.to_le_bytes()) + .write(&hex!("f0")) + .write_error(std::io::Error::new(std::io::ErrorKind::Other, "๐ฟ")) + .build(); + let mut w = BytesWriter::new(&mut mock, payload.len() as u64); + + assert_ok!(w.write(&hex!("f0")).await); + assert_err!(w.write(&hex!("ff")).await); + } + + /// Write to a writer that fails to write during the padding (after 10 bytes). + /// Ensure this error gets propagated during a flush. + #[tokio::test] + async fn inner_writer_fail_during_padding_flush() { + let payload = &hex!("f0"); + + let mut mock = Builder::new() + .write(&1u64.to_le_bytes()) + .write(&hex!("f0")) + .write(&hex!("00")) + .write_error(std::io::Error::new(std::io::ErrorKind::Other, "๐ฟ")) + .build(); + let mut w = BytesWriter::new(&mut mock, payload.len() as u64); + + assert_ok!(w.write(&hex!("f0")).await); + assert_err!(w.flush().await); + } +} diff --git a/tvix/nix-compat/src/wire/mod.rs b/tvix/nix-compat/src/wire/mod.rs index cd7b87aacd..a197e3a1f4 100644 --- a/tvix/nix-compat/src/wire/mod.rs +++ b/tvix/nix-compat/src/wire/mod.rs @@ -1,11 +1,5 @@ //! Module parsing and emitting the wire format used by Nix, both in the //! nix-daemon protocol as well as in the NAR format. -#[cfg(feature = "async")] -pub mod bytes; - -#[cfg(feature = "async")] -pub mod primitive; - -#[cfg(feature = "async")] -pub mod worker_protocol; +mod bytes; +pub use bytes::*; diff --git a/tvix/nix-compat/src/wire/primitive.rs b/tvix/nix-compat/src/wire/primitive.rs deleted file mode 100644 index a925bd00df..0000000000 --- a/tvix/nix-compat/src/wire/primitive.rs +++ /dev/null @@ -1,113 +0,0 @@ -// SPDX-FileCopyrightText: 2023 embr <git@liclac.eu> -// -// SPDX-License-Identifier: EUPL-1.2 - -use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; - -// LE-encoded nixc on 64 bits. Because why not. -pub static MAGIC_HELLO: [u8; 8] = *b"cxin\0\0\0\0"; -// LE-encoded dxio on 64 bits. What's dxio? I have no clue. -pub static MAGIC_HELLO_RESPONSE: [u8; 8] = *b"oixd\0\0\0\0"; -// LE-encoded protocol version. -pub static PROTOCOL_VERSION: [u8; 8] = [0x23, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]; - -/// Read a LE u32 from the least-significant bytes of a LE u64. -/// -/// Overall, it looks like this on the wire: -/// -/// 00 0x12 0x32 0x00 0x00 0x00 0x00 0x00 0x00 -/// |------------------|-------------------| -/// LE u32 padding -/// -/// Not sure why the protocol does this instead of using a plain u64, -/// but well, it is what it is. -/// -/// Analogous to the readInt function in cppnix. -pub async fn read_u32<R: AsyncReadExt + Unpin>(r: &mut R) -> std::io::Result<u32> { - let val64 = r.read_u64_le().await?; - u32::try_from(val64).map_err(|_| { - std::io::Error::new(std::io::ErrorKind::InvalidData, "padding is not all zeroes") - }) -} - -#[allow(dead_code)] -/// Read a u64 from the AsyncRead (little endian). -pub async fn read_u64<R: AsyncReadExt + Unpin>(r: &mut R) -> std::io::Result<u64> { - r.read_u64_le().await -} - -/// Write a u64 to the AsyncWrite (little endian). -pub async fn write_u64<W: AsyncWrite + Unpin>(w: &mut W, v: u64) -> std::io::Result<()> { - w.write_u64_le(v).await -} - -#[allow(dead_code)] -/// Read a boolean from the AsyncRead, encoded as u64 (>0 is true). -pub async fn read_bool<R: AsyncRead + Unpin>(r: &mut R) -> std::io::Result<bool> { - Ok(read_u64(r).await? > 0) -} - -#[allow(dead_code)] -/// Write a boolean to the AsyncWrite, encoded as u64 (>0 is true). -pub async fn write_bool<W: AsyncWrite + Unpin>(w: &mut W, v: bool) -> std::io::Result<()> { - write_u64(w, if v { 1u64 } else { 0u64 }).await -} - -#[cfg(test)] -mod tests { - use super::*; - use hex_literal::hex; - use tokio_test::{assert_err, io::Builder}; - - // Integers. - #[tokio::test] - async fn test_read_u64() { - let mut mock = Builder::new().read(&1234567890u64.to_le_bytes()).build(); - assert_eq!(1234567890u64, read_u64(&mut mock).await.unwrap()); - } - #[tokio::test] - async fn test_write_u64() { - let mut mock = Builder::new().write(&1234567890u64.to_le_bytes()).build(); - write_u64(&mut mock, 1234567890).await.unwrap(); - } - - // Booleans. - #[tokio::test] - async fn test_read_bool_0() { - let mut mock = Builder::new().read(&0u64.to_le_bytes()).build(); - assert!(!read_bool(&mut mock).await.unwrap()); - } - #[tokio::test] - async fn test_read_bool_1() { - let mut mock = Builder::new().read(&1u64.to_le_bytes()).build(); - assert!(read_bool(&mut mock).await.unwrap()); - } - #[tokio::test] - async fn test_read_bool_2() { - let mut mock = Builder::new().read(&2u64.to_le_bytes()).build(); - assert!(read_bool(&mut mock).await.unwrap()); - } - - #[tokio::test] - async fn test_write_bool_false() { - let mut mock = Builder::new().write(&0u64.to_le_bytes()).build(); - write_bool(&mut mock, false).await.unwrap(); - } - #[tokio::test] - async fn test_write_bool_true() { - let mut mock = Builder::new().write(&1u64.to_le_bytes()).build(); - write_bool(&mut mock, true).await.unwrap(); - } - #[tokio::test] - async fn test_read_u32() { - let mut mock = Builder::new().read(&hex!("7856341200000000")).build(); - let res = read_u32(&mut mock).await.unwrap(); - assert_eq!(res, 0x12345678); - } - #[tokio::test] - async fn test_read_too_large_u32_fail() { - let mut mock = Builder::new().read(&hex!("7856341298760000")).build(); - let res = read_u32(&mut mock).await; - assert_err!(res); - } -} diff --git a/tvix/nix-compat/src/wire/worker_protocol.rs b/tvix/nix-compat/src/wire/worker_protocol.rs deleted file mode 100644 index 82f227c723..0000000000 --- a/tvix/nix-compat/src/wire/worker_protocol.rs +++ /dev/null @@ -1,83 +0,0 @@ -use std::io::{Error, ErrorKind}; - -use enum_primitive_derive::Primitive; -use num_traits::{FromPrimitive, ToPrimitive}; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; - -use crate::wire::primitive; - -/// Worker Operation -/// -/// These operations are encoded as unsigned 64 bits before being sent -/// to the wire. See the [read_op] and -/// [write_op] operations to serialize/deserialize the -/// operation on the wire. -/// -/// Note: for now, we're using the Nix 2.20 operation description. The -/// operations marked as obsolete are obsolete for Nix 2.20, not -/// necessarily for Nix 2.3. We'll revisit this later on. -#[derive(Debug, PartialEq, Primitive)] -pub enum Operation { - IsValidPath = 1, - HasSubstitutes = 3, - QueryPathHash = 4, // obsolete - QueryReferences = 5, // obsolete - QueryReferrers = 6, - AddToStore = 7, - AddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use WorkerProto::Op::AddToStore - BuildPaths = 9, - EnsurePath = 10, - AddTempRoot = 11, - AddIndirectRoot = 12, - SyncWithGC = 13, - FindRoots = 14, - ExportPath = 16, // obsolete - QueryDeriver = 18, // obsolete - SetOptions = 19, - CollectGarbage = 20, - QuerySubstitutablePathInfo = 21, - QueryDerivationOutputs = 22, // obsolete - QueryAllValidPaths = 23, - QueryFailedPaths = 24, - ClearFailedPaths = 25, - QueryPathInfo = 26, - ImportPaths = 27, // obsolete - QueryDerivationOutputNames = 28, // obsolete - QueryPathFromHashPart = 29, - QuerySubstitutablePathInfos = 30, - QueryValidPaths = 31, - QuerySubstitutablePaths = 32, - QueryValidDerivers = 33, - OptimiseStore = 34, - VerifyStore = 35, - BuildDerivation = 36, - AddSignatures = 37, - NarFromPath = 38, - AddToStoreNar = 39, - QueryMissing = 40, - QueryDerivationOutputMap = 41, - RegisterDrvOutput = 42, - QueryRealisation = 43, - AddMultipleToStore = 44, - AddBuildLog = 45, - BuildPathsWithResults = 46, - AddPermRoot = 47, -} - -/// Read a worker [Operation] from the wire. -pub async fn read_op<R: AsyncReadExt + Unpin>(r: &mut R) -> std::io::Result<Operation> { - let op_number = primitive::read_u64(r).await?; - Operation::from_u64(op_number).ok_or(Error::new( - ErrorKind::Other, - format!("Invalid OP number {}", op_number), - )) -} - -/// Write a worker [Operation] to the wire. -pub async fn write_op<W: AsyncWriteExt + Unpin>(w: &mut W, op: &Operation) -> std::io::Result<()> { - let op = Operation::to_u64(op).ok_or(Error::new( - ErrorKind::Other, - format!("Can't convert the OP {:?} to u64", op), - ))?; - w.write_u64(op).await -} diff --git a/tvix/serde/src/de.rs b/tvix/serde/src/de.rs index e9d4f6cf49..428b9e2e81 100644 --- a/tvix/serde/src/de.rs +++ b/tvix/serde/src/de.rs @@ -107,7 +107,7 @@ impl<'de> de::Deserializer<'de> for NixDeserializer { | Value::Blueprint(_) | Value::DeferredUpvalue(_) | Value::UnresolvedPath(_) - | Value::Json(_) + | Value::Json(..) | Value::Catchable(_) | Value::FinaliseRequest(_) => Err(Error::Unserializable { value_type: self.value.type_of(), diff --git a/tvix/shell.nix b/tvix/shell.nix index c09f4b7b31..f0d8ab1657 100644 --- a/tvix/shell.nix +++ b/tvix/shell.nix @@ -8,6 +8,16 @@ pkgs ? (import ./nixpkgs { depotOverlays = false; depot.third_party.sources = import ./sources { }; + additionalOverlays = [ + (self: super: { + # https://github.com/googleapis/google-cloud-go/pull/9665 + cbtemulator = super.cbtemulator.overrideAttrs (old: { + patches = old.patches or [ ] ++ [ + ./nixpkgs/cbtemulator-uds.patch + ]; + }); + }) + ]; }) , ... }: diff --git a/tvix/store/Cargo.toml b/tvix/store/Cargo.toml index 367e63c21e..f82cdef300 100644 --- a/tvix/store/Cargo.toml +++ b/tvix/store/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] anyhow = "1.0.68" +async-compression = { version = "0.4.9", features = ["tokio", "bzip2", "gzip", "xz", "zstd"]} async-stream = "0.3.5" blake3 = { version = "1.3.1", features = ["rayon", "std"] } bstr = "1.6.0" @@ -20,6 +21,10 @@ prost = "0.12.1" opentelemetry = { version = "0.21.0", optional = true} opentelemetry-otlp = { version = "0.14.0", optional = true } opentelemetry_sdk = { version = "0.21.0", features = ["rt-tokio"], optional = true} +serde = { version = "1.0.197", features = [ "derive" ] } +serde_json = "1.0" +serde_with = "3.7.0" +serde_qs = "0.12.0" sha2 = "0.10.6" sled = { version = "0.34.7" } thiserror = "1.0.38" @@ -35,29 +40,40 @@ tracing-subscriber = { version = "0.3.16", features = ["env-filter", "json"] } tvix-castore = { path = "../castore" } url = "2.4.0" walkdir = "2.4.0" -async-recursion = "1.0.5" reqwest = { version = "0.11.22", features = ["rustls-tls-native-roots", "stream"], default-features = false } -xz2 = "0.1.7" [dependencies.tonic-reflection] optional = true version = "0.11.0" +[dependencies.bigtable_rs] +optional = true +# https://github.com/liufuyang/bigtable_rs/pull/72 +git = "https://github.com/flokli/bigtable_rs" +rev = "0af404741dfc40eb9fa99cf4d4140a09c5c20df7" + [build-dependencies] prost-build = "0.12.1" tonic-build = "0.11.0" [dev-dependencies] -rstest = "0.18.2" +async-process = "2.1.0" +rstest = "0.19.0" rstest_reuse = "0.6.0" -test-case = "3.3.1" tempfile = "3.3.0" tokio-retry = "0.3.0" [features] default = ["cloud", "fuse", "otlp", "tonic-reflection"] -cloud = ["tvix-castore/cloud"] +cloud = [ + "dep:bigtable_rs", + "tvix-castore/cloud" +] fuse = ["tvix-castore/fuse"] otlp = ["dep:opentelemetry", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk"] tonic-reflection = ["dep:tonic-reflection", "tvix-castore/tonic-reflection"] virtiofs = ["tvix-castore/virtiofs"] +# Whether to run the integration tests. +# Requires the following packages in $PATH: +# cbtemulator, google-cloud-bigtable-tool +integration = [] diff --git a/tvix/store/default.nix b/tvix/store/default.nix index 2c07cdf2b3..ad47994f24 100644 --- a/tvix/store/default.nix +++ b/tvix/store/default.nix @@ -25,14 +25,28 @@ in (depot.tvix.crates.workspaceMembers.tvix-store.build.override { runTests = true; testPreRun = '' - export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt; + export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt ''; - # virtiofs feature currently fails to build on Darwin. - # we however can ship it for non-darwin. - features = if pkgs.stdenv.isDarwin then [ "default" ] else [ "default" "virtiofs" ]; + # enable some optional features. + features = [ "default" "cloud" ] + # virtiofs feature currently fails to build on Darwin. + ++ pkgs.lib.optional pkgs.stdenv.isLinux "virtiofs"; }).overrideAttrs (_: { + meta.ci.targets = [ "integration-tests" ]; meta.ci.extraSteps = { import-docs = (mkImportCheck "tvix/store/docs" ./docs); }; + passthru.integration-tests = depot.tvix.crates.workspaceMembers.tvix-store.build.override { + runTests = true; + testPreRun = '' + export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt + export PATH="$PATH:${pkgs.lib.makeBinPath [pkgs.cbtemulator pkgs.google-cloud-bigtable-tool]}" + ''; + + # enable some optional features. + features = [ "default" "cloud" "integration" ] + # virtiofs feature currently fails to build on Darwin. + ++ pkgs.lib.optional pkgs.stdenv.isLinux "virtiofs"; + }; }) diff --git a/tvix/store/src/bin/tvix-store.rs b/tvix/store/src/bin/tvix-store.rs index 27a67b7c91..fa30501e78 100644 --- a/tvix/store/src/bin/tvix-store.rs +++ b/tvix/store/src/bin/tvix-store.rs @@ -2,6 +2,11 @@ use clap::Parser; use clap::Subcommand; use futures::future::try_join_all; +use futures::StreamExt; +use futures::TryStreamExt; +use nix_compat::path_info::ExportedPathInfo; +use serde::Deserialize; +use serde::Serialize; use std::path::PathBuf; use std::sync::Arc; use tokio_listener::Listener; @@ -13,6 +18,9 @@ use tracing::Level; use tracing_subscriber::EnvFilter; use tracing_subscriber::Layer; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; +use tvix_castore::import::fs::ingest_path; +use tvix_store::proto::NarInfo; +use tvix_store::proto::PathInfo; use tvix_castore::proto::blob_service_server::BlobServiceServer; use tvix_castore::proto::directory_service_server::DirectoryServiceServer; @@ -48,10 +56,6 @@ use tvix_store::proto::FILE_DESCRIPTOR_SET; #[derive(Parser)] #[command(author, version, about, long_about = None)] struct Cli { - /// Whether to log in JSON - #[arg(long)] - json: bool, - /// Whether to configure OTLP. Set --otlp=false to disable. #[arg(long, default_missing_value = "true", default_value = "true", num_args(0..=1), require_equals(true), action(clap::ArgAction::Set))] otlp: bool, @@ -74,7 +78,11 @@ enum Commands { #[arg(long, short = 'l')] listen_address: Option<String>, - #[arg(long, env, default_value = "sled:///var/lib/tvix-store/blobs.sled")] + #[arg( + long, + env, + default_value = "objectstore+file:///var/lib/tvix-store/blobs.object_store" + )] blob_service_addr: String, #[arg( @@ -101,6 +109,30 @@ enum Commands { #[arg(long, env, default_value = "grpc+http://[::1]:8000")] path_info_service_addr: String, }, + + /// Copies a list of store paths on the system into tvix-store. + Copy { + #[arg(long, env, default_value = "grpc+http://[::1]:8000")] + blob_service_addr: String, + + #[arg(long, env, default_value = "grpc+http://[::1]:8000")] + directory_service_addr: String, + + #[arg(long, env, default_value = "grpc+http://[::1]:8000")] + path_info_service_addr: String, + + /// A path pointing to a JSON file produced by the Nix + /// `__structuredAttrs` containing reference graph information provided + /// by the `exportReferencesGraph` feature. + /// + /// This can be used to invoke tvix-store inside a Nix derivation + /// copying to a Tvix store (or outside, if the JSON file is copied + /// out). + /// + /// Currently limited to the `closure` key inside that JSON file. + #[arg(value_name = "NIX_ATTRS_JSON_FILE", env = "NIX_ATTRS_JSON_FILE")] + reference_graph_path: PathBuf, + }, /// Mounts a tvix-store at the given mountpoint #[cfg(feature = "fuse")] Mount { @@ -131,6 +163,10 @@ enum Commands { /// (exhaustive) listing. #[clap(long, short, action)] list_root: bool, + + #[arg(long, default_value_t = true)] + /// Whether to expose blob and directory digests as extended attributes. + show_xattr: bool, }, /// Starts a tvix-store virtiofs daemon at the given socket path. #[cfg(feature = "virtiofs")] @@ -153,6 +189,10 @@ enum Commands { /// (exhaustive) listing. #[clap(long, short, action)] list_root: bool, + + #[arg(long, default_value_t = true)] + /// Whether to expose blob and directory digests as extended attributes. + show_xattr: bool, }, } @@ -178,33 +218,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> { let level = cli.log_level.unwrap_or(Level::INFO); // Set up the tracing subscriber. - let subscriber = tracing_subscriber::registry() - .with( - cli.json.then_some( - tracing_subscriber::fmt::Layer::new() - .with_writer(std::io::stderr) - .json() - .with_filter( - EnvFilter::builder() - .with_default_directive(level.into()) - .from_env() - .expect("invalid RUST_LOG"), - ), - ), - ) - .with( - (!cli.json).then_some( - tracing_subscriber::fmt::Layer::new() - .with_writer(std::io::stderr) - .pretty() - .with_filter( - EnvFilter::builder() - .with_default_directive(level.into()) - .from_env() - .expect("invalid RUST_LOG"), - ), + let subscriber = tracing_subscriber::registry().with( + tracing_subscriber::fmt::Layer::new() + .with_writer(std::io::stderr) + .compact() + .with_filter( + EnvFilter::builder() + .with_default_directive(level.into()) + .from_env() + .expect("invalid RUST_LOG"), ), - ); + ); // Add the otlp layer (when otlp is enabled, and it's not disabled in the CLI) // then init the registry. @@ -357,6 +381,108 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> { try_join_all(tasks).await?; } + Commands::Copy { + blob_service_addr, + directory_service_addr, + path_info_service_addr, + reference_graph_path, + } => { + let (blob_service, directory_service, path_info_service) = + tvix_store::utils::construct_services( + blob_service_addr, + directory_service_addr, + path_info_service_addr, + ) + .await?; + + // Parse the file at reference_graph_path. + let reference_graph_json = tokio::fs::read(&reference_graph_path).await?; + + #[derive(Deserialize, Serialize)] + struct ReferenceGraph<'a> { + #[serde(borrow)] + closure: Vec<ExportedPathInfo<'a>>, + } + + let reference_graph: ReferenceGraph<'_> = + serde_json::from_slice(reference_graph_json.as_slice())?; + + // Arc the PathInfoService, as we clone it . + let path_info_service: Arc<dyn PathInfoService> = path_info_service.into(); + + // From our reference graph, lookup all pathinfos that might exist. + let elems: Vec<_> = futures::stream::iter(reference_graph.closure) + .map(|elem| { + let path_info_service = path_info_service.clone(); + async move { + path_info_service + .get(*elem.path.digest()) + .await + .map(|resp| (elem, resp)) + } + }) + .buffer_unordered(50) + // Filter out all that are already uploaded. + // TODO: check if there's a better combinator for this + .try_filter_map(|(elem, path_info)| { + std::future::ready(if path_info.is_none() { + Ok(Some(elem)) + } else { + Ok(None) + }) + }) + .try_collect() + .await?; + + // Run ingest_path on all of them. + let uploads: Vec<_> = futures::stream::iter(elems) + .map(|elem| { + // Map to a future returning the root node, alongside with the closure info. + let blob_service = blob_service.clone(); + let directory_service = directory_service.clone(); + async move { + // Ingest the given path. + + ingest_path( + blob_service, + directory_service, + PathBuf::from(elem.path.to_absolute_path()), + ) + .await + .map(|root_node| (elem, root_node)) + } + }) + .buffer_unordered(10) + .try_collect() + .await?; + + // Insert them into the PathInfoService. + // FUTUREWORK: do this properly respecting the reference graph. + for (elem, root_node) in uploads { + // Create and upload a PathInfo pointing to the root_node, + // annotated with information we have from the reference graph. + let path_info = PathInfo { + node: Some(tvix_castore::proto::Node { + node: Some(root_node), + }), + references: Vec::from_iter( + elem.references.iter().map(|e| e.digest().to_vec().into()), + ), + narinfo: Some(NarInfo { + nar_size: elem.nar_size, + nar_sha256: elem.nar_sha256.to_vec().into(), + signatures: vec![], + reference_names: Vec::from_iter( + elem.references.iter().map(|e| e.to_string()), + ), + deriver: None, + ca: None, + }), + }; + + path_info_service.put(path_info).await?; + } + } #[cfg(feature = "fuse")] Commands::Mount { dest, @@ -366,6 +492,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> { list_root, threads, allow_other, + show_xattr, } => { let (blob_service, directory_service, path_info_service) = tvix_store::utils::construct_services( @@ -381,6 +508,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> { directory_service, Arc::from(path_info_service), list_root, + show_xattr, ); info!(mount_path=?dest, "mounting"); @@ -406,6 +534,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> { directory_service_addr, path_info_service_addr, list_root, + show_xattr, } => { let (blob_service, directory_service, path_info_service) = tvix_store::utils::construct_services( @@ -421,6 +550,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> { directory_service, Arc::from(path_info_service), list_root, + show_xattr, ); info!(socket_path=?socket, "starting virtiofs-daemon"); diff --git a/tvix/store/src/import.rs b/tvix/store/src/import.rs index f2e7bdfe5d..2331fd77ea 100644 --- a/tvix/store/src/import.rs +++ b/tvix/store/src/import.rs @@ -1,16 +1,31 @@ use std::path::Path; use tracing::{debug, instrument}; use tvix_castore::{ - blobservice::BlobService, directoryservice::DirectoryService, proto::node::Node, B3Digest, + blobservice::BlobService, directoryservice::DirectoryService, import::fs::ingest_path, + proto::node::Node, B3Digest, }; -use nix_compat::store_path::{self, StorePath}; +use nix_compat::{ + nixhash::{CAHash, NixHash}, + store_path::{self, StorePath}, +}; use crate::{ pathinfoservice::PathInfoService, proto::{nar_info, NarInfo, PathInfo}, }; +impl From<CAHash> for nar_info::Ca { + fn from(value: CAHash) -> Self { + let hash_type: nar_info::ca::Hash = (&value).into(); + let digest: bytes::Bytes = value.hash().to_string().into(); + nar_info::Ca { + r#type: hash_type.into(), + digest, + } + } +} + pub fn log_node(node: &Node, path: &Path) { match node { Node::Directory(directory_node) => { @@ -54,13 +69,20 @@ pub fn path_to_name(path: &Path) -> std::io::Result<&str> { }) } -/// Takes the NAR size, SHA-256 of the NAR representation and the root node. -/// Returns the path information object for a content addressed NAR-style (recursive) object. +/// Takes the NAR size, SHA-256 of the NAR representation, the root node and optionally +/// a CA hash information. +/// +/// Returns the path information object for a NAR-style object. /// /// This [`PathInfo`] can be further filled for signatures, deriver or verified for the expected /// hashes. #[inline] -pub fn derive_nar_ca_path_info(nar_size: u64, nar_sha256: [u8; 32], root_node: Node) -> PathInfo { +pub fn derive_nar_ca_path_info( + nar_size: u64, + nar_sha256: [u8; 32], + ca: Option<CAHash>, + root_node: Node, +) -> PathInfo { // assemble the [crate::proto::PathInfo] object. PathInfo { node: Some(tvix_castore::proto::Node { @@ -74,10 +96,7 @@ pub fn derive_nar_ca_path_info(nar_size: u64, nar_sha256: [u8; 32], root_node: N signatures: vec![], reference_names: vec![], deriver: None, - ca: Some(nar_info::Ca { - r#type: nar_info::ca::Hash::NarSha256.into(), - digest: nar_sha256.to_vec().into(), - }), + ca: ca.map(|ca_hash| ca_hash.into()), }), } } @@ -94,12 +113,13 @@ pub async fn import_path_as_nar_ca<BS, DS, PS, P>( ) -> Result<StorePath, std::io::Error> where P: AsRef<Path> + std::fmt::Debug, - BS: AsRef<dyn BlobService> + Clone, - DS: AsRef<dyn DirectoryService>, + BS: BlobService + Clone, + DS: DirectoryService, PS: AsRef<dyn PathInfoService>, { - let root_node = - tvix_castore::import::ingest_path(blob_service, directory_service, &path).await?; + let root_node = ingest_path(blob_service, directory_service, path.as_ref()) + .await + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; // Ask the PathInfoService for the NAR size and sha256 let (nar_size, nar_sha256) = path_info_service.as_ref().calculate_nar(&root_node).await?; @@ -118,7 +138,12 @@ where let root_node = root_node.rename(output_path.to_string().into_bytes().into()); log_node(&root_node, path.as_ref()); - let path_info = derive_nar_ca_path_info(nar_size, nar_sha256, root_node); + let path_info = derive_nar_ca_path_info( + nar_size, + nar_sha256, + Some(CAHash::Nar(NixHash::Sha256(nar_sha256))), + root_node, + ); // This new [`PathInfo`] that we get back from there might contain additional signatures or // information set by the service itself. In this function, we silently swallow it because @@ -133,21 +158,22 @@ mod tests { use std::{ffi::OsStr, path::PathBuf}; use crate::import::path_to_name; - use test_case::test_case; + use rstest::rstest; - #[test_case("a/b/c", "c"; "simple path")] - #[test_case("a/b/../c", "c"; "simple path containing ..")] - #[test_case("a/b/../c/d/../e", "e"; "path containing multiple ..")] + #[rstest] + #[case::simple_path("a/b/c", "c")] + #[case::simple_path_containing_dotdot("a/b/../c", "c")] + #[case::path_containing_multiple_dotdot("a/b/../c/d/../e", "e")] - fn test_path_to_name(path: &str, expected_name: &str) { + fn test_path_to_name(#[case] path: &str, #[case] expected_name: &str) { let path: PathBuf = path.into(); assert_eq!(path_to_name(&path).expect("must succeed"), expected_name); } - #[test_case(b"a/b/.."; "path ending in ..")] - #[test_case(b"\xf8\xa1\xa1\xa1\xa1"; "non unicode path")] - - fn test_invalid_path_to_name(invalid_path: &[u8]) { + #[rstest] + #[case::path_ending_in_dotdot(b"a/b/..")] + #[case::non_unicode_path(b"\xf8\xa1\xa1\xa1\xa1")] + fn test_invalid_path_to_name(#[case] invalid_path: &[u8]) { let path: PathBuf = unsafe { OsStr::from_encoded_bytes_unchecked(invalid_path) }.into(); path_to_name(&path).expect_err("must fail"); } diff --git a/tvix/store/src/nar/import.rs b/tvix/store/src/nar/import.rs index 6f4dcdea5d..3d7c50014a 100644 --- a/tvix/store/src/nar/import.rs +++ b/tvix/store/src/nar/import.rs @@ -1,225 +1,122 @@ -use bytes::Bytes; -use nix_compat::nar; -use std::io::{self, BufRead}; -use tokio_util::io::SyncIoBridge; -use tracing::warn; +use nix_compat::nar::reader::r#async as nar_reader; +use tokio::{io::AsyncBufRead, sync::mpsc, try_join}; use tvix_castore::{ blobservice::BlobService, - directoryservice::{DirectoryPutter, DirectoryService}, - proto::{self as castorepb}, - B3Digest, + directoryservice::DirectoryService, + import::{ingest_entries, IngestionEntry, IngestionError}, + proto::{node::Node, NamedNode}, + PathBuf, }; -/// Accepts a reader providing a NAR. -/// Will traverse it, uploading blobs to the given [BlobService], and -/// directories to the given [DirectoryService]. -/// On success, the root node is returned. -/// This function is not async (because the NAR reader is not) -/// and calls [tokio::task::block_in_place] when interacting with backing -/// services, so make sure to only call this with spawn_blocking. -pub fn read_nar<R, BS, DS>( - r: &mut R, +/// Ingests the contents from a [AsyncRead] providing NAR into the tvix store, +/// interacting with a [BlobService] and [DirectoryService]. +/// It returns the castore root node or an error. +pub async fn ingest_nar<R, BS, DS>( blob_service: BS, directory_service: DS, -) -> io::Result<castorepb::node::Node> + r: &mut R, +) -> Result<Node, IngestionError<Error>> where - R: BufRead + Send, - BS: AsRef<dyn BlobService>, - DS: AsRef<dyn DirectoryService>, + R: AsyncBufRead + Unpin + Send, + BS: BlobService + Clone, + DS: DirectoryService, { - let handle = tokio::runtime::Handle::current(); + // open the NAR for reading. + // The NAR reader emits nodes in DFS preorder. + let root_node = nar_reader::open(r).await.map_err(Error::IO)?; - let directory_putter = directory_service.as_ref().put_multiple_start(); + let (tx, rx) = mpsc::channel(1); + let rx = tokio_stream::wrappers::ReceiverStream::new(rx); - let node = nix_compat::nar::reader::open(r)?; - let (root_node, mut directory_putter, _) = process_node( - handle.clone(), - "".into(), // this is the root node, it has an empty name - node, - &blob_service, - directory_putter, - )?; + let produce = async move { + let res = produce_nar_inner( + blob_service, + root_node, + "root".parse().unwrap(), // HACK: the root node sent to ingest_entries may not be ROOT. + tx.clone(), + ) + .await; - // In case the root node points to a directory, we need to close - // [directory_putter], and ensure the digest we got back from there matches - // what the root node is pointing to. - if let castorepb::node::Node::Directory(ref directory_node) = root_node { - // Close directory_putter to make sure all directories have been inserted. - let directory_putter_digest = - handle.block_on(handle.spawn(async move { directory_putter.close().await }))??; - let root_directory_node_digest: B3Digest = - directory_node.digest.clone().try_into().unwrap(); + tx.send(res) + .await + .map_err(|e| Error::IO(std::io::Error::new(std::io::ErrorKind::BrokenPipe, e)))?; - if directory_putter_digest != root_directory_node_digest { - warn!( - root_directory_node_digest = %root_directory_node_digest, - directory_putter_digest =%directory_putter_digest, - "directory digest mismatch", - ); - return Err(io::Error::new( - io::ErrorKind::Other, - "directory digest mismatch", - )); - } - } - // In case it's not a Directory, [directory_putter] doesn't need to be - // closed (as we didn't end up uploading anything). - // It can just be dropped, as documented in its trait. + Ok(()) + }; + + let consume = ingest_entries(directory_service, rx); - Ok(root_node) + let (_, node) = try_join!(produce, consume)?; + + // remove the fake "root" name again + debug_assert_eq!(&node.get_name(), b"root"); + Ok(node.rename("".into())) } -/// This is called on a [nar::reader::Node] and returns a [castorepb::node::Node]. -/// It does so by handling all three kinds, and recursing for directories. -/// -/// [DirectoryPutter] is passed around, so a single instance of it can be used, -/// which is sufficient, as this reads through the whole NAR linerarly. -fn process_node<BS>( - handle: tokio::runtime::Handle, - name: bytes::Bytes, - node: nar::reader::Node, +async fn produce_nar_inner<BS>( blob_service: BS, - directory_putter: Box<dyn DirectoryPutter>, -) -> io::Result<(castorepb::node::Node, Box<dyn DirectoryPutter>, BS)> + node: nar_reader::Node<'_, '_>, + path: PathBuf, + tx: mpsc::Sender<Result<IngestionEntry, Error>>, +) -> Result<IngestionEntry, Error> where - BS: AsRef<dyn BlobService>, + BS: BlobService + Clone, { Ok(match node { - nar::reader::Node::Symlink { target } => ( - castorepb::node::Node::Symlink(castorepb::SymlinkNode { - name, - target: target.into(), - }), - directory_putter, - blob_service, - ), - nar::reader::Node::File { executable, reader } => ( - castorepb::node::Node::File(process_file_reader( - handle, - name, - reader, + nar_reader::Node::Symlink { target } => IngestionEntry::Symlink { path, target }, + nar_reader::Node::File { + executable, + mut reader, + } => { + let (digest, size) = { + let mut blob_writer = blob_service.open_write().await; + let size = tokio::io::copy_buf(&mut reader, &mut blob_writer).await?; + + (blob_writer.close().await?, size) + }; + + IngestionEntry::Regular { + path, + size, executable, - &blob_service, - )?), - directory_putter, - blob_service, - ), - nar::reader::Node::Directory(dir_reader) => { - let (directory_node, directory_putter, blob_service_back) = - process_dir_reader(handle, name, dir_reader, blob_service, directory_putter)?; - - ( - castorepb::node::Node::Directory(directory_node), - directory_putter, - blob_service_back, - ) + digest, + } + } + nar_reader::Node::Directory(mut dir_reader) => { + while let Some(entry) = dir_reader.next().await? { + let mut path = path.clone(); + + // valid NAR names are valid castore names + path.try_push(entry.name) + .expect("Tvix bug: failed to join name"); + + let entry = Box::pin(produce_nar_inner( + blob_service.clone(), + entry.node, + path, + tx.clone(), + )) + .await?; + + tx.send(Ok(entry)).await.map_err(|e| { + Error::IO(std::io::Error::new(std::io::ErrorKind::BrokenPipe, e)) + })?; + } + + IngestionEntry::Dir { path } } }) } -/// Given a name and [nar::reader::FileReader], this ingests the file into the -/// passed [BlobService] and returns a [castorepb::FileNode]. -fn process_file_reader<BS>( - handle: tokio::runtime::Handle, - name: Bytes, - mut file_reader: nar::reader::FileReader, - executable: bool, - blob_service: BS, -) -> io::Result<castorepb::FileNode> -where - BS: AsRef<dyn BlobService>, -{ - // store the length. If we read any other length, reading will fail. - let expected_len = file_reader.len(); - - // prepare writing a new blob. - let blob_writer = handle.block_on(async { blob_service.as_ref().open_write().await }); - - // write the blob. - let mut blob_writer = { - let mut dst = SyncIoBridge::new(blob_writer); - - file_reader.copy(&mut dst)?; - dst.shutdown()?; - - // return back the blob_writer - dst.into_inner() - }; - - // close the blob_writer, retrieve the digest. - let blob_digest = handle.block_on(async { blob_writer.close().await })?; - - Ok(castorepb::FileNode { - name, - digest: blob_digest.into(), - size: expected_len, - executable, - }) -} - -/// Given a name and [nar::reader::DirReader], this returns a [castorepb::DirectoryNode]. -/// It uses [process_node] to iterate over all children. -/// -/// [DirectoryPutter] is passed around, so a single instance of it can be used, -/// which is sufficient, as this reads through the whole NAR linerarly. -fn process_dir_reader<BS>( - handle: tokio::runtime::Handle, - name: Bytes, - mut dir_reader: nar::reader::DirReader, - blob_service: BS, - directory_putter: Box<dyn DirectoryPutter>, -) -> io::Result<(castorepb::DirectoryNode, Box<dyn DirectoryPutter>, BS)> -where - BS: AsRef<dyn BlobService>, -{ - let mut directory = castorepb::Directory::default(); - - let mut directory_putter = directory_putter; - let mut blob_service = blob_service; - while let Some(entry) = dir_reader.next()? { - let (node, directory_putter_back, blob_service_back) = process_node( - handle.clone(), - entry.name.into(), - entry.node, - blob_service, - directory_putter, - )?; - - blob_service = blob_service_back; - directory_putter = directory_putter_back; - - match node { - castorepb::node::Node::Directory(node) => directory.directories.push(node), - castorepb::node::Node::File(node) => directory.files.push(node), - castorepb::node::Node::Symlink(node) => directory.symlinks.push(node), - } - } - - // calculate digest and size. - let directory_digest = directory.digest(); - let directory_size = directory.size(); - - // upload the directory. This is a bit more verbose, as we want to get back - // directory_putter for later reuse. - let directory_putter = handle.block_on(handle.spawn(async move { - directory_putter.put(directory).await?; - Ok::<_, io::Error>(directory_putter) - }))??; - - Ok(( - castorepb::DirectoryNode { - name, - digest: directory_digest.into(), - size: directory_size, - }, - directory_putter, - blob_service, - )) +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error(transparent)] + IO(#[from] std::io::Error), } #[cfg(test)] mod test { - use crate::nar::read_nar; + use crate::nar::ingest_nar; use std::io::Cursor; use std::sync::Arc; @@ -244,19 +141,13 @@ mod test { blob_service: Arc<dyn BlobService>, directory_service: Arc<dyn DirectoryService>, ) { - let handle = tokio::runtime::Handle::current(); - - let root_node = handle - .spawn_blocking(|| { - read_nar( - &mut Cursor::new(&NAR_CONTENTS_SYMLINK.clone()), - blob_service, - directory_service, - ) - }) - .await - .unwrap() - .expect("must parse"); + let root_node = ingest_nar( + blob_service, + directory_service, + &mut Cursor::new(&NAR_CONTENTS_SYMLINK.clone()), + ) + .await + .expect("must parse"); assert_eq!( castorepb::node::Node::Symlink(castorepb::SymlinkNode { @@ -273,22 +164,13 @@ mod test { blob_service: Arc<dyn BlobService>, directory_service: Arc<dyn DirectoryService>, ) { - let handle = tokio::runtime::Handle::current(); - - let root_node = handle - .spawn_blocking({ - let blob_service = blob_service.clone(); - move || { - read_nar( - &mut Cursor::new(&NAR_CONTENTS_HELLOWORLD.clone()), - blob_service, - directory_service, - ) - } - }) - .await - .unwrap() - .expect("must parse"); + let root_node = ingest_nar( + blob_service.clone(), + directory_service, + &mut Cursor::new(&NAR_CONTENTS_HELLOWORLD.clone()), + ) + .await + .expect("must parse"); assert_eq!( castorepb::node::Node::File(castorepb::FileNode { @@ -310,23 +192,13 @@ mod test { blob_service: Arc<dyn BlobService>, directory_service: Arc<dyn DirectoryService>, ) { - let handle = tokio::runtime::Handle::current(); - - let root_node = handle - .spawn_blocking({ - let blob_service = blob_service.clone(); - let directory_service = directory_service.clone(); - || { - read_nar( - &mut Cursor::new(&NAR_CONTENTS_COMPLICATED.clone()), - blob_service, - directory_service, - ) - } - }) - .await - .unwrap() - .expect("must parse"); + let root_node = ingest_nar( + blob_service.clone(), + directory_service.clone(), + &mut Cursor::new(&NAR_CONTENTS_COMPLICATED.clone()), + ) + .await + .expect("must parse"); assert_eq!( castorepb::node::Node::Directory(castorepb::DirectoryNode { diff --git a/tvix/store/src/nar/mod.rs b/tvix/store/src/nar/mod.rs index 49bb92fb0f..4d5101f9d5 100644 --- a/tvix/store/src/nar/mod.rs +++ b/tvix/store/src/nar/mod.rs @@ -2,7 +2,7 @@ use tvix_castore::B3Digest; mod import; mod renderer; -pub use import::read_nar; +pub use import::ingest_nar; pub use renderer::calculate_size_and_sha256; pub use renderer::write_nar; diff --git a/tvix/store/src/nar/renderer.rs b/tvix/store/src/nar/renderer.rs index 9ac363ff57..3b39f700bd 100644 --- a/tvix/store/src/nar/renderer.rs +++ b/tvix/store/src/nar/renderer.rs @@ -1,14 +1,10 @@ +use crate::utils::AsyncIoBridge; + use super::RenderError; -use async_recursion::async_recursion; use count_write::CountWrite; use nix_compat::nar::writer::r#async as nar_writer; use sha2::{Digest, Sha256}; -use std::{ - pin::Pin, - task::{self, Poll}, -}; use tokio::io::{self, AsyncWrite, BufReader}; -use tokio_util::compat::{TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt}; use tvix_castore::{ blobservice::BlobService, directoryservice::DirectoryService, @@ -42,37 +38,12 @@ where Ok((cw.count(), h.finalize().into())) } -/// The inverse of [tokio_util::io::SyncIoBridge]. -/// Don't use this with anything that actually does blocking I/O. -struct AsyncIoBridge<T>(T); - -impl<W: std::io::Write + Unpin> AsyncWrite for AsyncIoBridge<W> { - fn poll_write( - self: Pin<&mut Self>, - _cx: &mut task::Context<'_>, - buf: &[u8], - ) -> Poll<io::Result<usize>> { - Poll::Ready(self.get_mut().0.write(buf)) - } - - fn poll_flush(self: Pin<&mut Self>, _cx: &mut task::Context<'_>) -> Poll<io::Result<()>> { - Poll::Ready(self.get_mut().0.flush()) - } - - fn poll_shutdown( - self: Pin<&mut Self>, - _cx: &mut task::Context<'_>, - ) -> Poll<Result<(), io::Error>> { - Poll::Ready(Ok(())) - } -} - /// Accepts a [castorepb::node::Node] pointing to the root of a (store) path, /// and uses the passed blob_service and directory_service to perform the /// necessary lookups as it traverses the structure. /// The contents in NAR serialization are writen to the passed [AsyncWrite]. pub async fn write_nar<W, BS, DS>( - w: W, + mut w: W, proto_root_node: &castorepb::node::Node, blob_service: BS, directory_service: DS, @@ -83,7 +54,6 @@ where DS: DirectoryService + Send, { // Initialize NAR writer - let mut w = w.compat_write(); let nar_root_node = nar_writer::open(&mut w) .await .map_err(RenderError::NARWriterError)?; @@ -101,9 +71,8 @@ where /// Process an intermediate node in the structure. /// This consumes the node. -#[async_recursion] async fn walk_node<BS, DS>( - nar_node: nar_writer::Node<'async_recursion, '_>, + nar_node: nar_writer::Node<'_, '_>, proto_node: &castorepb::node::Node, blob_service: BS, directory_service: DS, @@ -128,7 +97,7 @@ where )) })?; - let blob_reader = match blob_service + let mut blob_reader = match blob_service .open_read(&digest) .await .map_err(RenderError::StoreError)? @@ -144,7 +113,7 @@ where .file( proto_file_node.executable, proto_file_node.size, - &mut blob_reader.compat(), + &mut blob_reader, ) .await .map_err(RenderError::NARWriterError)?; @@ -193,9 +162,13 @@ where .await .map_err(RenderError::NARWriterError)?; - (blob_service, directory_service) = - walk_node(child_node, &proto_node, blob_service, directory_service) - .await?; + (blob_service, directory_service) = Box::pin(walk_node( + child_node, + &proto_node, + blob_service, + directory_service, + )) + .await?; } // close the directory diff --git a/tvix/store/src/pathinfoservice/bigtable.rs b/tvix/store/src/pathinfoservice/bigtable.rs new file mode 100644 index 0000000000..6fb52abbfd --- /dev/null +++ b/tvix/store/src/pathinfoservice/bigtable.rs @@ -0,0 +1,401 @@ +use super::PathInfoService; +use crate::proto; +use crate::proto::PathInfo; +use async_stream::try_stream; +use bigtable_rs::{bigtable, google::bigtable::v2 as bigtable_v2}; +use bytes::Bytes; +use data_encoding::HEXLOWER; +use futures::stream::BoxStream; +use prost::Message; +use serde::{Deserialize, Serialize}; +use serde_with::{serde_as, DurationSeconds}; +use tonic::async_trait; +use tracing::trace; +use tvix_castore::proto as castorepb; +use tvix_castore::Error; + +/// There should not be more than 10 MiB in a single cell. +/// https://cloud.google.com/bigtable/docs/schema-design#cells +const CELL_SIZE_LIMIT: u64 = 10 * 1024 * 1024; + +/// Provides a [DirectoryService] implementation using +/// [Bigtable](https://cloud.google.com/bigtable/docs/) +/// as an underlying K/V store. +/// +/// # Data format +/// We use Bigtable as a plain K/V store. +/// The row key is the digest of the store path, in hexlower. +/// Inside the row, we currently have a single column/cell, again using the +/// hexlower store path digest. +/// Its value is the PathInfo message, serialized in canonical protobuf. +/// We currently only populate this column. +/// +/// Listing is ranging over all rows, and calculate_nar is returning a +/// "unimplemented" error. +#[derive(Clone)] +pub struct BigtablePathInfoService { + client: bigtable::BigTable, + params: BigtableParameters, + + #[cfg(test)] + #[allow(dead_code)] + /// Holds the temporary directory containing the unix socket, and the + /// spawned emulator process. + emulator: std::sync::Arc<(tempfile::TempDir, async_process::Child)>, +} + +/// Represents configuration of [BigtablePathInfoService]. +/// This currently conflates both connect parameters and data model/client +/// behaviour parameters. +#[serde_as] +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] +pub struct BigtableParameters { + project_id: String, + instance_name: String, + #[serde(default)] + is_read_only: bool, + #[serde(default = "default_channel_size")] + channel_size: usize, + + #[serde_as(as = "Option<DurationSeconds<String>>")] + #[serde(default = "default_timeout")] + timeout: Option<std::time::Duration>, + table_name: String, + family_name: String, + + #[serde(default = "default_app_profile_id")] + app_profile_id: String, +} + +fn default_app_profile_id() -> String { + "default".to_owned() +} + +fn default_channel_size() -> usize { + 4 +} + +fn default_timeout() -> Option<std::time::Duration> { + Some(std::time::Duration::from_secs(4)) +} + +impl BigtablePathInfoService { + #[cfg(not(test))] + pub async fn connect(params: BigtableParameters) -> Result<Self, bigtable::Error> { + let connection = bigtable::BigTableConnection::new( + ¶ms.project_id, + ¶ms.instance_name, + params.is_read_only, + params.channel_size, + params.timeout, + ) + .await?; + + Ok(Self { + client: connection.client(), + params, + }) + } + + #[cfg(test)] + pub async fn connect(params: BigtableParameters) -> Result<Self, bigtable::Error> { + use std::time::Duration; + + use async_process::{Command, Stdio}; + use tempfile::TempDir; + use tokio_retry::{strategy::ExponentialBackoff, Retry}; + + let tmpdir = TempDir::new().unwrap(); + + let socket_path = tmpdir.path().join("cbtemulator.sock"); + + let emulator_process = Command::new("cbtemulator") + .arg("-address") + .arg(socket_path.clone()) + .stderr(Stdio::piped()) + .stdout(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .expect("failed to spawn emulator"); + + Retry::spawn( + ExponentialBackoff::from_millis(20) + .max_delay(Duration::from_secs(1)) + .take(3), + || async { + if socket_path.exists() { + Ok(()) + } else { + Err(()) + } + }, + ) + .await + .expect("failed to wait for socket"); + + // populate the emulator + for cmd in &[ + vec!["createtable", ¶ms.table_name], + vec!["createfamily", ¶ms.table_name, ¶ms.family_name], + ] { + Command::new("cbt") + .args({ + let mut args = vec![ + "-instance", + ¶ms.instance_name, + "-project", + ¶ms.project_id, + ]; + args.extend_from_slice(cmd); + args + }) + .env( + "BIGTABLE_EMULATOR_HOST", + format!("unix://{}", socket_path.to_string_lossy()), + ) + .output() + .await + .expect("failed to run cbt setup command"); + } + + let connection = bigtable_rs::bigtable::BigTableConnection::new_with_emulator( + &format!("unix://{}", socket_path.to_string_lossy()), + ¶ms.project_id, + ¶ms.instance_name, + false, + None, + )?; + + Ok(Self { + client: connection.client(), + params, + emulator: (tmpdir, emulator_process).into(), + }) + } +} + +/// Derives the row/column key for a given output path. +/// We use hexlower encoding, also because it can't be misinterpreted as RE2. +fn derive_pathinfo_key(digest: &[u8; 20]) -> String { + HEXLOWER.encode(digest) +} + +#[async_trait] +impl PathInfoService for BigtablePathInfoService { + async fn get(&self, digest: [u8; 20]) -> Result<Option<PathInfo>, Error> { + let mut client = self.client.clone(); + let path_info_key = derive_pathinfo_key(&digest); + + let request = bigtable_v2::ReadRowsRequest { + app_profile_id: self.params.app_profile_id.to_string(), + table_name: client.get_full_table_name(&self.params.table_name), + rows_limit: 1, + rows: Some(bigtable_v2::RowSet { + row_keys: vec![path_info_key.clone().into()], + row_ranges: vec![], + }), + // Filter selected family name, and column qualifier matching the digest. + // The latter is to ensure we don't fail once we start adding more metadata. + filter: Some(bigtable_v2::RowFilter { + filter: Some(bigtable_v2::row_filter::Filter::Chain( + bigtable_v2::row_filter::Chain { + filters: vec![ + bigtable_v2::RowFilter { + filter: Some( + bigtable_v2::row_filter::Filter::FamilyNameRegexFilter( + self.params.family_name.to_string(), + ), + ), + }, + bigtable_v2::RowFilter { + filter: Some( + bigtable_v2::row_filter::Filter::ColumnQualifierRegexFilter( + path_info_key.clone().into(), + ), + ), + }, + ], + }, + )), + }), + ..Default::default() + }; + + let mut response = client + .read_rows(request) + .await + .map_err(|e| Error::StorageError(format!("unable to read rows: {}", e)))?; + + if response.len() != 1 { + if response.len() > 1 { + // This shouldn't happen, we limit number of rows to 1 + return Err(Error::StorageError( + "got more than one row from bigtable".into(), + )); + } + // else, this is simply a "not found". + return Ok(None); + } + + let (row_key, mut cells) = response.pop().unwrap(); + if row_key != path_info_key.as_bytes() { + // This shouldn't happen, we requested this row key. + return Err(Error::StorageError( + "got wrong row key from bigtable".into(), + )); + } + + let cell = cells + .pop() + .ok_or_else(|| Error::StorageError("found no cells".into()))?; + + // Ensure there's only one cell (so no more left after the pop()) + // This shouldn't happen, We filter out other cells in our query. + if !cells.is_empty() { + return Err(Error::StorageError( + "more than one cell returned from bigtable".into(), + )); + } + + // We also require the qualifier to be correct in the filter above, + // so this shouldn't happen. + if path_info_key.as_bytes() != cell.qualifier { + return Err(Error::StorageError("unexpected cell qualifier".into())); + } + + // Try to parse the value into a PathInfo message + let path_info = proto::PathInfo::decode(Bytes::from(cell.value)) + .map_err(|e| Error::StorageError(format!("unable to decode pathinfo proto: {}", e)))?; + + let store_path = path_info + .validate() + .map_err(|e| Error::StorageError(format!("invalid PathInfo: {}", e)))?; + + if store_path.digest() != &digest { + return Err(Error::StorageError("PathInfo has unexpected digest".into())); + } + + Ok(Some(path_info)) + } + + async fn put(&self, path_info: PathInfo) -> Result<PathInfo, Error> { + let store_path = path_info + .validate() + .map_err(|e| Error::InvalidRequest(format!("pathinfo failed validation: {}", e)))?; + + let mut client = self.client.clone(); + let path_info_key = derive_pathinfo_key(store_path.digest()); + + let data = path_info.encode_to_vec(); + if data.len() as u64 > CELL_SIZE_LIMIT { + return Err(Error::StorageError( + "PathInfo exceeds cell limit on Bigtable".into(), + )); + } + + let resp = client + .check_and_mutate_row(bigtable_v2::CheckAndMutateRowRequest { + table_name: client.get_full_table_name(&self.params.table_name), + app_profile_id: self.params.app_profile_id.to_string(), + row_key: path_info_key.clone().into(), + predicate_filter: Some(bigtable_v2::RowFilter { + filter: Some(bigtable_v2::row_filter::Filter::ColumnQualifierRegexFilter( + path_info_key.clone().into(), + )), + }), + // If the column was already found, do nothing. + true_mutations: vec![], + // Else, do the insert. + false_mutations: vec![ + // https://cloud.google.com/bigtable/docs/writes + bigtable_v2::Mutation { + mutation: Some(bigtable_v2::mutation::Mutation::SetCell( + bigtable_v2::mutation::SetCell { + family_name: self.params.family_name.to_string(), + column_qualifier: path_info_key.clone().into(), + timestamp_micros: -1, // use server time to fill timestamp + value: data, + }, + )), + }, + ], + }) + .await + .map_err(|e| Error::StorageError(format!("unable to mutate rows: {}", e)))?; + + if resp.predicate_matched { + trace!("already existed") + } + + Ok(path_info) + } + + async fn calculate_nar( + &self, + _root_node: &castorepb::node::Node, + ) -> Result<(u64, [u8; 32]), Error> { + return Err(Error::StorageError("unimplemented".into())); + } + + fn list(&self) -> BoxStream<'static, Result<PathInfo, Error>> { + let mut client = self.client.clone(); + + let request = bigtable_v2::ReadRowsRequest { + app_profile_id: self.params.app_profile_id.to_string(), + table_name: client.get_full_table_name(&self.params.table_name), + filter: Some(bigtable_v2::RowFilter { + filter: Some(bigtable_v2::row_filter::Filter::FamilyNameRegexFilter( + self.params.family_name.to_string(), + )), + }), + ..Default::default() + }; + + let stream = try_stream! { + // TODO: add pagination, we don't want to hold all of this in memory. + let response = client + .read_rows(request) + .await + .map_err(|e| Error::StorageError(format!("unable to read rows: {}", e)))?; + + for (row_key, mut cells) in response { + let cell = cells + .pop() + .ok_or_else(|| Error::StorageError("found no cells".into()))?; + + // The cell must have the same qualifier as the row key + if row_key != cell.qualifier { + Err(Error::StorageError("unexpected cell qualifier".into()))?; + } + + // Ensure there's only one cell (so no more left after the pop()) + // This shouldn't happen, We filter out other cells in our query. + if !cells.is_empty() { + + Err(Error::StorageError( + "more than one cell returned from bigtable".into(), + ))? + } + + // Try to parse the value into a PathInfo message. + let path_info = proto::PathInfo::decode(Bytes::from(cell.value)) + .map_err(|e| Error::StorageError(format!("unable to decode pathinfo proto: {}", e)))?; + + // Validate the containing PathInfo, ensure its StorePath digest + // matches row key. + let store_path = path_info + .validate() + .map_err(|e| Error::StorageError(format!("invalid PathInfo: {}", e)))?; + + if store_path.digest().as_slice() != row_key.as_slice() { + Err(Error::StorageError("PathInfo has unexpected digest".into()))? + } + + + yield path_info + } + }; + + Box::pin(stream) + } +} diff --git a/tvix/store/src/pathinfoservice/from_addr.rs b/tvix/store/src/pathinfoservice/from_addr.rs index 6109054d7a..f22884ca47 100644 --- a/tvix/store/src/pathinfoservice/from_addr.rs +++ b/tvix/store/src/pathinfoservice/from_addr.rs @@ -37,7 +37,8 @@ pub async fn from_addr( blob_service: Arc<dyn BlobService>, directory_service: Arc<dyn DirectoryService>, ) -> Result<Box<dyn PathInfoService>, Error> { - let url = + #[allow(unused_mut)] + let mut url = Url::parse(uri).map_err(|e| Error::StorageError(format!("unable to parse url: {}", e)))?; let path_info_service: Box<dyn PathInfoService> = match url.scheme() { @@ -108,6 +109,30 @@ pub async fn from_addr( PathInfoServiceClient::new(tvix_castore::tonic::channel_from_url(&url).await?); Box::new(GRPCPathInfoService::from_client(client)) } + #[cfg(feature = "cloud")] + "bigtable" => { + use super::bigtable::BigtableParameters; + use super::BigtablePathInfoService; + + // parse the instance name from the hostname. + let instance_name = url + .host_str() + .ok_or_else(|| Error::StorageError("instance name missing".into()))? + .to_string(); + + // โฆ but add it to the query string now, so we just need to parse that. + url.query_pairs_mut() + .append_pair("instance_name", &instance_name); + + let params: BigtableParameters = serde_qs::from_str(url.query().unwrap_or_default()) + .map_err(|e| Error::InvalidRequest(format!("failed to parse parameters: {}", e)))?; + + Box::new( + BigtablePathInfoService::connect(params) + .await + .map_err(|e| Error::StorageError(e.to_string()))?, + ) + } _ => Err(Error::StorageError(format!( "unknown scheme: {}", url.scheme() @@ -121,9 +146,9 @@ pub async fn from_addr( mod tests { use super::from_addr; use lazy_static::lazy_static; + use rstest::rstest; use std::sync::Arc; use tempfile::TempDir; - use test_case::test_case; use tvix_castore::{ blobservice::{BlobService, MemoryBlobService}, directoryservice::{DirectoryService, MemoryDirectoryService}, @@ -136,52 +161,66 @@ mod tests { // the gRPC tests below don't fail, because we connect lazily. + #[rstest] /// This uses a unsupported scheme. - #[test_case("http://foo.example/test", false; "unsupported scheme")] + #[case::unsupported_scheme("http://foo.example/test", false)] /// This configures sled in temporary mode. - #[test_case("sled://", true; "sled valid temporary")] + #[case::sled_temporary("sled://", true)] /// This configures sled with /, which should fail. - #[test_case("sled:///", false; "sled invalid root")] + #[case::sled_invalid_root("sled:///", false)] /// This configures sled with a host, not path, which should fail. - #[test_case("sled://foo.example", false; "sled invalid host")] + #[case::sled_invalid_host("sled://foo.example", false)] /// This configures sled with a valid path path, which should succeed. - #[test_case(&format!("sled://{}", &TMPDIR_SLED_1.path().to_str().unwrap()), true; "sled valid path")] + #[case::sled_valid_path(&format!("sled://{}", &TMPDIR_SLED_1.path().to_str().unwrap()), true)] /// This configures sled with a host, and a valid path path, which should fail. - #[test_case(&format!("sled://foo.example{}", &TMPDIR_SLED_2.path().to_str().unwrap()), false; "sled invalid host with valid path")] + #[case::sled_invalid_host_with_valid_path(&format!("sled://foo.example{}", &TMPDIR_SLED_2.path().to_str().unwrap()), false)] /// This correctly sets the scheme, and doesn't set a path. - #[test_case("memory://", true; "memory valid")] + #[case::memory_valid("memory://", true)] /// This sets a memory url host to `foo` - #[test_case("memory://foo", false; "memory invalid host")] + #[case::memory_invalid_host("memory://foo", false)] /// This sets a memory url path to "/", which is invalid. - #[test_case("memory:///", false; "memory invalid root path")] + #[case::memory_invalid_root_path("memory:///", false)] /// This sets a memory url path to "/foo", which is invalid. - #[test_case("memory:///foo", false; "memory invalid root path foo")] + #[case::memory_invalid_root_path_foo("memory:///foo", false)] /// Correct Scheme for the cache.nixos.org binary cache. - #[test_case("nix+https://cache.nixos.org", true; "correct nix+https")] + #[case::correct_nix_https("nix+https://cache.nixos.org", true)] /// Correct Scheme for the cache.nixos.org binary cache (HTTP URL). - #[test_case("nix+http://cache.nixos.org", true; "correct nix+http")] + #[case::correct_nix_http("nix+http://cache.nixos.org", true)] /// Correct Scheme for Nix HTTP Binary cache, with a subpath. - #[test_case("nix+http://192.0.2.1/foo", true; "correct nix http with subpath")] + #[case::correct_nix_http_with_subpath("nix+http://192.0.2.1/foo", true)] /// Correct Scheme for Nix HTTP Binary cache, with a subpath and port. - #[test_case("nix+http://[::1]:8080/foo", true; "correct nix http with subpath and port")] + #[case::correct_nix_http_with_subpath_and_port("nix+http://[::1]:8080/foo", true)] /// Correct Scheme for the cache.nixos.org binary cache, and correct trusted public key set - #[test_case("nix+https://cache.nixos.org?trusted-public-keys=cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=", true; "correct nix+https with trusted-public-key")] + #[case::correct_nix_https_with_trusted_public_key("nix+https://cache.nixos.org?trusted-public-keys=cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=", true)] /// Correct Scheme for the cache.nixos.org binary cache, and two correct trusted public keys set - #[test_case("nix+https://cache.nixos.org?trusted-public-keys=cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=%20foo:jp4fCEx9tBEId/L0ZsVJ26k0wC0fu7vJqLjjIGFkup8=", true; "correct nix+https with two trusted-public-key")] + #[case::correct_nix_https_with_two_trusted_public_keys("nix+https://cache.nixos.org?trusted-public-keys=cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=%20foo:jp4fCEx9tBEId/L0ZsVJ26k0wC0fu7vJqLjjIGFkup8=", true)] /// Correct scheme to connect to a unix socket. - #[test_case("grpc+unix:///path/to/somewhere", true; "grpc valid unix socket")] + #[case::grpc_valid_unix_socket("grpc+unix:///path/to/somewhere", true)] /// Correct scheme for unix socket, but setting a host too, which is invalid. - #[test_case("grpc+unix://host.example/path/to/somewhere", false; "grpc invalid unix socket and host")] + #[case::grpc_invalid_unix_socket_and_host("grpc+unix://host.example/path/to/somewhere", false)] /// Correct scheme to connect to localhost, with port 12345 - #[test_case("grpc+http://[::1]:12345", true; "grpc valid IPv6 localhost port 12345")] + #[case::grpc_valid_ipv6_localhost_port_12345("grpc+http://[::1]:12345", true)] /// Correct scheme to connect to localhost over http, without specifying a port. - #[test_case("grpc+http://localhost", true; "grpc valid http host without port")] + #[case::grpc_valid_http_host_without_port("grpc+http://localhost", true)] /// Correct scheme to connect to localhost over http, without specifying a port. - #[test_case("grpc+https://localhost", true; "grpc valid https host without port")] + #[case::grpc_valid_https_host_without_port("grpc+https://localhost", true)] /// Correct scheme to connect to localhost over http, but with additional path, which is invalid. - #[test_case("grpc+http://localhost/some-path", false; "grpc valid invalid host and path")] + #[case::grpc_invalid_host_and_path("grpc+http://localhost/some-path", false)] + /// A valid example for Bigtable. + #[cfg_attr( + all(feature = "cloud", feature = "integration"), + case::bigtable_valid( + "bigtable://instance-1?project_id=project-1&table_name=table-1&family_name=cf1", + true + ) + )] + /// An invalid example for Bigtable, missing fields + #[cfg_attr( + all(feature = "cloud", feature = "integration"), + case::bigtable_invalid_missing_fields("bigtable://instance-1", false) + )] #[tokio::test] - async fn test_from_addr_tokio(uri_str: &str, exp_succeed: bool) { + async fn test_from_addr_tokio(#[case] uri_str: &str, #[case] exp_succeed: bool) { let blob_service: Arc<dyn BlobService> = Arc::from(MemoryBlobService::default()); let directory_service: Arc<dyn DirectoryService> = Arc::from(MemoryDirectoryService::default()); diff --git a/tvix/store/src/pathinfoservice/fs/mod.rs b/tvix/store/src/pathinfoservice/fs/mod.rs index 45d59fd0bc..aa64b1c01f 100644 --- a/tvix/store/src/pathinfoservice/fs/mod.rs +++ b/tvix/store/src/pathinfoservice/fs/mod.rs @@ -17,6 +17,7 @@ pub fn make_fs<BS, DS, PS>( directory_service: DS, path_info_service: PS, list_root: bool, + show_xattr: bool, ) -> TvixStoreFs<BS, DS, RootNodesWrapper<PS>> where BS: AsRef<dyn BlobService> + Send + Clone + 'static, @@ -28,6 +29,7 @@ where directory_service, RootNodesWrapper(path_info_service), list_root, + show_xattr, ) } diff --git a/tvix/store/src/pathinfoservice/grpc.rs b/tvix/store/src/pathinfoservice/grpc.rs index 7d740429cf..1138ebdc19 100644 --- a/tvix/store/src/pathinfoservice/grpc.rs +++ b/tvix/store/src/pathinfoservice/grpc.rs @@ -1,8 +1,10 @@ use super::PathInfoService; use crate::proto::{self, ListPathInfoRequest, PathInfo}; use async_stream::try_stream; +use data_encoding::BASE64; use futures::stream::BoxStream; use tonic::{async_trait, transport::Channel, Code}; +use tracing::instrument; use tvix_castore::{proto as castorepb, Error}; /// Connects to a (remote) tvix-store PathInfoService over gRPC. @@ -25,6 +27,7 @@ impl GRPCPathInfoService { #[async_trait] impl PathInfoService for GRPCPathInfoService { + #[instrument(level = "trace", skip_all, fields(path_info.digest = BASE64.encode(&digest)))] async fn get(&self, digest: [u8; 20]) -> Result<Option<PathInfo>, Error> { let path_info = self .grpc_client @@ -51,6 +54,7 @@ impl PathInfoService for GRPCPathInfoService { } } + #[instrument(level = "trace", skip_all, fields(path_info.root_node = ?path_info.node))] async fn put(&self, path_info: PathInfo) -> Result<PathInfo, Error> { let path_info = self .grpc_client @@ -63,6 +67,7 @@ impl PathInfoService for GRPCPathInfoService { Ok(path_info) } + #[instrument(level = "trace", skip_all, fields(root_node = ?root_node))] async fn calculate_nar( &self, root_node: &castorepb::node::Node, @@ -86,6 +91,7 @@ impl PathInfoService for GRPCPathInfoService { Ok((path_info.nar_size, nar_sha256)) } + #[instrument(level = "trace", skip_all)] fn list(&self) -> BoxStream<'static, Result<PathInfo, Error>> { let mut grpc_client = self.grpc_client.clone(); @@ -201,7 +207,7 @@ mod tests { }; let path_info = grpc_client - .get(fixtures::DUMMY_OUTPUT_HASH) + .get(fixtures::DUMMY_PATH_DIGEST) .await .expect("must not be error"); diff --git a/tvix/store/src/pathinfoservice/mod.rs b/tvix/store/src/pathinfoservice/mod.rs index c334c8dc56..c1a482bbb5 100644 --- a/tvix/store/src/pathinfoservice/mod.rs +++ b/tvix/store/src/pathinfoservice/mod.rs @@ -23,6 +23,11 @@ pub use self::memory::MemoryPathInfoService; pub use self::nix_http::NixHTTPPathInfoService; pub use self::sled::SledPathInfoService; +#[cfg(feature = "cloud")] +mod bigtable; +#[cfg(feature = "cloud")] +pub use self::bigtable::BigtablePathInfoService; + #[cfg(any(feature = "fuse", feature = "virtiofs"))] pub use self::fs::make_fs; diff --git a/tvix/store/src/pathinfoservice/nix_http.rs b/tvix/store/src/pathinfoservice/nix_http.rs index bdb0e2c3cb..581eb7ca7a 100644 --- a/tvix/store/src/pathinfoservice/nix_http.rs +++ b/tvix/store/src/pathinfoservice/nix_http.rs @@ -1,5 +1,3 @@ -use std::io::{self, BufRead, Read, Write}; - use data_encoding::BASE64; use futures::{stream::BoxStream, TryStreamExt}; use nix_compat::{ @@ -8,7 +6,10 @@ use nix_compat::{ nixhash::NixHash, }; use reqwest::StatusCode; -use sha2::{digest::FixedOutput, Digest, Sha256}; +use sha2::Digest; +use std::io::{self, Write}; +use tokio::io::{AsyncRead, BufReader}; +use tokio_util::io::InspectReader; use tonic::async_trait; use tracing::{debug, instrument, warn}; use tvix_castore::{ @@ -171,85 +172,83 @@ where ))); } - // get an AsyncRead of the response body. - let async_r = tokio_util::io::StreamReader::new(resp.bytes_stream().map_err(|e| { + // get a reader of the response body. + let r = tokio_util::io::StreamReader::new(resp.bytes_stream().map_err(|e| { let e = e.without_url(); warn!(e=%e, "failed to get response body"); io::Error::new(io::ErrorKind::BrokenPipe, e.to_string()) })); - let sync_r = tokio_util::io::SyncIoBridge::new(async_r); - // handle decompression, by wrapping the reader. - let sync_r: Box<dyn BufRead + Send> = match narinfo.compression { - Some("none") => Box::new(sync_r), - Some("xz") => Box::new(io::BufReader::new(xz2::read::XzDecoder::new(sync_r))), - Some(comp) => { - return Err(Error::InvalidRequest( - format!("unsupported compression: {}", comp).to_string(), - )) - } - None => { - return Err(Error::InvalidRequest( - "unsupported compression: bzip2".to_string(), - )) + // handle decompression, depending on the compression field. + let r: Box<dyn AsyncRead + Send + Unpin> = match narinfo.compression { + Some("none") => Box::new(r) as Box<dyn AsyncRead + Send + Unpin>, + Some("bzip2") | None => Box::new(async_compression::tokio::bufread::BzDecoder::new(r)) + as Box<dyn AsyncRead + Send + Unpin>, + Some("gzip") => Box::new(async_compression::tokio::bufread::GzipDecoder::new(r)) + as Box<dyn AsyncRead + Send + Unpin>, + Some("xz") => Box::new(async_compression::tokio::bufread::XzDecoder::new(r)) + as Box<dyn AsyncRead + Send + Unpin>, + Some("zstd") => Box::new(async_compression::tokio::bufread::ZstdDecoder::new(r)) + as Box<dyn AsyncRead + Send + Unpin>, + Some(comp_str) => { + return Err(Error::StorageError(format!( + "unsupported compression: {comp_str}" + ))); } }; - - let res = tokio::task::spawn_blocking({ - let blob_service = self.blob_service.clone(); - let directory_service = self.directory_service.clone(); - move || -> io::Result<_> { - // Wrap the reader once more, so we can calculate NarSize and NarHash - let mut sync_r = io::BufReader::new(NarReader::from(sync_r)); - let root_node = crate::nar::read_nar(&mut sync_r, blob_service, directory_service)?; - - let (_, nar_hash, nar_size) = sync_r.into_inner().into_inner(); - - Ok((root_node, nar_hash, nar_size)) - } - }) + let mut nar_hash = sha2::Sha256::new(); + let mut nar_size = 0; + + // Assemble NarHash and NarSize as we read bytes. + let r = InspectReader::new(r, |b| { + nar_size += b.len() as u64; + nar_hash.write_all(b).unwrap(); + }); + + // HACK: InspectReader doesn't implement AsyncBufRead, but neither do our decompressors. + let mut r = BufReader::new(r); + + let root_node = crate::nar::ingest_nar( + self.blob_service.clone(), + self.directory_service.clone(), + &mut r, + ) .await - .unwrap(); - - match res { - Ok((root_node, nar_hash, nar_size)) => { - // ensure the ingested narhash and narsize do actually match. - if narinfo.nar_size != nar_size { - warn!( - narinfo.nar_size = narinfo.nar_size, - http.nar_size = nar_size, - "NARSize mismatch" - ); - Err(io::Error::new( - io::ErrorKind::InvalidData, - "NarSize mismatch".to_string(), - ))?; - } - if narinfo.nar_hash != nar_hash { - warn!( - narinfo.nar_hash = %NixHash::Sha256(narinfo.nar_hash), - http.nar_hash = %NixHash::Sha256(nar_hash), - "NarHash mismatch" - ); - Err(io::Error::new( - io::ErrorKind::InvalidData, - "NarHash mismatch".to_string(), - ))?; - } - - Ok(Some(PathInfo { - node: Some(castorepb::Node { - // set the name of the root node to the digest-name of the store path. - node: Some( - root_node.rename(narinfo.store_path.to_string().to_owned().into()), - ), - }), - references: pathinfo.references, - narinfo: pathinfo.narinfo, - })) - } - Err(e) => Err(e.into()), + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + + // ensure the ingested narhash and narsize do actually match. + if narinfo.nar_size != nar_size { + warn!( + narinfo.nar_size = narinfo.nar_size, + http.nar_size = nar_size, + "NarSize mismatch" + ); + Err(io::Error::new( + io::ErrorKind::InvalidData, + "NarSize mismatch".to_string(), + ))?; } + let nar_hash: [u8; 32] = nar_hash.finalize().into(); + if narinfo.nar_hash != nar_hash { + warn!( + narinfo.nar_hash = %NixHash::Sha256(narinfo.nar_hash), + http.nar_hash = %NixHash::Sha256(nar_hash), + "NarHash mismatch" + ); + Err(io::Error::new( + io::ErrorKind::InvalidData, + "NarHash mismatch".to_string(), + ))?; + } + + Ok(Some(PathInfo { + node: Some(castorepb::Node { + // set the name of the root node to the digest-name of the store path. + node: Some(root_node.rename(narinfo.store_path.to_string().to_owned().into())), + }), + references: pathinfo.references, + narinfo: pathinfo.narinfo, + })) } #[instrument(skip_all, fields(path_info=?_path_info))] @@ -277,38 +276,3 @@ where })) } } - -/// Small helper reader implementing [std::io::Read]. -/// It can be used to wrap another reader, counts the number of bytes read -/// and the sha256 digest of the contents. -struct NarReader<R: Read> { - r: R, - - sha256: sha2::Sha256, - bytes_read: u64, -} - -impl<R: Read> NarReader<R> { - pub fn from(inner: R) -> Self { - Self { - r: inner, - sha256: Sha256::new(), - bytes_read: 0, - } - } - - /// Returns the (remaining) inner reader, the sha256 digest and the number of bytes read. - pub fn into_inner(self) -> (R, [u8; 32], u64) { - (self.r, self.sha256.finalize_fixed().into(), self.bytes_read) - } -} - -impl<R: Read> Read for NarReader<R> { - fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { - self.r.read(buf).map(|n| { - self.bytes_read += n as u64; - self.sha256.write_all(&buf[..n]).unwrap(); - n - }) - } -} diff --git a/tvix/store/src/pathinfoservice/sled.rs b/tvix/store/src/pathinfoservice/sled.rs index 7b6d7fd7ab..0255c031e2 100644 --- a/tvix/store/src/pathinfoservice/sled.rs +++ b/tvix/store/src/pathinfoservice/sled.rs @@ -1,11 +1,13 @@ use super::PathInfoService; use crate::nar::calculate_size_and_sha256; use crate::proto::PathInfo; +use data_encoding::BASE64; use futures::stream::iter; use futures::stream::BoxStream; use prost::Message; use std::path::Path; use tonic::async_trait; +use tracing::instrument; use tracing::warn; use tvix_castore::proto as castorepb; use tvix_castore::{blobservice::BlobService, directoryservice::DirectoryService, Error}; @@ -57,53 +59,46 @@ where BS: AsRef<dyn BlobService> + Send + Sync, DS: AsRef<dyn DirectoryService> + Send + Sync, { + #[instrument(level = "trace", skip_all, fields(path_info.digest = BASE64.encode(&digest)))] async fn get(&self, digest: [u8; 20]) -> Result<Option<PathInfo>, Error> { - match self.db.get(digest) { - Ok(None) => Ok(None), - Ok(Some(data)) => match PathInfo::decode(&*data) { - Ok(path_info) => Ok(Some(path_info)), - Err(e) => { + match self.db.get(digest).map_err(|e| { + warn!("failed to retrieve PathInfo: {}", e); + Error::StorageError(format!("failed to retrieve PathInfo: {}", e)) + })? { + None => Ok(None), + Some(data) => { + let path_info = PathInfo::decode(&*data).map_err(|e| { warn!("failed to decode stored PathInfo: {}", e); - Err(Error::StorageError(format!( - "failed to decode stored PathInfo: {}", - e - ))) - } - }, - Err(e) => { - warn!("failed to retrieve PathInfo: {}", e); - Err(Error::StorageError(format!( - "failed to retrieve PathInfo: {}", - e - ))) + Error::StorageError(format!("failed to decode stored PathInfo: {}", e)) + })?; + Ok(Some(path_info)) } } } + #[instrument(level = "trace", skip_all, fields(path_info.root_node = ?path_info.node))] async fn put(&self, path_info: PathInfo) -> Result<PathInfo, Error> { // Call validate on the received PathInfo message. - match path_info.validate() { - Err(e) => Err(Error::InvalidRequest(format!( - "failed to validate PathInfo: {}", - e - ))), - // In case the PathInfo is valid, and we were able to extract a NixPath, store it in the database. - // This overwrites existing PathInfo objects. - Ok(nix_path) => match self - .db - .insert(*nix_path.digest(), path_info.encode_to_vec()) - { - Ok(_) => Ok(path_info), - Err(e) => { - warn!("failed to insert PathInfo: {}", e); - Err(Error::StorageError(format! { - "failed to insert PathInfo: {}", e - })) - } - }, - } + let store_path = path_info + .validate() + .map_err(|e| Error::InvalidRequest(format!("failed to validate PathInfo: {}", e)))?; + + // In case the PathInfo is valid, we were able to parse a StorePath. + // Store it in the database, keyed by its digest. + // This overwrites existing PathInfo objects. + self.db + .insert(store_path.digest(), path_info.encode_to_vec()) + .map_err(|e| { + warn!("failed to insert PathInfo: {}", e); + Error::StorageError(format! { + "failed to insert PathInfo: {}", e + }) + })?; + + Ok(path_info) } + #[instrument(level = "trace", skip_all, fields(root_node = ?root_node))] async fn calculate_nar( &self, root_node: &castorepb::node::Node, @@ -114,27 +109,17 @@ where } fn list(&self) -> BoxStream<'static, Result<PathInfo, Error>> { - Box::pin(iter(self.db.iter().values().map(|v| match v { - Ok(data) => { - // we retrieved some bytes - match PathInfo::decode(&*data) { - Ok(path_info) => Ok(path_info), - Err(e) => { - warn!("failed to decode stored PathInfo: {}", e); - Err(Error::StorageError(format!( - "failed to decode stored PathInfo: {}", - e - ))) - } - } - } - Err(e) => { + Box::pin(iter(self.db.iter().values().map(|v| { + let data = v.map_err(|e| { warn!("failed to retrieve PathInfo: {}", e); - Err(Error::StorageError(format!( - "failed to retrieve PathInfo: {}", - e - ))) - } + Error::StorageError(format!("failed to retrieve PathInfo: {}", e)) + })?; + + let path_info = PathInfo::decode(&*data).map_err(|e| { + warn!("failed to decode stored PathInfo: {}", e); + Error::StorageError(format!("failed to decode stored PathInfo: {}", e)) + })?; + Ok(path_info) }))) } } diff --git a/tvix/store/src/pathinfoservice/tests/mod.rs b/tvix/store/src/pathinfoservice/tests/mod.rs index a3035d094d..9719371592 100644 --- a/tvix/store/src/pathinfoservice/tests/mod.rs +++ b/tvix/store/src/pathinfoservice/tests/mod.rs @@ -10,7 +10,7 @@ use tvix_castore::{blobservice::BlobService, directoryservice::DirectoryService} use super::PathInfoService; use crate::proto::PathInfo; -use crate::tests::fixtures::DUMMY_OUTPUT_HASH; +use crate::tests::fixtures::DUMMY_PATH_DIGEST; mod utils; use self::utils::make_grpc_path_info_service_client; @@ -51,6 +51,7 @@ pub async fn make_path_info_service(uri: &str) -> BSDSPS { #[case::memory(make_path_info_service("memory://").await)] #[case::grpc(make_grpc_path_info_service_client().await)] #[case::sled(make_path_info_service("sled://").await)] +#[cfg_attr(all(feature = "cloud",feature="integration"), case::bigtable(make_path_info_service("bigtable://instance-1?project_id=project-1&table_name=table-1&family_name=cf1").await))] pub fn path_info_services( #[case] services: ( impl BlobService, @@ -70,7 +71,7 @@ pub fn path_info_services( async fn not_found(services: BSDSPS) { let (_, _, path_info_service) = services; assert!(path_info_service - .get(DUMMY_OUTPUT_HASH) + .get(DUMMY_PATH_DIGEST) .await .expect("must succeed") .is_none()); @@ -104,7 +105,7 @@ async fn put_get(services: BSDSPS) { // get it back let resp = path_info_service - .get(DUMMY_OUTPUT_HASH) + .get(DUMMY_PATH_DIGEST) .await .expect("must succeed"); diff --git a/tvix/store/src/proto/mod.rs b/tvix/store/src/proto/mod.rs index 10fde33527..a09839c8bd 100644 --- a/tvix/store/src/proto/mod.rs +++ b/tvix/store/src/proto/mod.rs @@ -74,23 +74,20 @@ pub enum ValidatePathInfoError { /// Parses a root node name. /// -/// On success, this returns the parsed [store_path::StorePath]. +/// On success, this returns the parsed [store_path::StorePathRef]. /// On error, it returns an error generated from the supplied constructor. fn parse_node_name_root<E>( name: &[u8], err: fn(Vec<u8>, store_path::Error) -> E, -) -> Result<store_path::StorePath, E> { - match store_path::StorePath::from_bytes(name) { - Ok(np) => Ok(np), - Err(e) => Err(err(name.to_vec(), e)), - } +) -> Result<store_path::StorePathRef<'_>, E> { + store_path::StorePathRef::from_bytes(name).map_err(|e| err(name.to_vec(), e)) } impl PathInfo { /// validate performs some checks on the PathInfo struct, /// Returning either a [store_path::StorePath] of the root node, or a /// [ValidatePathInfoError]. - pub fn validate(&self) -> Result<store_path::StorePath, ValidatePathInfoError> { + pub fn validate(&self) -> Result<store_path::StorePathRef<'_>, ValidatePathInfoError> { // ensure the references have the right number of bytes. for (i, reference) in self.references.iter().enumerate() { if reference.len() != store_path::DIGEST_SIZE { @@ -154,8 +151,7 @@ impl PathInfo { // converting to this field. if let Some(deriver) = &narinfo.deriver { store_path::StorePathRef::from_name_and_digest(&deriver.name, &deriver.digest) - .map_err(ValidatePathInfoError::InvalidDeriverField)? - .to_owned(); + .map_err(ValidatePathInfoError::InvalidDeriverField)?; } } } @@ -177,12 +173,16 @@ impl PathInfo { Ok(root_nix_path) } - /// With self and a given StorePathRef, this reconstructs a + /// With self and its store path name, this reconstructs a /// [nix_compat::narinfo::NarInfo<'_>]. /// It can be used to validate Signatures, or get back a (sparse) NarInfo /// struct to prepare writing it out. /// - /// This doesn't allocate any new data. + /// It assumes self to be validated first, and will only return None if the + /// `narinfo` field is unpopulated. + /// + /// It does very little allocation (a Vec each for `signatures` and + /// `references`), the rest points to data owned elsewhere. /// /// Keep in mind this is not able to reconstruct all data present in the /// NarInfo<'_>, as some of it is not stored at all: @@ -192,7 +192,7 @@ impl PathInfo { /// /// If you want to render it out to a string and be able to parse it back /// in, at least URL *must* be set again. - pub fn as_narinfo<'a>( + pub fn to_narinfo<'a>( &'a self, store_path: store_path::StorePathRef<'a>, ) -> Option<nix_compat::narinfo::NarInfo<'_>> { @@ -201,7 +201,11 @@ impl PathInfo { Some(nix_compat::narinfo::NarInfo { flags: Flags::empty(), store_path, - nar_hash: narinfo.nar_sha256.to_vec().try_into().unwrap(), + nar_hash: narinfo + .nar_sha256 + .as_ref() + .try_into() + .expect("invalid narhash"), nar_size: narinfo.nar_size, references: narinfo .reference_names diff --git a/tvix/store/src/proto/tests/pathinfo.rs b/tvix/store/src/proto/tests/pathinfo.rs index dca74dc92f..4d0834878d 100644 --- a/tvix/store/src/proto/tests/pathinfo.rs +++ b/tvix/store/src/proto/tests/pathinfo.rs @@ -3,97 +3,82 @@ use crate::tests::fixtures::*; use bytes::Bytes; use data_encoding::BASE64; use nix_compat::nixbase32; -use nix_compat::store_path::{self, StorePath, StorePathRef}; -use std::str::FromStr; -use test_case::test_case; +use nix_compat::store_path::{self, StorePathRef}; +use rstest::rstest; use tvix_castore::proto as castorepb; -#[test_case( - None, - Err(ValidatePathInfoError::NoNodePresent) ; - "No node" -)] -#[test_case( - Some(castorepb::Node { node: None }), - Err(ValidatePathInfoError::NoNodePresent); - "No node 2" -)] -fn validate_no_node( - t_node: Option<castorepb::Node>, - t_result: Result<StorePath, ValidatePathInfoError>, +#[rstest] +#[case::no_node(None, Err(ValidatePathInfoError::NoNodePresent))] +#[case::no_node_2(Some(castorepb::Node { node: None}), Err(ValidatePathInfoError::NoNodePresent))] + +fn validate_pathinfo( + #[case] node: Option<castorepb::Node>, + #[case] exp_result: Result<StorePathRef, ValidatePathInfoError>, ) { // construct the PathInfo object let p = PathInfo { - node: t_node, + node, ..Default::default() }; - assert_eq!(t_result, p.validate()); + + assert_eq!(exp_result, p.validate()); + + let err = p.validate().expect_err("validation should fail"); + assert!(matches!(err, ValidatePathInfoError::NoNodePresent)); } -#[test_case( - castorepb::DirectoryNode { - name: DUMMY_NAME.into(), +#[rstest] +#[case::ok(castorepb::DirectoryNode { + name: DUMMY_PATH.into(), digest: DUMMY_DIGEST.clone().into(), size: 0, - }, - Ok(StorePath::from_str(DUMMY_NAME).expect("must succeed")); - "ok" -)] -#[test_case( - castorepb::DirectoryNode { - name: DUMMY_NAME.into(), +}, Ok(StorePathRef::from_bytes(DUMMY_PATH.as_bytes()).unwrap()))] +#[case::invalid_digest_length(castorepb::DirectoryNode { + name: DUMMY_PATH.into(), digest: Bytes::new(), size: 0, - }, - Err(ValidatePathInfoError::InvalidRootNode(castorepb::ValidateNodeError::InvalidDigestLen(0))); - "invalid digest length" -)] -#[test_case( - castorepb::DirectoryNode { +}, Err(ValidatePathInfoError::InvalidRootNode(castorepb::ValidateNodeError::InvalidDigestLen(0))))] +#[case::invalid_node_name_no_storepath(castorepb::DirectoryNode { name: "invalid".into(), digest: DUMMY_DIGEST.clone().into(), size: 0, - }, - Err(ValidatePathInfoError::InvalidNodeName( +}, Err(ValidatePathInfoError::InvalidNodeName( "invalid".into(), store_path::Error::InvalidLength - )); - "invalid node name" -)] +)))] fn validate_directory( - t_directory_node: castorepb::DirectoryNode, - t_result: Result<StorePath, ValidatePathInfoError>, + #[case] directory_node: castorepb::DirectoryNode, + #[case] exp_result: Result<StorePathRef, ValidatePathInfoError>, ) { // construct the PathInfo object let p = PathInfo { node: Some(castorepb::Node { - node: Some(castorepb::node::Node::Directory(t_directory_node)), + node: Some(castorepb::node::Node::Directory(directory_node)), }), ..Default::default() }; - assert_eq!(t_result, p.validate()); + assert_eq!(exp_result, p.validate()); } -#[test_case( +#[rstest] +#[case::ok( castorepb::FileNode { - name: DUMMY_NAME.into(), + name: DUMMY_PATH.into(), digest: DUMMY_DIGEST.clone().into(), size: 0, executable: false, }, - Ok(StorePath::from_str(DUMMY_NAME).expect("must succeed")); - "ok" + Ok(StorePathRef::from_bytes(DUMMY_PATH.as_bytes()).unwrap()) )] -#[test_case( +#[case::invalid_digest_len( castorepb::FileNode { - name: DUMMY_NAME.into(), + name: DUMMY_PATH.into(), digest: Bytes::new(), ..Default::default() }, - Err(ValidatePathInfoError::InvalidRootNode(castorepb::ValidateNodeError::InvalidDigestLen(0))); - "invalid digest length" + Err(ValidatePathInfoError::InvalidRootNode(castorepb::ValidateNodeError::InvalidDigestLen(0))) )] -#[test_case( +#[case::invalid_node_name( castorepb::FileNode { name: "invalid".into(), digest: DUMMY_DIGEST.clone().into(), @@ -102,32 +87,31 @@ fn validate_directory( Err(ValidatePathInfoError::InvalidNodeName( "invalid".into(), store_path::Error::InvalidLength - )); - "invalid node name" + )) )] fn validate_file( - t_file_node: castorepb::FileNode, - t_result: Result<StorePath, ValidatePathInfoError>, + #[case] file_node: castorepb::FileNode, + #[case] exp_result: Result<StorePathRef, ValidatePathInfoError>, ) { // construct the PathInfo object let p = PathInfo { node: Some(castorepb::Node { - node: Some(castorepb::node::Node::File(t_file_node)), + node: Some(castorepb::node::Node::File(file_node)), }), ..Default::default() }; - assert_eq!(t_result, p.validate()); + assert_eq!(exp_result, p.validate()); } -#[test_case( +#[rstest] +#[case::ok( castorepb::SymlinkNode { - name: DUMMY_NAME.into(), + name: DUMMY_PATH.into(), target: "foo".into(), }, - Ok(StorePath::from_str(DUMMY_NAME).expect("must succeed")); - "ok" + Ok(StorePathRef::from_bytes(DUMMY_PATH.as_bytes()).unwrap()) )] -#[test_case( +#[case::invalid_node_name( castorepb::SymlinkNode { name: "invalid".into(), target: "foo".into(), @@ -135,21 +119,20 @@ fn validate_file( Err(ValidatePathInfoError::InvalidNodeName( "invalid".into(), store_path::Error::InvalidLength - )); - "invalid node name" + )) )] fn validate_symlink( - t_symlink_node: castorepb::SymlinkNode, - t_result: Result<StorePath, ValidatePathInfoError>, + #[case] symlink_node: castorepb::SymlinkNode, + #[case] exp_result: Result<StorePathRef, ValidatePathInfoError>, ) { // construct the PathInfo object let p = PathInfo { node: Some(castorepb::Node { - node: Some(castorepb::node::Node::Symlink(t_symlink_node)), + node: Some(castorepb::node::Node::Symlink(symlink_node)), }), ..Default::default() }; - assert_eq!(t_result, p.validate()); + assert_eq!(exp_result, p.validate()); } /// Ensure parsing a correct PathInfo without narinfo populated succeeds. @@ -236,7 +219,7 @@ fn validate_inconsistent_narinfo_reference_name_digest() { match path_info.validate().expect_err("must fail") { ValidatePathInfoError::InconsistentNarinfoReferenceNameDigest(0, e_expected, e_actual) => { assert_eq!(path_info.references[0][..], e_expected[..]); - assert_eq!(DUMMY_OUTPUT_HASH, e_actual); + assert_eq!(DUMMY_PATH_DIGEST, e_actual); } e => panic!("unexpected error: {:?}", e), } @@ -274,7 +257,7 @@ fn validate_valid_deriver() { let narinfo = path_info.narinfo.as_mut().unwrap(); narinfo.deriver = Some(crate::proto::StorePath { name: "foo".to_string(), - digest: Bytes::from(DUMMY_OUTPUT_HASH.as_slice()), + digest: Bytes::from(DUMMY_PATH_DIGEST.as_slice()), }); path_info.validate().expect("must validate"); @@ -425,7 +408,7 @@ CA: fixed:sha256:086vqwk2wl8zfs47sq2xpjc9k066ilmb8z6dn0q6ymwjzlm196cd"# let path_info: PathInfo = (&narinfo_parsed).into(); let mut narinfo_returned = path_info - .as_narinfo( + .to_narinfo( StorePathRef::from_bytes(b"pa10z4ngm0g83kx9mssrqzz30s84vq7k-hello-2.12.1.tar.gz") .expect("invalid storepath"), ) diff --git a/tvix/store/src/tests/fixtures.rs b/tvix/store/src/tests/fixtures.rs index 500ac0aa5b..1c8359a2c0 100644 --- a/tvix/store/src/tests/fixtures.rs +++ b/tvix/store/src/tests/fixtures.rs @@ -13,8 +13,8 @@ use crate::proto::{ NarInfo, PathInfo, }; -pub const DUMMY_NAME: &str = "00000000000000000000000000000000-dummy"; -pub const DUMMY_OUTPUT_HASH: [u8; 20] = [0; 20]; +pub const DUMMY_PATH: &str = "00000000000000000000000000000000-dummy"; +pub const DUMMY_PATH_DIGEST: [u8; 20] = [0; 20]; lazy_static! { /// The NAR representation of a symlink pointing to `/nix/store/somewhereelse` @@ -106,12 +106,12 @@ lazy_static! { pub static ref PATH_INFO_WITHOUT_NARINFO : PathInfo = PathInfo { node: Some(castorepb::Node { node: Some(castorepb::node::Node::Directory(castorepb::DirectoryNode { - name: DUMMY_NAME.into(), + name: DUMMY_PATH.into(), digest: DUMMY_DIGEST.clone().into(), size: 0, })), }), - references: vec![DUMMY_OUTPUT_HASH.as_slice().into()], + references: vec![DUMMY_PATH_DIGEST.as_slice().into()], narinfo: None, }; @@ -123,7 +123,7 @@ lazy_static! { nar_size: 0, nar_sha256: DUMMY_DIGEST.clone().into(), signatures: vec![], - reference_names: vec![DUMMY_NAME.to_string()], + reference_names: vec![DUMMY_PATH.to_string()], deriver: None, ca: Some(Ca { r#type: ca::Hash::NarSha256.into(), digest: DUMMY_DIGEST.clone().into() }) }), diff --git a/tvix/store/src/utils.rs b/tvix/store/src/utils.rs index 041a9e683d..0b171377bd 100644 --- a/tvix/store/src/utils.rs +++ b/tvix/store/src/utils.rs @@ -1,4 +1,9 @@ use std::sync::Arc; +use std::{ + pin::Pin, + task::{self, Poll}, +}; +use tokio::io::{self, AsyncWrite}; use tvix_castore::{ blobservice::{self, BlobService}, @@ -33,3 +38,28 @@ pub async fn construct_services( Ok((blob_service, directory_service, path_info_service)) } + +/// The inverse of [tokio_util::io::SyncIoBridge]. +/// Don't use this with anything that actually does blocking I/O. +pub struct AsyncIoBridge<T>(pub T); + +impl<W: std::io::Write + Unpin> AsyncWrite for AsyncIoBridge<W> { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut task::Context<'_>, + buf: &[u8], + ) -> Poll<io::Result<usize>> { + Poll::Ready(self.get_mut().0.write(buf)) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut task::Context<'_>) -> Poll<io::Result<()>> { + Poll::Ready(self.get_mut().0.flush()) + } + + fn poll_shutdown( + self: Pin<&mut Self>, + _cx: &mut task::Context<'_>, + ) -> Poll<Result<(), io::Error>> { + Poll::Ready(Ok(())) + } +} diff --git a/tvix/tools/crunch-v2/Cargo.lock b/tvix/tools/crunch-v2/Cargo.lock index cff5509d0b..3748d7e4e9 100644 --- a/tvix/tools/crunch-v2/Cargo.lock +++ b/tvix/tools/crunch-v2/Cargo.lock @@ -752,6 +752,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] +name = "enum-primitive-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba7795da175654fe16979af73f81f26a8ea27638d8d9823d317016888a63dc4c" +dependencies = [ + "num-traits", + "quote", + "syn 2.0.39", +] + +[[package]] name = "enum_dispatch" version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1394,12 +1405,16 @@ dependencies = [ "data-encoding", "ed25519", "ed25519-dalek", + "enum-primitive-derive", "glob", "nom", + "num-traits", + "pin-project-lite", "serde", "serde_json", "sha2 0.10.8", "thiserror", + "tokio", ] [[package]] @@ -1432,9 +1447,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", "libm", @@ -2682,9 +2697,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.34.0" +version = "1.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" dependencies = [ "backtrace", "bytes", diff --git a/tvix/tools/crunch-v2/Cargo.toml b/tvix/tools/crunch-v2/Cargo.toml index 1e3f025250..d2b7126bd2 100644 --- a/tvix/tools/crunch-v2/Cargo.toml +++ b/tvix/tools/crunch-v2/Cargo.toml @@ -14,7 +14,7 @@ bstr = "1.8.0" bytes = "1.5.0" futures = "0.3.29" -tokio = { version = "1.34.0", features = ["full"] } +tokio = { version = "1.37.0", features = ["full"] } rusoto_core = { version = "0.48.0", default-features = false, features = ["hyper-rustls"] } rusoto_s3 = { version = "0.48.0", default-features = false, features = ["rustls"] } diff --git a/tvix/tools/crunch-v2/src/main.rs b/tvix/tools/crunch-v2/src/main.rs index a5d538f6be..5be8c28e29 100644 --- a/tvix/tools/crunch-v2/src/main.rs +++ b/tvix/tools/crunch-v2/src/main.rs @@ -147,7 +147,7 @@ fn ingest(node: nar::Node, name: Vec<u8>, avg_chunk_size: u32) -> Result<proto:: let mut symlinks = vec![]; while let Some(node) = reader.next()? { - match ingest(node.node, node.name, avg_chunk_size)? { + match ingest(node.node, node.name.to_owned(), avg_chunk_size)? { proto::path::Node::Directory(node) => { directories.push(node); } diff --git a/users/Profpatsch/cas-serve/cas-serve.cabal b/users/Profpatsch/cas-serve/cas-serve.cabal index 82db1f5fd8..d14776700a 100644 --- a/users/Profpatsch/cas-serve/cas-serve.cabal +++ b/users/Profpatsch/cas-serve/cas-serve.cabal @@ -65,10 +65,9 @@ executable cas-serve text, sqlite-simple, http-types, - ihp-hsx, wai, warp, mtl, bytestring, memory, - cryptonite, + crypton, diff --git a/users/Profpatsch/cas-serve/default.nix b/users/Profpatsch/cas-serve/default.nix index 1b4fbe03e7..14c3e4aa13 100644 --- a/users/Profpatsch/cas-serve/default.nix +++ b/users/Profpatsch/cas-serve/default.nix @@ -15,7 +15,7 @@ let libraryHaskellDepends = [ pkgs.haskellPackages.pa-prelude pkgs.haskellPackages.pa-label - pkgs.haskellPackages.ihp-hsx + pkgs.haskellPackages.crypton pkgs.haskellPackages.wai pkgs.haskellPackages.warp pkgs.haskellPackages.sqlite-simple diff --git a/users/Profpatsch/ical-smolify/default.nix b/users/Profpatsch/ical-smolify/default.nix index f67d175ce3..bf766db0e9 100644 --- a/users/Profpatsch/ical-smolify/default.nix +++ b/users/Profpatsch/ical-smolify/default.nix @@ -1,7 +1,7 @@ { depot, pkgs, lib, ... }: let - cas-serve = pkgs.writers.writeHaskell "ical-smolify" + ical-smolify = pkgs.writers.writeHaskell "ical-smolify" { libraries = [ pkgs.haskellPackages.iCalendar @@ -13,4 +13,11 @@ let } ./IcalSmolify.hs; in -cas-serve + +ical-smolify.overrideAttrs (old: { + meta = lib.recursiveUpdate old.meta or { } { + # Dependency iCalendar no longer builds in nixpkgs due to a lack of maintenance upstream + # https://github.com/nixos/nixpkgs/commit/13d10cc6e302e7d5800c6a08c1728b14c3801e26 + ci.skip = true; + }; +}) diff --git a/users/Profpatsch/my-prelude/src/MyPrelude.hs b/users/Profpatsch/my-prelude/src/MyPrelude.hs index ffc03c5ecd..880983c47e 100644 --- a/users/Profpatsch/my-prelude/src/MyPrelude.hs +++ b/users/Profpatsch/my-prelude/src/MyPrelude.hs @@ -345,7 +345,8 @@ stringToText = Data.Text.pack -- -- ATTN: Donโt use `String` in code if you can avoid it, prefer `Text` instead. stringToBytesUtf8 :: String -> ByteString -stringToBytesUtf8 = GHC.utf8EncodeString +-- TODO(Profpatsch): use a stable interface +stringToBytesUtf8 = GHC.utf8EncodeByteString -- | Like `show`, but generate a 'Text' -- diff --git a/users/Profpatsch/my-prelude/src/Postgres/MonadPostgres.hs b/users/Profpatsch/my-prelude/src/Postgres/MonadPostgres.hs index 2585f6637c..f83a6d7fcf 100644 --- a/users/Profpatsch/my-prelude/src/Postgres/MonadPostgres.hs +++ b/users/Profpatsch/my-prelude/src/Postgres/MonadPostgres.hs @@ -8,7 +8,6 @@ module Postgres.MonadPostgres where import AtLeast (AtLeast) import Control.Exception import Control.Foldl qualified as Fold -import Control.Monad.Except import Control.Monad.Logger.CallStack (MonadLogger, logDebug, logWarn) import Control.Monad.Reader (MonadReader (ask), ReaderT (..)) import Control.Monad.Trans.Resource diff --git a/users/Profpatsch/whatcd-resolver/default.nix b/users/Profpatsch/whatcd-resolver/default.nix index 6d5d0834b1..27468507ac 100644 --- a/users/Profpatsch/whatcd-resolver/default.nix +++ b/users/Profpatsch/whatcd-resolver/default.nix @@ -31,7 +31,6 @@ let pkgs.haskellPackages.pa-run-command pkgs.haskellPackages.aeson-better-errors pkgs.haskellPackages.blaze-html - pkgs.haskellPackages.dlist pkgs.haskellPackages.hs-opentelemetry-sdk pkgs.haskellPackages.http-conduit pkgs.haskellPackages.http-types @@ -39,7 +38,6 @@ let pkgs.haskellPackages.monad-logger pkgs.haskellPackages.resource-pool pkgs.haskellPackages.postgresql-simple - pkgs.haskellPackages.selective pkgs.haskellPackages.tmp-postgres pkgs.haskellPackages.unliftio pkgs.haskellPackages.wai-extra diff --git a/users/Profpatsch/whatcd-resolver/whatcd-resolver.cabal b/users/Profpatsch/whatcd-resolver/whatcd-resolver.cabal index cad1fabe99..a9bd04827b 100644 --- a/users/Profpatsch/whatcd-resolver/whatcd-resolver.cabal +++ b/users/Profpatsch/whatcd-resolver/whatcd-resolver.cabal @@ -86,13 +86,11 @@ library aeson-better-errors, aeson, blaze-html, - blaze-markup, bytestring, case-insensitive, containers, unordered-containers, directory, - dlist, exceptions, filepath, hs-opentelemetry-sdk, @@ -107,8 +105,6 @@ library resource-pool, postgresql-simple, punycode, - scientific, - selective, tmp-postgres, unliftio, wai-extra, diff --git a/users/aspen/emacs/config.org b/users/aspen/emacs/config.org index 9eaf1c4561..b3762affe4 100644 --- a/users/aspen/emacs/config.org +++ b/users/aspen/emacs/config.org @@ -1,4 +1,5 @@ # Local variables: +# lexical-binding: t # eval: (paxedit-mode 1) # eval: (display-line-numbers-mode 1) # eval: (flyspell-mode -1) @@ -6,6 +7,12 @@ # End: #+title: Emacs Config +#+PROPERTY: header-args:emacs-lisp :results silent +#+PROPERTY: header-args:elisp :results silent + +#+begin_src emacs-lisp :tangle yes +;; -*- lexical-binding: t; -*- +#+end_src * Utils #+begin_src elisp :tangle yes @@ -37,16 +44,10 @@ `(and symbol-start (group (or ,@words)) symbol-end))) #+end_src -#+RESULTS: -: rx-words - #+begin_src elisp :tangle no :results example (average (list 1 2 3 4)) #+end_src -#+RESULTS: -: 2 - ** Text editing utils *** Reading strings #+begin_src elisp :tangle yes @@ -107,8 +108,6 @@ Returns nil if the regex did not match, non-nil otherwise" (match-beginning 1)) #+end_src -#+RESULTS: -: 3 *** Changing file contents #+begin_src elisp :tangle yes (defmacro saving-excursion (&rest body) @@ -139,18 +138,12 @@ Returns nil if the regex did not match, non-nil otherwise" ,@(when ignore-case (list ignore-case))))) #+end_src -#+RESULTS: -: s-contains--pcase-macroexpander - #+begin_src elisp :tangle no (pcase "foo" ((s-contains "bar") 1) ((s-contains "o") 2)) #+end_src -#+RESULTS: -: 2 - ** Evil utils #+begin_src elisp :tangle yes (defmacro define-move-and-insert @@ -174,24 +167,18 @@ Returns nil if the regex did not match, non-nil otherwise" (evil-insert count vcount skip-empty-lines)))) #+end_src -#+RESULTS: -: define-move-and-insert - * Name and email #+begin_src emacs-lisp (setq user-full-name "Aspen Smith" user-mail-address "root@gws.fyi") #+end_src -#+RESULTS: -: root@gws.fyi - * Visual style #+begin_src elisp :tangle yes (let ((font-family (pcase system-type ('darwin "MesloLGSDZ NF") ('gnu/linux "Meslo LGSDZ Nerd Font")))) - (setq doom-font (font-spec :family font-family :size 14) + (setq doom-font (font-spec :family font-family :height 113) doom-big-font (font-spec :family font-family :size 24) doom-big-font-increment 5 doom-variable-pitch-font (font-spec :family font-family) @@ -205,18 +192,12 @@ Returns nil if the regex did not match, non-nil otherwise" doom-modeline-height 12) #+end_src -#+RESULTS: -: 12 - #+begin_src elisp :tangle yes (setq whitespace-style '(face lines-tail)) (global-whitespace-mode t) (add-hook 'org-mode-hook (lambda () (whitespace-mode -1)) t) #+end_src -#+RESULTS: -| er/add-org-mode-expansions | +aspen/org-setup | +lookup--init-org-mode-handlers-h | (closure (t) (&rest _) (add-hook 'before-save-hook 'org-encrypt-entries nil t)) | #[0 \300\301\302\303\304$\207 [add-hook change-major-mode-hook org-fold-show-all append local] 5] | #[0 \301\211\207 [imenu-create-index-function org-imenu-get-tree] 2] | doom-disable-show-paren-mode-h | doom-disable-show-trailing-whitespace-h | +org-make-last-point-visible-h | org-appear-mode | org-fancy-priorities-mode | org-superstar-mode | evil-org-mode | toc-org-enable | #[0 \300\301\302\303\304$\207 [add-hook change-major-mode-hook org-babel-show-result-all append local] 5] | org-babel-result-hide-spec | org-babel-hide-all-hashes | flyspell-mode | embrace-org-mode-hook | org-eldoc-load | +literate-enable-recompile-h | (lambda nil (whitespace-mode -1)) | - ** Theme [[https://davidjohnstone.net/lch-lab-colour-gradient-picker][LAB colour gradient picker]] is a good tool for trying to find "halfway points" between two colours @@ -225,9 +206,6 @@ Returns nil if the regex did not match, non-nil otherwise" (rainbow-mode) #+end_src -#+RESULTS: -: t - #+name: solarized-vars #+begin_src elisp :tangle yes (setq +solarized-s-base03 "#002b36" @@ -276,9 +254,6 @@ Returns nil if the regex did not match, non-nil otherwise" +solarized-green-l "#B4C342") #+end_src -#+RESULTS: solarized-vars -: #B4C342 - *** Overrides #+name: overrides-for-solarized-light @@ -331,15 +306,19 @@ Returns nil if the regex did not match, non-nil otherwise" ) #+end_src -#+RESULTS: overrides-for-solarized-light -| doom--customize-themes-h-91 | doom--customize-themes-h-92 | doom--customize-themes-h-93 | doom--customize-themes-h-94 | doom--customize-themes-h-95 | doom--customize-themes-h-96 | doom--customize-themes-h-97 | doom--customize-themes-h-98 | - * Keybindings and navigation Get the hell out of here, snipe! #+begin_src elisp :tangle yes (remove-hook 'doom-first-input-hook #'evil-snipe-mode) #+end_src +#+begin_src emacs-lisp :tangle yes +(map! + (:leader + "b" #'consult-buffer + "r" #'consult-recent-file)) +#+end_src + ** Flycheck #+begin_src elisp :tangle yes (evil-set-command-property 'flycheck-next-error :repeat nil) @@ -352,8 +331,6 @@ Get the hell out of here, snipe! :m "[e" #'flycheck-previous-error)) #+end_src -#+RESULTS: - ** Smerge #+begin_src elisp :tangle yes (evil-set-command-property 'smerge-next :repeat nil) @@ -371,9 +348,6 @@ Get the hell out of here, snipe! t #+end_src -#+RESULTS: -: t - ** Vinegar-style dired #+begin_src elisp :tangle yes (defun dired-mode-p () (eq 'dired-mode major-mode)) @@ -393,8 +367,6 @@ t "-" #'aspen/dired-minus)) #+end_src -#+RESULTS: - ** Lisp mappings *** Use paxedit #+begin_src elisp :tangle yes @@ -404,9 +376,6 @@ t (common-lisp-mode . paxedit-mode))) #+end_src -#+RESULTS: -| paxedit-mode | - *** Paxedit functions #+begin_src elisp :tangle yes @@ -439,9 +408,6 @@ t (message paxedit-message-kill))) #+end_src -#+RESULTS: -: aspen/paxedit-kill - *** Paxedit mappings #+begin_src elisp :tangle yes (map! @@ -481,8 +447,6 @@ t )) #+end_src -#+RESULTS: - *** Eval functions #+begin_src elisp :tangle yes (use-package! predd) @@ -583,8 +547,6 @@ If invoked with a prefix ARG eval the expression after inserting it" ;;; #+end_src -#+RESULTS: -: fireplace-eval-context *** Eval bindings fireplace-esque eval binding @@ -653,8 +615,6 @@ fireplace-esque eval binding #+end_src -#+RESULTS: - ** Coerce #+begin_src elisp :tangle yes @@ -674,18 +634,14 @@ fireplace-esque eval binding ))) #+end_src -#+RESULTS: -: t - * Mode-specific config ** org-mode #+begin_src elisp :tangle yes (after! org - (load! "org-config")) + (load! "org-config") + (load! "org-query")) #+end_src -#+RESULTS: -: t *** Theme overrides #+begin_src elisp :tangle yes @@ -718,9 +674,6 @@ fireplace-esque eval binding `(org-level-8 :foreground ,+solarized-blue)) #+end_src -#+RESULTS: -| doom--customize-themes-h-91 | doom--customize-themes-h-92 | doom--customize-themes-h-93 | doom--customize-themes-h-94 | doom--customize-themes-h-95 | doom--customize-themes-h-96 | doom--customize-themes-h-97 | doom--customize-themes-h-98 | doom--customize-themes-h-99 | - *** Commands #+begin_src elisp :tangle yes (defun grfn/insert-new-src-block () @@ -763,11 +716,14 @@ fireplace-esque eval binding (map! (:after org :n "C-c C-x C-o" #'org-clock-out + (:leader + "n k" #'org-archive-subtree-default) (:map org-capture-mode-map :n "g RET" #'org-capture-finalize :n "g \\" #'org-captue-refile))) #+end_src + ** magit #+begin_src elisp :tangle yes (after! magit @@ -803,8 +759,6 @@ fireplace-esque eval binding (list "o" "Reset HEAD@{1}" #'magit-reset-head-previous))) #+end_src -#+RESULTS: - ** elisp *** Org config mode The minor-mode for *this file*! @@ -822,9 +776,6 @@ The minor-mode for *this file*! (sp-update-local-pairs 'org-config-mode)) #+end_src -#+RESULTS: -| keymap | - *** Bindings #+begin_src elisp :tangle yes (map! @@ -833,8 +784,6 @@ The minor-mode for *this file*! :n "g RET" (ฮป! () (ert t)) )) #+end_src -#+RESULTS: - ** tuareg *** Config @@ -869,15 +818,11 @@ The minor-mode for *this file*! nil #+end_src -#+RESULTS: - #+begin_src elisp :tangle yes (after! dune-mode (add-hook 'dune-mode-hook 'paxedit-mode)) #+end_src -#+RESULTS: - *** Bindings #+begin_src elisp :tangle yes (map! @@ -889,8 +834,6 @@ nil "C-c C-f" (ฮป! () (compile "dune fmt")))) #+end_src -#+RESULTS: - *** Theme overrides #+begin_src elisp :tangle yes (custom-set-faces! @@ -908,9 +851,6 @@ nil `(vertico-group-header :foreground ,+solarized-s-base1)) #+end_src -#+RESULTS: -| doom--customize-themes-h-30 | doom--customize-themes-h-31 | doom--customize-themes-h-32 | doom--customize-themes-h-33 | doom--customize-themes-h-34 | doom--customize-themes-h-35 | doom--customize-themes-h-52 | - ** clojure *** Setup @@ -970,8 +910,6 @@ nil nil #+end_src -#+RESULTS: - *** Commands #+begin_src elisp :tangle yes @@ -1011,9 +949,6 @@ nil #+end_src -#+RESULTS: -: cider-copy-last-result - *** Bindings @@ -1050,26 +985,17 @@ nil :n "g \\" 'cider-switch-to-last-clojure-buffer))) #+end_src - #+RESULTS: - ** rust #+begin_src elisp :tangle yes (defun aspen/rust-setup () (interactive) (+evil-embrace-angle-bracket-modes-hook-h) (setq-local whitespace-line-column 100 - fill-column 100) - (setq lsp-rust-analyzer-cargo-watch-command "clippy" - lsp-rust-analyzer-cargo-watch-args ["--target-dir" "/home/grfn/code/readyset/readyset/target/rust-analyzer"] - rustic-format-trigger 'on-save - lsp-ui-doc-enable t)) + fill-column 100)) (add-hook 'rust-mode-hook #'aspen/rust-setup) #+end_src -#+RESULTS: -| aspen/rust-setup | grfn/rust-setup | doom-modeline-env-setup-rust | - *** Bindings #+begin_src elisp :tangle yes @@ -1083,26 +1009,90 @@ nil "m" #'lsp-rust-analyzer-expand-macro))) #+end_src -#+RESULTS: - *** Theme overrides #+begin_src elisp :tangle yes (custom-set-faces! `(rust-unsafe :foreground ,+solarized-red)) #+end_src -#+RESULTS: -| doom--customize-themes-h-30 | doom--customize-themes-h-31 | doom--customize-themes-h-32 | doom--customize-themes-h-33 | doom--customize-themes-h-54 | +** common-lisp +*** Commands +#+begin_src emacs-lisp :tangle yes +(defun aspen/sly-panettone () + (interactive) + (sly + (concat + (s-trim + (shell-command-to-string + "nix-build -o sbcl -E 'with import ~/code/depot {}; nix.buildLisp.sbclWith [web.panettone]'")) + "/bin/sbcl"))) + +(defun aspen/setup-lisp () + (interactive) + (rainbow-delimiters-mode) + (paxedit-mode 1) + (flycheck-mode -1)) + +(add-hook 'common-lisp-mode-hook #'aspen/setup-lisp) + +(defun sly-run-tests () + (interactive) + ;; TODO: handle other test frameworks + (let ((orig-window (get-buffer-window))) + (sly-eval '(fiveam:run!)) + (funcall-interactively #'sly-mrepl-sync) + (select-window orig-window))) +#+end_src + +*** Bindings -* Fuzzy search #+begin_src emacs-lisp :tangle yes -(use-package! hotfuzz +(map! + (:map sly-mode-map + :n "g \\" #'sly-mrepl-sync + :n "g d" #'sly-edit-definition + :n "K" #'sly-documentation + :n "g SPC" #'sly-compile-and-load-file + :n "g RET" #'sly-run-tests) + + (:map sly-mrepl-mode-map + "C-k" #'sly-mrepl-previous-prompt + "C-r" #'isearch-backward)) +#+end_src + +* Completion +** Corfu +#+begin_src emacs-lisp :tangle yes +(setopt +corfu-want-ret-to-confirm nil) + +(use-package! corfu + :demand t + :bind (:map corfu-map + ("TAB" . corfu-next) + ([tab] . corfu-next) + ("S-TAB" . corfu-previous) + ([backtab] . corfu-previous)) + :init (setopt corfu-on-exact-match 'insert + corfu-preselect 'prompt + completion-cycle-threshold 1 + corfu-quit-no-match t + corfu-quit-at-boundary t) :config - (setopt completion-styles '(hotfuzz))) + (map! :map corfu-map + :i "TAB" #'corfu-next + :i [tab] #'corfu-next + :i "S-TAB" #'corfu-previous + :i [backtab] #'corfu-previous)) #+end_src -#+RESULTS: -: t +** Fuzzy search +#+begin_src emacs-lisp :tangle yes +(use-package! hotfuzz + :after (orderless corfu) + :config + (setopt completion-styles '(hotfuzz basic) + completion-ignore-case t)) +#+end_src * Email #+begin_src elisp :tangle yes @@ -1121,10 +1111,38 @@ nil message-sendmail-f-is-evil 't message-sendmail-envelope-from 'header message-sendmail-extra-arguments '("--read-envelope-from"))) -#+end_src -#+RESULTS: -| --read-envelope-from | +(defun aspen/notmuch-sync () + (interactive) + (let* ((search-buffer (current-buffer)) + (proc (start-process-shell-command + "notmuch-sync" + "*notmuch-sync*" + "cd ~/mail/personal/ && gmi sync")) + (buf (process-buffer proc))) + + (set-process-sentinel + proc + (lambda (proc msg) + (internal-default-process-sentinel proc msg) + (when (and (string= msg "finished\n")) + (kill-buffer buf) + (with-current-buffer search-buffer + (when (eq major-mode 'notmuch-search-mode) + (notmuch-refresh-this-buffer)))))) + + (with-current-buffer buf + (+popup-buffer-mode)) + (display-buffer buf '(display-buffer-at-bottom . ())))) + +(set-popup-rule! + "^\\*notmuch-sync\\*$" + :select nil + :quit 'other) + +(map! :map notmuch-search-mode-map + :n "g SPC" #'aspen/notmuch-sync) +#+end_src ** Bindings #+begin_src emacs-lisp :tangle yes @@ -1134,10 +1152,6 @@ nil :desc "Search email" "s M" #'consult-notmuch)) #+end_src -#+RESULTS: -: notmuch-jump-search - - ** Theme #+begin_src emacs-lisp :tangle yes @@ -1146,17 +1160,12 @@ nil :background ,+solarized-halfway-highlight)) #+end_src -#+RESULTS: -| doom--customize-themes-h-91 | doom--customize-themes-h-92 | doom--customize-themes-h-93 | doom--customize-themes-h-94 | doom--customize-themes-h-95 | doom--customize-themes-h-96 | doom--customize-themes-h-97 | doom--customize-themes-h-98 | doom--customize-themes-h-99 | doom--customize-themes-h-100 | - * Misc -** Make underscores word chars -#+begin_src elisp :tangle yes -(modify-syntax-entry ?_ "w") +** TVL +#+begin_src emacs-lisp :tangle yes +(require 'tvl) #+end_src -#+RESULTS: - ** Matchit #+begin_src elisp :tangle yes (use-package! evil-matchit) @@ -1198,17 +1207,11 @@ nil (call-interactively #'irc-connect))) #+end_src -#+RESULTS: -: aspen/switch-to-erc-buffer-or-connect - #+begin_src elisp :tangle yes (map! :leader "o I" #'irc-connect :leader "o i" #'aspen/switch-to-erc-buffer-or-connect) #+end_src -#+RESULTS: -: aspen/switch-to-erc-buffer-or-connect - *** IRC alerts #+begin_src elisp :tangle yes (use-package! alert) @@ -1312,9 +1315,6 @@ If the buffer is currently not visible, makes it sticky." (add-hook 'erc-connect-pre-hook 'my-erc-define-alerts) #+end_src -#+RESULTS: -| my-erc-define-alerts | - *** Don't send ~:q~, etc, to the server #+begin_src elisp :tangle yes (defun fix-irc-message (msg) @@ -1323,17 +1323,12 @@ If the buffer is currently not visible, makes it sticky." (advice-add #'erc-user-input :filter-return #'fix-irc-message) #+end_src -#+RESULTS: - *** Theme overrides #+begin_src elisp :tangle yes (custom-set-faces! `(erc-button :foreground ,+solarized-blue)) #+end_src -#+RESULTS: -| doom--customize-themes-h-30 | doom--customize-themes-h-31 | doom--customize-themes-h-32 | doom--customize-themes-h-43 | doom--customize-themes-h-47 | doom--customize-themes-h-48 | doom--customize-themes-h-49 | doom--customize-themes-h-50 | doom--customize-themes-h-51 | doom--customize-themes-h-52 | doom--customize-themes-h-53 | doom--customize-themes-h-54 | doom--customize-themes-h-56 | doom--customize-themes-h-57 | doom--customize-themes-h-58 | doom--customize-themes-h-59 | doom--customize-themes-h-60 | doom--customize-themes-h-61 | doom--customize-themes-h-62 | doom--customize-themes-h-63 | doom--customize-themes-h-64 | - *** TODO Nick rainbow colors Stole this from https://github.com/jtdaugherty/emacs-config/blob/master/common/erc-nick-colors.el. @@ -1391,14 +1386,8 @@ you'll change the colors used for nicks." (add-hook 'erc-insert-modify-hook 'erc-insert-nick-colors) #+end_src -#+RESULTS: -| erc-insert-nick-colors | erc-controls-highlight | erc-fill | my-erc-hook | erc-button-add-buttons | erc-match-message | erc-add-timestamp | - * Hacks Not having this breaks elisp documentation :( #+begin_src elisp :tangle yes (defvar elisp-demos-user-files nil) #+end_src - -#+RESULTS: -: elisp-demos-user-files diff --git a/users/aspen/emacs/init.el b/users/aspen/emacs/init.el index 2fe2316753..7674d088b5 100644 --- a/users/aspen/emacs/init.el +++ b/users/aspen/emacs/init.el @@ -21,7 +21,8 @@ ;;layout ; auie,ctsrnm is the superior home row :completion - company ; the ultimate code completion backend + ;; company ; the ultimate code completion backend + corfu ;;helm ; the *other* search engine for love and life ;;ido ; the other *other* search engine... ;;ivy ; a search engine for love and life @@ -30,7 +31,7 @@ :ui ;;deft ; notational velocity for Emacs doom ; what makes DOOM look the way it does - ;; doom-dashboard ; a nifty splash screen for Emacs + doom-dashboard ; a nifty splash screen for Emacs ;;doom-quit ; DOOM quit-message prompts when you quit Emacs (emoji +unicode) ; ๐ hl-todo ; highlight TODO/FIXME/NOTE/DEPRECATED/HACK/REVIEW @@ -95,7 +96,7 @@ (eval +overlay) ; run code, run (also, repls) ;;gist ; interacting with github gists lookup ; navigate your code and its documentation - (lsp +eglot) ; M-x vscode + lsp ; M-x vscode magit ; a git porcelain for Emacs ;;make ; run make tasks from Emacs pass ; password manager for nerds @@ -142,7 +143,7 @@ ;;idris ; a language you can depend on json ; At least it ain't XML ;;(java +lsp) ; the poster child for carpal tunnel syndrome - ;;javascript ; all(hope(abandon(ye(who(enter(here)))))) + (javascript +lsp) ; all(hope(abandon(ye(who(enter(here)))))) ;;julia ; a better, faster MATLAB ;;kotlin ; a better, slicker Java(Script) ;;latex ; writing papers in Emacs has never been so fun @@ -156,7 +157,8 @@ (org ; organize your plain life in plain text +gnuplot +present - +pretty) + +pretty + ) ;;php ; perl's insecure younger brother ;;plantuml ; diagrams for confusing people more ;;purescript ; javascript, but functional diff --git a/users/aspen/emacs/org-config.el b/users/aspen/emacs/org-config.el index df93142b0f..89cf7486fb 100644 --- a/users/aspen/emacs/org-config.el +++ b/users/aspen/emacs/org-config.el @@ -128,7 +128,7 @@ ("nw" "Next @Work" tags-todo "@work&next") ("nt" "Next tooling" tags-todo "tooling") - ("p" . "Project...") + ;; ("p" . "Project...") ;; ,@(aspen/org-projects->agenda-commands (aspen/org-projects)) ) diff --git a/users/aspen/emacs/packages.el b/users/aspen/emacs/packages.el index b72fb45a9f..0bcc345d88 100644 --- a/users/aspen/emacs/packages.el +++ b/users/aspen/emacs/packages.el @@ -5,6 +5,7 @@ (package! paxedit) (package! predd :recipe (:host github :repo "skeeto/predd")) +(package! direnv) (package! alert) (package! flycheck-clojure) (package! evil-matchit) diff --git a/users/aspen/pkgs/py3status.nix b/users/aspen/pkgs/py3status.nix new file mode 100644 index 0000000000..89f52d9674 --- /dev/null +++ b/users/aspen/pkgs/py3status.nix @@ -0,0 +1,12 @@ +{ pkgs, ... }: + +pkgs.python3Packages.py3status.overridePythonAttrs (old: rec { + name = "${pname}-${old.version}"; + pname = "py3status-glittershark"; + src = pkgs.fetchFromGitHub { + owner = "glittershark"; + repo = "py3status"; + rev = "f243be1458cdabd5a7524adb76b5db99006c810c"; + sha256 = "0ffmv91562yk0wigriw4d5nfg2b32wqx8x78qvdqkawzvgbwrwvl"; + }; +}) diff --git a/users/aspen/system/home/default.nix b/users/aspen/system/home/default.nix index f821999b99..90df02b378 100644 --- a/users/aspen/system/home/default.nix +++ b/users/aspen/system/home/default.nix @@ -30,7 +30,13 @@ rec { yerenHome = yeren.activation-script; + lusca = home ./machines/lusca.nix; + + luscaHome = lusca.activation-script; + meta.ci.targets = [ + "ogopogoHome" + "luscaHome" "yerenHome" ]; } diff --git a/users/aspen/system/home/machines/lusca.nix b/users/aspen/system/home/machines/lusca.nix new file mode 100644 index 0000000000..fc5f606639 --- /dev/null +++ b/users/aspen/system/home/machines/lusca.nix @@ -0,0 +1,34 @@ +{ pkgs, lib, config, ... }: + +let + inherit (builtins) pathExists; +in +{ + imports = [ + ../platforms/linux.nix + ../modules/common.nix + + ../modules/email.nix + ../modules/desktop.nix + ] ++ (lib.optional (pathExists ../modules/private.nix) + ../modules/private.nix); + + home.username = lib.mkForce "aspen"; + home.homeDirectory = lib.mkForce "/home/aspen"; + + # for when hacking + programs.home-manager.enable = true; + home.stateVersion = "20.03"; + + system.machine = { + wirelessInterface = "wlp1s0"; + i3FontSize = 9; + battery = 1; + }; + + programs.alacritty.settings.font.size = lib.mkForce 5.5; + + home.packages = with pkgs; [ discord steam tdesktop slack ]; + + xsession.windowManager.i3.config.keybindings.XF86AudioMedia = "exec lock"; +} diff --git a/users/aspen/system/home/machines/ogopogo.nix b/users/aspen/system/home/machines/ogopogo.nix index 22ef65bb73..37396a5aa1 100644 --- a/users/aspen/system/home/machines/ogopogo.nix +++ b/users/aspen/system/home/machines/ogopogo.nix @@ -10,6 +10,8 @@ in ../platforms/linux.nix ../modules/common.nix ../modules/desktop.nix + ../modules/games.nix + ../modules/obs.nix ../modules/development/agda.nix ../modules/development/readyset.nix ../modules/development/ocaml.nix @@ -21,7 +23,7 @@ in system.machine = { wirelessInterface = "wlp4s0"; i3FontSize = 9; - battery = false; + battery = null; }; home.packages = with pkgs; [ diff --git a/users/aspen/system/home/modules/common.nix b/users/aspen/system/home/modules/common.nix index b82827798e..b51ae1c7db 100644 --- a/users/aspen/system/home/modules/common.nix +++ b/users/aspen/system/home/modules/common.nix @@ -14,12 +14,12 @@ ../modules/lib/cloneRepo.nix ]; - home.username = "grfn"; - home.homeDirectory = "/home/grfn"; + home.username = "aspen"; + home.homeDirectory = "/home/aspen"; programs.password-store.enable = true; - grfn.impure.clonedRepos.passwordStore = { + aspen.impure.clonedRepos.passwordStore = { github = "glittershark/pass"; path = ".local/share/password-store"; }; @@ -72,39 +72,13 @@ (writeShellScriptBin "rebuild-home" '' set -eo pipefail cd ~/code/depot - nix build -f . users.aspen.system.home.$(hostname)Home -o /tmp/home - /tmp/home/activate + home=$(nix-build -A users.aspen.system.home.$(hostname)Home -o /tmp/home) + nix-env -p /nix/var/nix/per-user/aspen/home --set $home + $home/activate '') ]; - programs.ssh = { - enable = true; - - matchBlocks = { - "home" = { - host = "home.gws.fyi"; - forwardAgent = true; - }; - - "cerberus" = { - host = "cerberus"; - hostname = "172.16.0.3"; - forwardAgent = true; - user = "griffin"; - }; - - "mugwump" = { - host = "mugwump"; - hostname = "172.16.0.5"; - forwardAgent = true; - }; - - "roswell" = { - host = "roswell"; - forwardAgent = true; - }; - }; - }; + programs.ssh = { enable = true; }; programs.direnv = { enable = true; diff --git a/users/aspen/system/home/modules/desktop.nix b/users/aspen/system/home/modules/desktop.nix index cab3b658e0..e1841cd3c3 100644 --- a/users/aspen/system/home/modules/desktop.nix +++ b/users/aspen/system/home/modules/desktop.nix @@ -5,15 +5,10 @@ { imports = [ ./i3.nix - ./obs.nix - ./games.nix + ./alacritty.nix ]; home.packages = with pkgs; [ - (ntfy.override { - # Slack support is broken as of 2023-06-15 - withSlack = false; - }) (writeShellApplication { name = "edit-input"; @@ -38,8 +33,8 @@ gtk = { enable = true; gtk3.bookmarks = [ - "file:///home/grfn/code" - "file:///home/grfn/notes" + "file:///home/aspen/code" + "file:///home/aspen/notes" ]; }; } diff --git a/users/aspen/system/home/modules/development.nix b/users/aspen/system/home/modules/development.nix index 26817af497..ca6ef131a3 100644 --- a/users/aspen/system/home/modules/development.nix +++ b/users/aspen/system/home/modules/development.nix @@ -32,8 +32,7 @@ with lib; { imports = [ ./lib/zshFunctions.nix - ./development/kube.nix - # TODO(grfn): agda build is broken in the nixpkgs checkout + # TODO(aspen): agda build is broken in the nixpkgs checkout # ./development/agda.nix ./development/rust.nix ]; @@ -53,6 +52,7 @@ with lib; jsonnet ngrok amber + ocamlPackages.patdiff gdb lldb @@ -68,7 +68,7 @@ with lib; nodePackages.prettier ] ++ optionals (stdenv.isLinux) [ - # TODO(grfn): replace with stable again once the current julia debacle + # TODO(aspen): replace with stable again once the current julia debacle # is resolved upstream, see https://github.com/NixOS/nixpkgs/pull/121114 julia_16-bin valgrind diff --git a/users/aspen/system/home/modules/development/agda.nix b/users/aspen/system/home/modules/development/agda.nix index afd22a306d..55381994c4 100644 --- a/users/aspen/system/home/modules/development/agda.nix +++ b/users/aspen/system/home/modules/development/agda.nix @@ -29,7 +29,7 @@ in ])) ]; - grfn.impure.clonedRepos = { + aspen.impure.clonedRepos = { agda-stdlib = { github = "agda/agda-stdlib"; path = "code/agda-stdlib"; @@ -51,8 +51,8 @@ in ''; home.file.".agda/libraries".text = '' - /home/grfn/code/agda-stdlib/standard-library.agda-lib - /home/grfn/code/agda-categories/agda-categories.agda-lib + /home/aspen/code/agda-stdlib/standard-library.agda-lib + /home/aspen/code/agda-categories/agda-categories.agda-lib ''; } diff --git a/users/aspen/system/home/modules/emacs.nix b/users/aspen/system/home/modules/emacs.nix index 3db4e293f8..6936da4b9d 100644 --- a/users/aspen/system/home/modules/emacs.nix +++ b/users/aspen/system/home/modules/emacs.nix @@ -76,7 +76,7 @@ in ); }; - grfn.impure.clonedRepos = { + aspen.impure.clonedRepos = { orgClubhouse = { github = "glittershark/org-clubhouse"; path = "code/org-clubhouse"; diff --git a/users/aspen/system/home/modules/email.nix b/users/aspen/system/home/modules/email.nix index 5cf5a67eaa..cb92c40cee 100644 --- a/users/aspen/system/home/modules/email.nix +++ b/users/aspen/system/home/modules/email.nix @@ -16,7 +16,7 @@ let personal = { primary = true; address = "root@gws.fyi"; - aliases = [ "grfn@gws.fyi" "aspen@gws.fyi" ]; + aliases = [ "aspen@gws.fyi" "aspen@gws.fyi" ]; passEntry = "root-gws-msmtp"; }; }; diff --git a/users/aspen/system/home/modules/games.nix b/users/aspen/system/home/modules/games.nix index 5e28003892..dc6331d648 100644 --- a/users/aspen/system/home/modules/games.nix +++ b/users/aspen/system/home/modules/games.nix @@ -15,14 +15,14 @@ let }); init = runCommand "init.txt" { } '' - substitute "${df-orig}/data/init/init.txt" $out \ + substitute "${df-orig}/data/init/init_default.txt" $out \ --replace "[INTRO:YES]" "[INTRO:NO]" \ --replace "[VOLUME:255]" "[VOLUME:0]" \ --replace "[FPS:NO]" "[FPS:YES]" ''; d_init = runCommand "d_init.txt" { } '' - substitute "${df-orig}/data/init/d_init.txt" $out \ + substitute "${df-orig}/data/init/d_init_default.txt" $out \ --replace "[AUTOSAVE:NONE]" "[AUTOSAVE:SEASONAL]" \ --replace "[AUTOSAVE_PAUSE:NO]" "[AUTOSAVE_PAUSE:YES]" \ --replace "[INITIAL_SAVE:NO]" "[INITIAL_SAVE:YES]" \ @@ -46,16 +46,6 @@ let in mkMerge [ - { - home.packages = [ - crawl - xonotic - ]; - } - (mkIf stdenv.isLinux { - home.packages = [ - df - prismlauncher - ]; - }) + { home.packages = [ crawl ]; } + (mkIf stdenv.isLinux { home.packages = [ df prismlauncher ]; }) ] diff --git a/users/aspen/system/home/modules/i3.nix b/users/aspen/system/home/modules/i3.nix index 1ce842af20..f91527da44 100644 --- a/users/aspen/system/home/modules/i3.nix +++ b/users/aspen/system/home/modules/i3.nix @@ -1,5 +1,7 @@ { config, lib, pkgs, ... }: let + inherit (config.lib) depot; + mod = "Mod4"; solarized = import ../common/solarized.nix; # TODO pull this out into lib @@ -7,16 +9,112 @@ let msg=$(emacsclient --eval '${eval}' 2>&1) echo "''${msg:1:-1}" ''; - screenlayout = { - home = pkgs.writeShellScript "screenlayout_home.sh" '' - xrandr \ - --output eDP-1 --mode 1920x1200 --pos 0x960 --rotate normal \ - --output DP-3 --primary --mode 3840x2160 --pos 1920x0 --rotate normal \ - --output DP-1 --off \ - --output DP-2 --off \ - --output DP-4 --off - ''; - }; + + i3status-conf = pkgs.writeText "i3status.conf" '' + general { + output_format = i3bar + colors = true + color_good = "#859900" + + interval = 1 + } + + order += "external_script current_task" + order += "external_script inbox" + order += "spotify" + order += "weather_owm" + order += "volume_status" + order += "wireless ${config.system.machine.wirelessInterface}" + # order += "ethernet enp3s0f0" + order += "cpu_usage" + ${lib.optionalString (!isNull config.system.machine.battery) '' + order += "battery ${toString config.system.machine.battery}" + ''} + # order += "volume master" + order += "time" + order += "tztime utc" + + mpd { + format = "%artist - %album - %title" + } + + wireless ${config.system.machine.wirelessInterface} { + format_up = "W: (%quality - %essid - %bitrate) %ip" + format_down = "W: -" + } + + ethernet enp3s0f0 { + format_up = "E: %ip" + format_down = "E: -" + } + + battery ${toString config.system.machine.battery} { + format = "%status %percentage (%remaining)" + path = "/sys/class/power_supply/BAT%d/uevent" + low_threshold = 10 + } + + cpu_usage { + format = "CPU: %usage" + } + + load { + format = "%5min" + } + + time { + format = " %a %h %d โ %I:%M " + } + + spotify { + color_playing = "#fdf6e3" + color_paused = "#93a1a1" + format_stopped = "" + format_down = "" + format = "{title} - {artist} ({album})" + } + + external_script inbox { + script_path = '${emacsclient "(aspen/num-inbox-items-message)"}' + format = 'Inbox: {output}' + cache_timeout = 120 + color = "#93a1a1" + } + + external_script current_task { + script_path = '${ + emacsclient "(aspen/org-current-clocked-in-task-message)" + }' + # format = '{output}' + cache_timeout = 60 + color = "#93a1a1" + } + + tztime utc { + timezone = "UTC" + format = " %Hยท%M " + } + + volume_status { + format = "โ {percentage}" + format_muted = "โ X" + # device = "default" + # mixer_idx = 0 + } + + weather_owm { + api_key = '@owmApiKey@' + unit_temperature = 'c' + format = '{icon} {temperature}[ {rain}]' + } + ''; + + i3status-command = pkgs.writeShellScript "i3status.sh" '' + sed -s "s/@owmApiKey@/$(pass owm-api-key)/" \ + < ${i3status-conf} \ + > /tmp/i3status.conf + py3status -c /tmp/i3status.conf + ''; inherit (builtins) map; inherit (lib) mkMerge range; @@ -39,9 +137,9 @@ in }; battery = mkOption { - description = "Does this system have a battery?"; - default = true; - type = types.bool; + description = "Battery index for this system's battery"; + default = 0; + type = types.nullOr types.int; }; }; }; @@ -60,7 +158,7 @@ in home.packages = with pkgs; [ rofi rofi-pass - python3Packages.py3status + depot.users.aspen.pkgs.py3status i3lock i3status dconf # for gtk @@ -179,10 +277,6 @@ in # Screen Layout "${mod}+Shift+t" = "exec xrandr --auto"; - "${mod}+t" = "exec ${screenlayout.home}"; - "${mod}+Ctrl+t" = "exec ${pkgs.writeShellScript "fix_term.sh" '' - xrandr --output eDP-1 --off && ${screenlayout.home} - ''}"; # Notifications "${mod}+Shift+n" = "exec killall -SIGUSR1 .dunst-wrapped"; @@ -224,99 +318,7 @@ in }; bars = [{ - statusCommand = - let - i3status-conf = pkgs.writeText "i3status.conf" '' - general { - output_format = i3bar - colors = true - color_good = "#859900" - - interval = 1 - } - - order += "external_script current_task" - order += "external_script inbox" - order += "spotify" - order += "volume_status" - order += "wireless ${config.system.machine.wirelessInterface}" - # order += "ethernet enp3s0f0" - order += "cpu_usage" - ${lib.optionalString (config.system.machine.battery) '' - order += "battery 0" - ''} - # order += "volume master" - order += "time" - order += "tztime utc" - - mpd { - format = "%artist - %album - %title" - } - - wireless ${config.system.machine.wirelessInterface} { - format_up = "W: (%quality - %essid - %bitrate) %ip" - format_down = "W: -" - } - - ethernet enp3s0f0 { - format_up = "E: %ip" - format_down = "E: -" - } - - battery 0 { - format = "%status %percentage" - path = "/sys/class/power_supply/BAT%d/uevent" - low_threshold = 10 - } - - cpu_usage { - format = "CPU: %usage" - } - - load { - format = "%5min" - } - - time { - format = " %a %h %d โ %I:%M " - } - - spotify { - color_playing = "#fdf6e3" - color_paused = "#93a1a1" - format_stopped = "" - format_down = "" - format = "{title} - {artist} ({album})" - } - - external_script inbox { - script_path = '${emacsclient "(grfn/num-inbox-items-message)"}' - format = 'Inbox: {output}' - cache_timeout = 120 - color = "#93a1a1" - } - - external_script current_task { - script_path = '${emacsclient "(grfn/org-current-clocked-in-task-message)"}' - # format = '{output}' - cache_timeout = 60 - color = "#93a1a1" - } - - tztime utc { - timezone = "UTC" - format = " %Hยท%M " - } - - volume_status { - format = "โ {percentage}" - format_muted = "โ X" - # device = "default" - # mixer_idx = 0 - } - ''; - in - "py3status -c ${i3status-conf}"; + statusCommand = "${i3status-command}"; inherit fonts; position = "top"; colors = with solarized; rec { diff --git a/users/aspen/system/home/modules/lib/cloneRepo.nix b/users/aspen/system/home/modules/lib/cloneRepo.nix index 806a8be03c..f3cf59d249 100644 --- a/users/aspen/system/home/modules/lib/cloneRepo.nix +++ b/users/aspen/system/home/modules/lib/cloneRepo.nix @@ -2,7 +2,7 @@ with lib; { options = { - grfn.impure.clonedRepos = mkOption { + aspen.impure.clonedRepos = mkOption { description = "Repositories to clone"; default = { }; type = with types; attrsOf ( @@ -71,6 +71,6 @@ with lib; fi fi '') - config.grfn.impure.clonedRepos; + config.aspen.impure.clonedRepos; }; } diff --git a/users/aspen/system/home/modules/shell.nix b/users/aspen/system/home/modules/shell.nix index 676617d937..844f76c286 100644 --- a/users/aspen/system/home/modules/shell.nix +++ b/users/aspen/system/home/modules/shell.nix @@ -57,7 +57,7 @@ in programs.zsh = { enable = true; - enableAutosuggestions = true; + autosuggestion.enable = true; autocd = true; inherit shellAliases; diff --git a/users/aspen/system/home/modules/tarsnap.nix b/users/aspen/system/home/modules/tarsnap.nix index 87002610cb..fb2a050852 100644 --- a/users/aspen/system/home/modules/tarsnap.nix +++ b/users/aspen/system/home/modules/tarsnap.nix @@ -9,10 +9,10 @@ ### Recommended options # Tarsnap cache directory - cachedir /home/grfn/.cache/tarsnap + cachedir /home/aspen/.cache/tarsnap # Tarsnap key file - keyfile /home/grfn/.private/tarsnap.key + keyfile /home/aspen/.private/tarsnap.key # Don't archive files which have the nodump flag set. nodump diff --git a/users/aspen/system/home/modules/vimrc b/users/aspen/system/home/modules/vimrc index 3e33b5e2be..01572f3946 100644 --- a/users/aspen/system/home/modules/vimrc +++ b/users/aspen/system/home/modules/vimrc @@ -22,7 +22,7 @@ set expandtab set noerrorbells visualbell t_vb= set laststatus=2 set hidden -let mapleader = ',' +let mapleader = ' ' let maplocalleader = '\' set undofile " set undodir=~/.vim/undo diff --git a/users/aspen/system/home/platforms/linux.nix b/users/aspen/system/home/platforms/linux.nix index 1bcfa08e0d..d7a04e872d 100644 --- a/users/aspen/system/home/platforms/linux.nix +++ b/users/aspen/system/home/platforms/linux.nix @@ -12,9 +12,7 @@ in ../modules/emacs.nix ../modules/email.nix ../modules/firefox.nix - ../modules/games.nix ../modules/shell.nix - ../modules/tarsnap.nix ../modules/vim.nix ]; @@ -37,8 +35,7 @@ in gnutls pandoc barrier - depot.tools.nsfv-setup - gimp # TODO(grfn): use glimpse once it build again + gimp # TODO(aspen): use glimpse once it build again # System utilities powertop @@ -55,7 +52,7 @@ in keybase openssl yubikey-manager - # TODO(grfn): lagging behind yubikey-manager and doesn't support cryptography >= 39 + # TODO(aspen): lagging behind yubikey-manager and doesn't support cryptography >= 39 # yubikey-manager-qt # Spotify...etc diff --git a/users/aspen/system/system/default.nix b/users/aspen/system/system/default.nix index ec3a002051..07bc886c6c 100644 --- a/users/aspen/system/system/default.nix +++ b/users/aspen/system/system/default.nix @@ -16,7 +16,7 @@ rec { # Use the same nixpkgs as everything else home-manager.useGlobalPkgs = true; - home-manager.users.grfn = { config, lib, ... }: { + home-manager.users.aspen = { config, lib, ... }: { imports = [ ../home/machines/roswell.nix ]; lib.depot = depot; }; @@ -30,11 +30,16 @@ rec { yerenSystem = (depot.ops.nixos.nixosFor yeren).system; + lusca = import ./machines/lusca.nix; + + luscaSystem = (depot.ops.nixos.nixosFor lusca).system; + iso = import ./iso.nix args; meta.ci.targets = [ "mugwumpSystem" "roswellSystem" + "luscaSystem" "ogopogoSystem" "yerenSystem" diff --git a/users/aspen/system/system/iso.nix b/users/aspen/system/system/iso.nix index 398145f1cf..ef5d3ed78b 100644 --- a/users/aspen/system/system/iso.nix +++ b/users/aspen/system/system/iso.nix @@ -12,7 +12,7 @@ let networking.firewall.enable = false; networking.wireless.enable = lib.mkForce false; - # TODO(grfn): enabling this (in the minimal profile) fails the iso build, + # TODO(aspen): enabling this (in the minimal profile) fails the iso build, # since gtk+3 needs to be built which fails due to cairo without xlibs environment.noXlibs = false; }; diff --git a/users/aspen/system/system/machines/lusca.nix b/users/aspen/system/system/machines/lusca.nix new file mode 100644 index 0000000000..782d504aa9 --- /dev/null +++ b/users/aspen/system/system/machines/lusca.nix @@ -0,0 +1,142 @@ +{ depot, modulesPath, config, lib, pkgs, ... }: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ../modules/common.nix + ../modules/laptop.nix + ../modules/xserver.nix + ../modules/fonts.nix + ../modules/sound.nix + ../modules/tvl.nix + ../modules/development.nix + ]; + + networking.hostName = "lusca"; + + system.stateVersion = "24.05"; + + time.timeZone = "America/New_York"; + + services.avahi = { + enable = true; + nssmdns4 = true; + }; + + boot = { + initrd = { + availableKernelModules = + [ "nvme" "xhci_pci" "thunderbolt" "usb_storage" "sd_mod" ]; + kernelModules = [ ]; + + luks.devices."cryptroot".device = + "/dev/disk/by-uuid/9e525746-5bca-4451-8710-a6f0e09b751c"; + }; + + kernelModules = [ "kvm-amd" ]; + + kernelParams = [ + "resume=LABEL=SWAP" + "resume_offset=795904" # sudo btrfs inspect-internal map-swapfile -r /swap/swapfile + ]; + + resumeDevice = "/dev/disk/by-uuid/4c099cee-8d42-49c1-916c-62a0b5effbd2"; + + kernel.sysctl = { "kernel.perf_event_paranoid" = -1; }; + }; + + hardware.cpu.amd.updateMicrocode = + lib.mkDefault config.hardware.enableRedistributableFirmware; + + fileSystems = { + "/" = { + device = "/dev/disk/by-uuid/4c099cee-8d42-49c1-916c-62a0b5effbd2"; + fsType = "btrfs"; + options = [ "subvol=root" ]; + }; + + "/home" = { + device = "/dev/disk/by-uuid/4c099cee-8d42-49c1-916c-62a0b5effbd2"; + fsType = "btrfs"; + options = [ "subvol=home" ]; + }; + + "/nix" = { + device = "/dev/disk/by-uuid/4c099cee-8d42-49c1-916c-62a0b5effbd2"; + fsType = "btrfs"; + options = [ "subvol=nix" ]; + }; + + "/swap" = { + device = "/dev/disk/by-uuid/4c099cee-8d42-49c1-916c-62a0b5effbd2"; + fsType = "btrfs"; + options = [ "subvol=swap" ]; + }; + + "/boot" = { + device = "/dev/disk/by-uuid/0E7D-3C3F"; + fsType = "vfat"; + }; + }; + + swapDevices = [{ device = "/swap/swapfile"; }]; + + systemd.sleep.extraConfig = '' + HibernateDelaySec=30m + SuspendState=mem + ''; + + services.earlyoom = { + enable = true; + freeMemThreshold = 5; + }; + + services.tailscale.enable = true; + + services.fwupd = { + enable = true; + extraRemotes = [ "lvfs-testing" ]; + }; + + services.tlp.enable = lib.mkForce false; + services.power-profiles-daemon.enable = true; + + services.thermald.enable = true; + + services.fprintd.enable = true; + security.pam.services = { + login.fprintAuth = true; + sudo.fprintAuth = true; + i3lock.fprintAuth = true; + i3lock-color.fprintAuth = true; + lightdm.fprintAuth = true; + lightdm-greeter.fprintAuth = true; + }; + + security.polkit.extraConfig = '' + polkit.addRule(function(action, subject) { + if (action.id.indexOf("net.reactivated.fprint.") == 0 || action.id.indexOf("net.reactivated.Fprint.") == 0) { + polkit.log("action=" + action); + polkit.log("subject=" + subject); + return polkit.Result.YES; + } + }); + ''; + + services.udev.extraRules = '' + # Ethernet expansion card support + ACTION=="add", SUBSYSTEM=="usb", ATTR{idVendor}=="0bda", ATTR{idProduct}=="8156", ATTR{power/autosuspend}="20" + ''; + + hardware.sensor.iio.enable = true; + + hardware.opengl.driSupport32Bit = true; + + # TPM + security.tpm2 = { + enable = true; + pkcs11.enable = true; + tctiEnvironment.enable = true; + }; + users.users.aspen.extraGroups = [ "tss" ]; +} diff --git a/users/aspen/system/system/machines/mugwump.nix b/users/aspen/system/system/machines/mugwump.nix index 446f7cd92d..4cfa117134 100644 --- a/users/aspen/system/system/machines/mugwump.nix +++ b/users/aspen/system/system/machines/mugwump.nix @@ -73,7 +73,7 @@ with lib; nix.gc.dates = "monthly"; - users.users.grfn.openssh.authorizedKeys.keys = [ + users.users.aspen.openssh.authorizedKeys.keys = [ depot.users.aspen.keys.whitby ]; diff --git a/users/aspen/system/system/machines/ogopogo.nix b/users/aspen/system/system/machines/ogopogo.nix index d64d5931e3..e80a0906db 100644 --- a/users/aspen/system/system/machines/ogopogo.nix +++ b/users/aspen/system/system/machines/ogopogo.nix @@ -96,48 +96,6 @@ }; }; - services.buildkite-agents.ogopogo-1 = rec { - enable = true; - tokenPath = config.age.secretsDir + "/buildkite-token"; - privateSshKeyPath = config.age.secretsDir + "/buildkite-ssh-key"; - runtimePackages = with pkgs; [ - docker - nix - gnutar - gzip - bash - ]; - tags = { - queue = "ogopogo"; - }; - dataDir = "/home/grfn/buildkite-agent"; - - hooks.environment = '' - export BUILDKITE_AGENT_HOME=${dataDir} - ''; - }; - systemd.services.buildkite-agent-ogopogo-1.serviceConfig.User = - lib.mkForce "grfn"; - users.users.grfn.extraGroups = [ "keys" ]; - - age.secrets = - let - secret = name: depot.users.aspen.secrets."${name}.age"; - in - { - buildkite-ssh-key = { - file = secret "buildkite-ssh-key"; - group = "keys"; - mode = "0440"; - }; - - buildkite-token = { - file = secret "buildkite-token"; - group = "keys"; - mode = "0440"; - }; - }; - nix.settings.substituters = [ "ssh://grfn@172.16.0.5" ]; nix.settings.trusted-substituters = [ "ssh://grfn@172.16.0.5" ]; programs.ssh.knownHosts.mugwump = { diff --git a/users/aspen/system/system/machines/roswell.nix b/users/aspen/system/system/machines/roswell.nix index e7529542a8..da62eec93e 100644 --- a/users/aspen/system/system/machines/roswell.nix +++ b/users/aspen/system/system/machines/roswell.nix @@ -12,10 +12,6 @@ networking.hostName = "roswell"; - users.users.grfn.openssh.authorizedKeys.keys = [ - depot.users.aspen.keys.main - ]; - boot.loader.systemd-boot.enable = lib.mkForce false; boot.loader.efi.canTouchEfiVariables = lib.mkForce false; diff --git a/users/aspen/system/system/modules/common.nix b/users/aspen/system/system/modules/common.nix index 4ed1f8d7b1..3eaeb2efc6 100644 --- a/users/aspen/system/system/modules/common.nix +++ b/users/aspen/system/system/modules/common.nix @@ -44,7 +44,13 @@ with lib; documentation.dev.enable = true; documentation.man.generateCaches = true; - services.openssh.enable = true; + services.openssh = { + enable = true; + settings = { X11Forwarding = true; }; + }; + + users.users.aspen.openssh.authorizedKeys.keys = + [ depot.users.aspen.keys.main ]; programs.ssh.startAgent = true; @@ -53,7 +59,7 @@ with lib; users.mutableUsers = true; programs.zsh.enable = true; environment.pathsToLink = [ "/share/zsh" ]; - users.users.grfn = { + users.users.aspen = { isNormalUser = true; initialPassword = "password"; extraGroups = [ @@ -65,7 +71,7 @@ with lib; }; nix = { - settings.trusted-users = [ "grfn" ]; + settings.trusted-users = [ "aspen" ]; distributedBuilds = true; gc = { diff --git a/users/aspen/system/system/modules/desktop.nix b/users/aspen/system/system/modules/desktop.nix index 3adbd9d9b0..9a5fc825e1 100644 --- a/users/aspen/system/system/modules/desktop.nix +++ b/users/aspen/system/system/modules/desktop.nix @@ -10,7 +10,7 @@ programs.nm-applet.enable = true; - users.users.grfn.extraGroups = [ + users.users.aspen.extraGroups = [ "audio" "video" ]; diff --git a/users/aspen/system/system/modules/development.nix b/users/aspen/system/system/modules/development.nix index d17e9d424c..bd5e326b2e 100644 --- a/users/aspen/system/system/modules/development.nix +++ b/users/aspen/system/system/modules/development.nix @@ -2,11 +2,11 @@ { virtualisation.docker.enable = true; - users.users.grfn.extraGroups = [ "docker" ]; + users.users.aspen.extraGroups = [ "docker" ]; security.pam.loginLimits = [ { - domain = "grfn"; + domain = "aspen"; type = "soft"; item = "nofile"; value = "65535"; diff --git a/users/aspen/system/system/modules/fonts.nix b/users/aspen/system/system/modules/fonts.nix index 46c7667538..598336790a 100644 --- a/users/aspen/system/system/modules/fonts.nix +++ b/users/aspen/system/system/modules/fonts.nix @@ -5,6 +5,7 @@ nerdfonts noto-fonts-emoji twitter-color-emoji + weather-icons ]; fontconfig.defaultFonts.emoji = [ "Twitter Color Emoji" ]; diff --git a/users/aspen/system/system/modules/laptop.nix b/users/aspen/system/system/modules/laptop.nix index 05c5333e51..89c880973d 100644 --- a/users/aspen/system/system/modules/laptop.nix +++ b/users/aspen/system/system/modules/laptop.nix @@ -1,15 +1,23 @@ { config, lib, pkgs, ... }: { - imports = [ - ./reusable/battery.nix - ]; + services.logind = { + powerKey = "hibernate"; + powerKeyLongPress = "poweroff"; + lidSwitch = "suspend-then-hibernate"; + lidSwitchExternalPower = "ignore"; + }; - laptop.onLowBattery.enable = true; - - services.logind.extraConfig = '' - HandlePowerKey=hibernate + systemd.sleep.extraConfig = '' + HibernateDelaySec=30m + SuspendState=mem ''; services.tlp.enable = true; + + services.upower = { + enable = true; + criticalPowerAction = "Hibernate"; + percentageAction = 3; + }; } diff --git a/users/aspen/system/system/modules/reusable/battery.nix b/users/aspen/system/system/modules/reusable/battery.nix deleted file mode 100644 index 151c2a246f..0000000000 --- a/users/aspen/system/system/modules/reusable/battery.nix +++ /dev/null @@ -1,32 +0,0 @@ -{ config, lib, pkgs, ... }: -with lib; -{ - options = { - laptop.onLowBattery = { - enable = mkEnableOption "Perform action on low battery"; - - thresholdPercentage = mkOption { - description = "Threshold battery percentage on which to perform the action"; - default = 8; - type = types.int; - }; - - action = mkOption { - description = "Action to perform on low battery"; - default = "hibernate"; - type = types.enum [ "hibernate" "suspend" "suspend-then-hibernate" ]; - }; - }; - }; - - config = - let cfg = config.laptop.onLowBattery; - in mkIf cfg.enable { - services.udev.extraRules = concatStrings [ - ''SUBSYSTEM=="power_supply", '' - ''ATTR{status}=="Discharging", '' - ''ATTR{capacity}=="[0-${toString cfg.thresholdPercentage}]", '' - ''RUN+="${pkgs.systemd}/bin/systemctl ${cfg.action}"'' - ]; - }; -} diff --git a/users/aspen/system/system/modules/wireshark.nix b/users/aspen/system/system/modules/wireshark.nix index 30658419fc..b233d40041 100644 --- a/users/aspen/system/system/modules/wireshark.nix +++ b/users/aspen/system/system/modules/wireshark.nix @@ -5,5 +5,5 @@ enable = true; package = pkgs.wireshark; }; - users.users.grfn.extraGroups = [ "wireshark" ]; + users.users.aspen.extraGroups = [ "wireshark" ]; } diff --git a/users/aspen/web/index.org b/users/aspen/web/index.org index f4c45fd3aa..4be79fd797 100644 --- a/users/aspen/web/index.org +++ b/users/aspen/web/index.org @@ -27,7 +27,6 @@ that's wire-compatible with postgresql and mysql, based on [[https://github.com/ - https://sacrosanct.bandcamp.com/, a post-rock project with a [[https://bandcamp.com/h34rken][friend of mine]] - [[https://soundcloud.com/missingggg][my current soundcloud]], releasing instrumental music under the name *missing* - i play bass in [[https://goodcry.band][good cry]], a rock band based in brooklyn - - we're playing [[https://www.eventbrite.com/e/skylar-pocket-w-leche-malo-good-cry-tickets-836634866407?aff=oddtdtcreator][a show on febuary 24th at the broadway in brooklyn]] - you can also find a log of all the music I listen to [[https://www.last.fm/user/wildgriffin45][on last.fm]] * contact diff --git a/users/picnoir/tvix-daemon/.gitignore b/users/picnoir/tvix-daemon/.gitignore new file mode 100644 index 0000000000..ea8c4bf7f3 --- /dev/null +++ b/users/picnoir/tvix-daemon/.gitignore @@ -0,0 +1 @@ +/target diff --git a/users/picnoir/tvix-daemon/Cargo.lock b/users/picnoir/tvix-daemon/Cargo.lock index 635c54fe4a..683203f5ca 100644 --- a/users/picnoir/tvix-daemon/Cargo.lock +++ b/users/picnoir/tvix-daemon/Cargo.lock @@ -18,30 +18,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] -name = "aho-corasick" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" -dependencies = [ - "memchr", -] - -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] name = "anstream" version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -90,41 +66,6 @@ dependencies = [ ] [[package]] -name = "anyhow" -version = "1.0.81" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" - -[[package]] -name = "arc-swap" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b3d0060af21e8d11a926981cc00c6c1541aa91dd64b9f881985c3da1094425f" - -[[package]] -name = "arrayref" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" - -[[package]] -name = "arrayvec" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" - -[[package]] -name = "async-recursion" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.52", -] - -[[package]] name = "async-stream" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -143,17 +84,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", -] - -[[package]] -name = "async-tempfile" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b37d4bb113c47e4f263d4b0221912ff5aa840a51bc9b7b47b024e1cf1926fd9b" -dependencies = [ - "tokio", - "uuid", + "syn", ] [[package]] @@ -164,7 +95,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn", ] [[package]] @@ -175,46 +106,18 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.6.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" -dependencies = [ - "async-trait", - "axum-core 0.3.4", - "bitflags 1.3.2", - "bytes", - "futures-util", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.28", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum" version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1236b4b292f6c4d6dc34604bb5120d85c3fe1d1aa596bd5cc52ca054d13e7b9e" dependencies = [ "async-trait", - "axum-core 0.4.3", + "axum-core", "bytes", "futures-util", - "http 1.1.0", - "http-body 1.0.0", + "http", + "http-body", "http-body-util", - "hyper 1.2.0", + "hyper", "hyper-util", "itoa", "matchit", @@ -237,23 +140,6 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http 0.2.12", - "http-body 0.4.6", - "mime", - "rustversion", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" @@ -261,8 +147,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.1.0", - "http-body 1.0.0", + "http", + "http-body", "http-body-util", "mime", "pin-project-lite", @@ -289,12 +175,6 @@ dependencies = [ ] [[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - -[[package]] name = "base64ct" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -313,21 +193,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] -name = "blake3" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" -dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", - "digest", - "rayon", -] - -[[package]] name = "block-buffer" version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -343,47 +208,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" dependencies = [ "memchr", - "regex-automata 0.4.6", + "regex-automata", "serde", ] [[package]] -name = "bumpalo" -version = "3.15.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] name = "bytes" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] -name = "caps" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190baaad529bcfbde9e1a19022c42781bdb6ff9de25721abdb8fd98c0807730b" -dependencies = [ - "libc", - "thiserror", -] - -[[package]] name = "cc" version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" -dependencies = [ - "jobserver", - "libc", -] [[package]] name = "cfg-if" @@ -392,19 +231,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] -name = "chrono" -version = "0.4.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaf5903dcbc0a39312feb77df2ff4c76387d591b9fc7b04a238dcf8bb62639a" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "num-traits", - "serde", - "windows-targets 0.52.4", -] - -[[package]] name = "clap" version = "4.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -432,10 +258,10 @@ version = "4.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90239a040c80f5e14809ca132ddc4176ab33d5e17e49691793296e3fcb34d72f" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", - "syn 2.0.52", + "syn", ] [[package]] @@ -457,34 +283,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] -name = "constant_time_eq" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" - -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" - -[[package]] -name = "count-write" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced507ab50aa0123e2c54db8b5f44fdfee04b1c93744d69e924307945fe57a85" - -[[package]] name = "cpufeatures" version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -494,49 +292,6 @@ dependencies = [ ] [[package]] -name = "crc32fast" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" - -[[package]] name = "crypto-common" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -571,7 +326,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn", ] [[package]] @@ -598,16 +353,9 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", - "subtle", ] [[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - -[[package]] name = "document-features" version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -641,21 +389,6 @@ dependencies = [ ] [[package]] -name = "either" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" - -[[package]] -name = "encoding_rs" -version = "0.8.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" -dependencies = [ - "cfg-if", -] - -[[package]] name = "enum-primitive-derive" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -663,7 +396,7 @@ checksum = "ba7795da175654fe16979af73f81f26a8ea27638d8d9823d317016888a63dc4c" dependencies = [ "num-traits", "quote", - "syn 2.0.52", + "syn", ] [[package]] @@ -673,45 +406,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] -name = "errno" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "fastcdc" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a71061d097bfa9a5a4d2efdec57990d9a88745020b365191d37e48541a1628f2" -dependencies = [ - "async-stream", - "tokio", - "tokio-stream", -] - -[[package]] -name = "fastrand" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" - -[[package]] name = "fiat-crypto" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1676f435fc1dadde4d03e43f5d62b259e1ce5f40bd4ffb21db2b42ebe59c1382" [[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - -[[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -727,57 +427,12 @@ dependencies = [ ] [[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "fuse-backend-rs" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5a63a89f40ec26a0a1434e89de3f4ee939a920eae15d641053ee09ee6ed44b" -dependencies = [ - "arc-swap", - "bitflags 1.3.2", - "caps", - "core-foundation-sys", - "lazy_static", - "libc", - "log", - "mio", - "nix 0.24.3", - "vm-memory", - "vmm-sys-util", -] - -[[package]] -name = "futures" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] name = "futures-channel" version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", - "futures-sink", ] [[package]] @@ -787,23 +442,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] -name = "futures-executor" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" - -[[package]] name = "futures-macro" version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -811,7 +449,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn", ] [[package]] @@ -832,28 +470,15 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ - "futures-channel", "futures-core", - "futures-io", "futures-macro", - "futures-sink", "futures-task", - "memchr", "pin-project-lite", "pin-utils", "slab", ] [[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - -[[package]] name = "generic-array" version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -888,25 +513,6 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.3.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap 2.2.5", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "h2" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31d030e59af851932b72ceebadf4a2b5986dba4c3b99dd2493f8273a0f151943" @@ -916,8 +522,8 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http 1.1.0", - "indexmap 2.2.5", + "http", + "indexmap", "slab", "tokio", "tokio-util", @@ -926,24 +532,12 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "hashbrown" version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" [[package]] name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - -[[package]] -name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" @@ -955,26 +549,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] -name = "home" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" -dependencies = [ - "windows-sys 0.52.0", -] - -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] name = "http" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -987,23 +561,12 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - -[[package]] -name = "http-body" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", - "http 1.1.0", + "http", ] [[package]] @@ -1014,8 +577,8 @@ checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" dependencies = [ "bytes", "futures-core", - "http 1.1.0", - "http-body 1.0.0", + "http", + "http-body", "pin-project-lite", ] @@ -1032,36 +595,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - -[[package]] -name = "hyper" -version = "0.14.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.3.24", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] name = "hyper" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1070,9 +603,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.2", - "http 1.1.0", - "http-body 1.0.0", + "h2", + "http", + "http-body", "httparse", "httpdate", "itoa", @@ -1082,32 +615,6 @@ dependencies = [ ] [[package]] -name = "hyper-rustls" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" -dependencies = [ - "futures-util", - "http 0.2.12", - "hyper 0.14.28", - "rustls 0.21.10", - "tokio", - "tokio-rustls 0.24.1", -] - -[[package]] -name = "hyper-timeout" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" -dependencies = [ - "hyper 0.14.28", - "pin-project-lite", - "tokio", - "tokio-io-timeout", -] - -[[package]] name = "hyper-util" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1115,107 +622,22 @@ checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" dependencies = [ "bytes", "futures-util", - "http 1.1.0", - "http-body 1.0.0", - "hyper 1.2.0", + "http", + "http-body", + "hyper", "pin-project-lite", "socket2", "tokio", ] [[package]] -name = "iana-time-zone" -version = "0.1.60" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "idna" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - -[[package]] name = "indexmap" version = "2.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" dependencies = [ "equivalent", - "hashbrown 0.14.3", -] - -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "ipnet" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", + "hashbrown", ] [[package]] @@ -1225,24 +647,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] -name = "jobserver" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" -dependencies = [ - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" -dependencies = [ - "wasm-bindgen", -] - -[[package]] name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1255,12 +659,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] -name = "linux-raw-sys" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" - -[[package]] name = "litrs" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1283,57 +681,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - -[[package]] -name = "matchers" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" -dependencies = [ - "regex-automata 0.1.10", -] - -[[package]] name = "matchit" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if", - "digest", -] - -[[package]] name = "memchr" version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] -name = "memoffset" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" -dependencies = [ - "autocfg", -] - -[[package]] name = "mime" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1361,30 +720,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", - "log", "wasi", "windows-sys 0.48.0", ] [[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - -[[package]] -name = "nix" -version = "0.24.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" -dependencies = [ - "bitflags 1.3.2", - "cfg-if", - "libc", - "memoffset", -] - -[[package]] name = "nix" version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1405,10 +745,10 @@ dependencies = [ "ed25519", "ed25519-dalek", "enum-primitive-derive", - "futures-util", "glob", "nom", "num-traits", + "pin-project-lite", "serde", "serde_json", "sha2", @@ -1465,136 +805,12 @@ dependencies = [ ] [[package]] -name = "object_store" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8718f8b65fdf67a45108d1548347d4af7d71fb81ce727bbf9e3b2535e079db3" -dependencies = [ - "async-trait", - "base64", - "bytes", - "chrono", - "futures", - "humantime", - "hyper 0.14.28", - "itertools 0.12.1", - "md-5", - "parking_lot 0.12.1", - "percent-encoding", - "quick-xml", - "rand", - "reqwest", - "ring", - "rustls-pemfile 2.1.1", - "serde", - "serde_json", - "snafu", - "tokio", - "tracing", - "url", - "walkdir", -] - -[[package]] name = "once_cell" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - -[[package]] -name = "opentelemetry" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" -dependencies = [ - "futures-core", - "futures-sink", - "indexmap 2.2.5", - "js-sys", - "once_cell", - "pin-project-lite", - "thiserror", - "urlencoding", -] - -[[package]] -name = "opentelemetry-otlp" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f24cda83b20ed2433c68241f918d0f6fdec8b1d43b7a9590ab4420c5095ca930" -dependencies = [ - "async-trait", - "futures-core", - "http 0.2.12", - "opentelemetry", - "opentelemetry-proto", - "opentelemetry-semantic-conventions", - "opentelemetry_sdk", - "prost 0.11.9", - "thiserror", - "tokio", - "tonic 0.9.2", -] - -[[package]] -name = "opentelemetry-proto" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2e155ce5cc812ea3d1dffbd1539aed653de4bf4882d60e6e04dcf0901d674e1" -dependencies = [ - "opentelemetry", - "opentelemetry_sdk", - "prost 0.11.9", - "tonic 0.9.2", -] - -[[package]] -name = "opentelemetry-semantic-conventions" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84" -dependencies = [ - "opentelemetry", -] - -[[package]] -name = "opentelemetry_sdk" -version = "0.21.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4" -dependencies = [ - "async-trait", - "crossbeam-channel", - "futures-channel", - "futures-executor", - "futures-util", - "glob", - "once_cell", - "opentelemetry", - "ordered-float", - "percent-encoding", - "rand", - "thiserror", - "tokio", - "tokio-stream", -] - -[[package]] -name = "ordered-float" -version = "4.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" -dependencies = [ - "num-traits", -] - -[[package]] name = "overload" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1602,37 +818,12 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] - -[[package]] -name = "parking_lot" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.9", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "winapi", + "parking_lot_core", ] [[package]] @@ -1643,7 +834,7 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.4.1", + "redox_syscall", "smallvec", "windows-targets 0.48.5", ] @@ -1655,16 +846,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] -name = "petgraph" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" -dependencies = [ - "fixedbitset", - "indexmap 2.2.5", -] - -[[package]] name = "pin-project" version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1681,7 +862,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn", ] [[package]] @@ -1707,34 +888,12 @@ dependencies = [ ] [[package]] -name = "pkg-config" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" - -[[package]] name = "platforms" version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "626dec3cac7cc0e1577a2ec3fc496277ec2baa084bebad95bb6fdbfae235f84c" [[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - -[[package]] -name = "prettyplease" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" -dependencies = [ - "proc-macro2", - "syn 2.0.52", -] - -[[package]] name = "proc-macro2" version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1744,93 +903,6 @@ dependencies = [ ] [[package]] -name = "prost" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" -dependencies = [ - "bytes", - "prost-derive 0.11.9", -] - -[[package]] -name = "prost" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" -dependencies = [ - "bytes", - "prost-derive 0.12.3", -] - -[[package]] -name = "prost-build" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" -dependencies = [ - "bytes", - "heck 0.4.1", - "itertools 0.11.0", - "log", - "multimap", - "once_cell", - "petgraph", - "prettyplease", - "prost 0.12.3", - "prost-types", - "regex", - "syn 2.0.52", - "tempfile", - "which", -] - -[[package]] -name = "prost-derive" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" -dependencies = [ - "anyhow", - "itertools 0.10.5", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "prost-derive" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" -dependencies = [ - "anyhow", - "itertools 0.11.0", - "proc-macro2", - "quote", - "syn 2.0.52", -] - -[[package]] -name = "prost-types" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" -dependencies = [ - "prost 0.12.3", -] - -[[package]] -name = "quick-xml" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" -dependencies = [ - "memchr", - "serde", -] - -[[package]] name = "quote" version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1840,27 +912,6 @@ dependencies = [ ] [[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1870,35 +921,6 @@ dependencies = [ ] [[package]] -name = "rayon" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] name = "redox_syscall" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1908,106 +930,10 @@ dependencies = [ ] [[package]] -name = "regex" -version = "1.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata 0.4.6", - "regex-syntax 0.8.2", -] - -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", -] - -[[package]] name = "regex-automata" version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax 0.8.2", -] - -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - -[[package]] -name = "regex-syntax" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" - -[[package]] -name = "reqwest" -version = "0.11.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bf93c4af7a8bb7d879d51cebe797356ff10ae8516ace542b5182d9dcac10b2" -dependencies = [ - "base64", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2 0.3.24", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.28", - "hyper-rustls", - "ipnet", - "js-sys", - "log", - "mime", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls 0.21.10", - "rustls-native-certs 0.6.3", - "rustls-pemfile 1.0.4", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "system-configuration", - "tokio", - "tokio-rustls 0.24.1", - "tokio-util", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", - "winreg", -] - -[[package]] -name = "ring" -version = "0.17.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" -dependencies = [ - "cc", - "cfg-if", - "getrandom", - "libc", - "spin", - "untrusted", - "windows-sys 0.52.0", -] [[package]] name = "rustc-demangle" @@ -2025,116 +951,6 @@ dependencies = [ ] [[package]] -name = "rustix" -version = "0.38.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" -dependencies = [ - "bitflags 2.4.2", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.52.0", -] - -[[package]] -name = "rustls" -version = "0.21.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" -dependencies = [ - "log", - "ring", - "rustls-webpki 0.101.7", - "sct", -] - -[[package]] -name = "rustls" -version = "0.22.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41" -dependencies = [ - "log", - "ring", - "rustls-pki-types", - "rustls-webpki 0.102.2", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-native-certs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" -dependencies = [ - "openssl-probe", - "rustls-pemfile 1.0.4", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-native-certs" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" -dependencies = [ - "openssl-probe", - "rustls-pemfile 2.1.1", - "rustls-pki-types", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64", -] - -[[package]] -name = "rustls-pemfile" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f48172685e6ff52a556baa527774f61fcaa884f59daf3375c62a3f1cd2549dab" -dependencies = [ - "base64", - "rustls-pki-types", -] - -[[package]] -name = "rustls-pki-types" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ede67b28608b4c60685c7d54122d4400d90f62b40caee7700e700380a390fa8" - -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "rustls-webpki" -version = "0.102.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted", -] - -[[package]] name = "rustversion" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2147,63 +963,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "schannel" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" -dependencies = [ - "windows-sys 0.52.0", -] - -[[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "security-framework" -version = "2.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] name = "semver" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2226,7 +991,7 @@ checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn", ] [[package]] @@ -2310,50 +1075,12 @@ dependencies = [ ] [[package]] -name = "sled" -version = "0.34.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" -dependencies = [ - "crc32fast", - "crossbeam-epoch", - "crossbeam-utils", - "fs2", - "fxhash", - "libc", - "log", - "parking_lot 0.11.2", -] - -[[package]] name = "smallvec" version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] -name = "snafu" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" -dependencies = [ - "doc-comment", - "snafu-derive", -] - -[[package]] -name = "snafu-derive" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] name = "socket2" version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2364,12 +1091,6 @@ dependencies = [ ] [[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" - -[[package]] name = "spki" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2393,17 +1114,6 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" version = "2.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" @@ -2420,39 +1130,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] -name = "system-configuration" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "tempfile" -version = "3.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" -dependencies = [ - "cfg-if", - "fastrand", - "rustix", - "windows-sys 0.52.0", -] - -[[package]] name = "thiserror" version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2469,7 +1146,7 @@ checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn", ] [[package]] @@ -2483,21 +1160,6 @@ dependencies = [ ] [[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] name = "tokio" version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2508,7 +1170,7 @@ dependencies = [ "libc", "mio", "num_cpus", - "parking_lot 0.12.1", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -2517,31 +1179,20 @@ dependencies = [ ] [[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - -[[package]] name = "tokio-listener" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96367e127b4cf47b92592a5154a563435fe28fe3fccf25917d4a34ee59c87303" dependencies = [ - "axum 0.7.4", + "axum", "document-features", "futures-core", "futures-util", - "nix 0.26.4", + "nix", "pin-project", "socket2", "tokio", "tokio-util", - "tonic 0.11.0", "tracing", ] @@ -2553,28 +1204,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", -] - -[[package]] -name = "tokio-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" -dependencies = [ - "rustls 0.21.10", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" -dependencies = [ - "rustls 0.22.2", - "rustls-pki-types", - "tokio", + "syn", ] [[package]] @@ -2609,7 +1239,6 @@ checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", - "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -2617,91 +1246,6 @@ dependencies = [ ] [[package]] -name = "tonic" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" -dependencies = [ - "async-trait", - "axum 0.6.20", - "base64", - "bytes", - "futures-core", - "futures-util", - "h2 0.3.24", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.28", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost 0.11.9", - "tokio", - "tokio-stream", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tonic" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" -dependencies = [ - "async-stream", - "async-trait", - "axum 0.6.20", - "base64", - "bytes", - "h2 0.3.24", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.28", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost 0.12.3", - "rustls-native-certs 0.7.0", - "rustls-pemfile 2.1.1", - "rustls-pki-types", - "tokio", - "tokio-rustls 0.25.0", - "tokio-stream", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tonic-build" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4ef6dd70a610078cb4e338a0f79d06bc759ff1b22d2120c2ff02ae264ba9c2" -dependencies = [ - "prettyplease", - "proc-macro2", - "prost-build", - "quote", - "syn 2.0.52", -] - -[[package]] -name = "tonic-reflection" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "548c227bd5c0fae5925812c4ec6c66ffcfced23ea370cb823f4d18f0fc1cb6a7" -dependencies = [ - "prost 0.12.3", - "prost-types", - "tokio", - "tokio-stream", - "tonic 0.11.0", -] - -[[package]] name = "tower" version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2709,13 +1253,9 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", - "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand", - "slab", "tokio", - "tokio-util", "tower-layer", "tower-service", "tracing", @@ -2753,7 +1293,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn", ] [[package]] @@ -2778,101 +1318,23 @@ dependencies = [ ] [[package]] -name = "tracing-opentelemetry" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596" -dependencies = [ - "js-sys", - "once_cell", - "opentelemetry", - "opentelemetry_sdk", - "smallvec", - "tracing", - "tracing-core", - "tracing-log", - "tracing-subscriber", - "web-time", -] - -[[package]] -name = "tracing-serde" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" -dependencies = [ - "serde", - "tracing-core", -] - -[[package]] name = "tracing-subscriber" version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ - "matchers", "nu-ansi-term", - "once_cell", - "regex", - "serde", - "serde_json", "sharded-slab", "smallvec", "thread_local", - "tracing", "tracing-core", "tracing-log", - "tracing-serde", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "tvix-castore" -version = "0.1.0" -dependencies = [ - "async-stream", - "async-tempfile", - "blake3", - "bstr", - "bytes", - "data-encoding", - "digest", - "fastcdc", - "fuse-backend-rs", - "futures", - "lazy_static", - "libc", - "object_store", - "parking_lot 0.12.1", - "pin-project-lite", - "prost 0.12.3", - "prost-build", - "sled", - "thiserror", - "tokio", - "tokio-stream", - "tokio-util", - "tonic 0.11.0", - "tonic-build", - "tonic-reflection", - "tower", - "tracing", - "url", - "walkdir", - "zstd", ] [[package]] name = "tvix-daemon" version = "0.1.0" dependencies = [ - "anyhow", "clap", "nix-compat", "tokio", @@ -2880,50 +1342,6 @@ dependencies = [ "tokio-test", "tracing", "tracing-subscriber", - "tvix-store", -] - -[[package]] -name = "tvix-store" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-recursion", - "async-stream", - "blake3", - "bstr", - "bytes", - "clap", - "count-write", - "data-encoding", - "futures", - "lazy_static", - "nix-compat", - "opentelemetry", - "opentelemetry-otlp", - "opentelemetry_sdk", - "pin-project-lite", - "prost 0.12.3", - "prost-build", - "reqwest", - "sha2", - "sled", - "thiserror", - "tokio", - "tokio-listener", - "tokio-stream", - "tokio-util", - "tonic 0.11.0", - "tonic-build", - "tonic-reflection", - "tower", - "tracing", - "tracing-opentelemetry", - "tracing-subscriber", - "tvix-castore", - "url", - "walkdir", - "xz2", ] [[package]] @@ -2933,65 +1351,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] -name = "unicode-bidi" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" - -[[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] -name = "unicode-normalization" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "url" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" - -[[package]] name = "utf8parse" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] -name = "uuid" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" -dependencies = [ - "getrandom", -] - -[[package]] name = "valuable" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3004,162 +1375,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] -name = "vm-memory" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688a70366615b45575a424d9c665561c1b5ab2224d494f706b6a6812911a827c" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "vmm-sys-util" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48b7b084231214f7427041e4220d77dfe726897a6d41fddee450696e66ff2a29" -dependencies = [ - "bitflags 1.3.2", - "libc", -] - -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] -name = "wasm-bindgen" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" -dependencies = [ - "cfg-if", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.52", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.52", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" - -[[package]] -name = "wasm-streams" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "web-sys" -version = "0.3.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "web-time" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa30049b1c872b72c89866d458eae9f20380ab280ffd1b1e18df2d3e2d98cfe0" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix", -] - -[[package]] name = "winapi" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3176,30 +1397,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] -name = "winapi-util" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" -dependencies = [ - "winapi", -] - -[[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows-core" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" -dependencies = [ - "windows-targets 0.52.4", -] - -[[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3332,54 +1535,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" [[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - -[[package]] name = "zeroize" version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" - -[[package]] -name = "zstd" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.9+zstd.1.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/users/picnoir/tvix-daemon/Cargo.nix b/users/picnoir/tvix-daemon/Cargo.nix index f284ba3342..2382027f9b 100644 --- a/users/picnoir/tvix-daemon/Cargo.nix +++ b/users/picnoir/tvix-daemon/Cargo.nix @@ -125,57 +125,6 @@ rec { "rustc-dep-of-std" = [ "core" "compiler_builtins" ]; }; }; - "aho-corasick" = rec { - crateName = "aho-corasick"; - version = "1.1.2"; - edition = "2021"; - sha256 = "1w510wnixvlgimkx1zjbvlxh6xps2vjgfqgwf5a6adlbjp5rv5mj"; - libName = "aho_corasick"; - authors = [ - "Andrew Gallant <jamslam@gmail.com>" - ]; - dependencies = [ - { - name = "memchr"; - packageId = "memchr"; - optional = true; - usesDefaultFeatures = false; - } - ]; - features = { - "default" = [ "std" "perf-literal" ]; - "logging" = [ "dep:log" ]; - "perf-literal" = [ "dep:memchr" ]; - "std" = [ "memchr?/std" ]; - }; - resolvedDefaultFeatures = [ "std" ]; - }; - "android-tzdata" = rec { - crateName = "android-tzdata"; - version = "0.1.1"; - edition = "2018"; - sha256 = "1w7ynjxrfs97xg3qlcdns4kgfpwcdv824g611fq32cag4cdr96g9"; - authors = [ - "RumovZ" - ]; - - }; - "android_system_properties" = rec { - crateName = "android_system_properties"; - version = "0.1.5"; - edition = "2018"; - sha256 = "04b3wrz12837j7mdczqd95b732gw5q7q66cv4yn4646lvccp57l1"; - authors = [ - "Nicolas Silva <nical@fastmail.com>" - ]; - dependencies = [ - { - name = "libc"; - packageId = "libc"; - } - ]; - - }; "anstream" = rec { crateName = "anstream"; version = "0.6.13"; @@ -280,85 +229,6 @@ rec { ]; }; - "anyhow" = rec { - crateName = "anyhow"; - version = "1.0.81"; - edition = "2018"; - sha256 = "0ivj2k7vajai9hc11lhjp73n6rqs1ykg6wbjjjl1mz9adj580lh9"; - authors = [ - "David Tolnay <dtolnay@gmail.com>" - ]; - features = { - "backtrace" = [ "dep:backtrace" ]; - "default" = [ "std" ]; - }; - resolvedDefaultFeatures = [ "default" "std" ]; - }; - "arc-swap" = rec { - crateName = "arc-swap"; - version = "1.7.0"; - edition = "2018"; - sha256 = "0ps2jh8dmhw53649yjyn3nlilm61qq0cr0b9j8dd3s11mxh00gbv"; - authors = [ - "Michal 'vorner' Vaner <vorner@vorner.cz>" - ]; - features = { - "serde" = [ "dep:serde" ]; - }; - }; - "arrayref" = rec { - crateName = "arrayref"; - version = "0.3.7"; - edition = "2015"; - sha256 = "0ia5ndyxqkzdymqr4ls53jdmajf09adjimg5kvw65kkprg930jbb"; - authors = [ - "David Roundy <roundyd@physics.oregonstate.edu>" - ]; - - }; - "arrayvec" = rec { - crateName = "arrayvec"; - version = "0.7.4"; - edition = "2018"; - sha256 = "04b7n722jij0v3fnm3qk072d5ysc2q30rl9fz33zpfhzah30mlwn"; - authors = [ - "bluss" - ]; - features = { - "default" = [ "std" ]; - "serde" = [ "dep:serde" ]; - "zeroize" = [ "dep:zeroize" ]; - }; - }; - "async-recursion" = rec { - crateName = "async-recursion"; - version = "1.0.5"; - edition = "2018"; - sha256 = "1l2vlgyaa9a2dd0y1vbqyppzsvpdr1y4rar4gn1qi68pl5dmmmaz"; - procMacro = true; - authors = [ - "Robert Usher <266585+dcchut@users.noreply.github.com>" - ]; - dependencies = [ - { - name = "proc-macro2"; - packageId = "proc-macro2"; - usesDefaultFeatures = false; - } - { - name = "quote"; - packageId = "quote"; - usesDefaultFeatures = false; - } - { - name = "syn"; - packageId = "syn 2.0.52"; - usesDefaultFeatures = false; - features = [ "full" "parsing" "printing" "proc-macro" "clone-impls" ]; - } - ]; - - }; "async-stream" = rec { crateName = "async-stream"; version = "0.3.5"; @@ -403,46 +273,12 @@ rec { } { name = "syn"; - packageId = "syn 2.0.52"; + packageId = "syn"; features = [ "full" "visit-mut" ]; } ]; }; - "async-tempfile" = rec { - crateName = "async-tempfile"; - version = "0.4.0"; - edition = "2021"; - sha256 = "16zx4qcwzq94n13pp6xwa4589apm5y8j20jb7lk4yzn42fqlnzdk"; - authors = [ - "Markus Mayer" - ]; - dependencies = [ - { - name = "tokio"; - packageId = "tokio"; - features = [ "fs" ]; - } - { - name = "uuid"; - packageId = "uuid"; - optional = true; - features = [ "v4" ]; - } - ]; - devDependencies = [ - { - name = "tokio"; - packageId = "tokio"; - features = [ "rt-multi-thread" "macros" ]; - } - ]; - features = { - "default" = [ "uuid" ]; - "uuid" = [ "dep:uuid" ]; - }; - resolvedDefaultFeatures = [ "default" "uuid" ]; - }; "async-trait" = rec { crateName = "async-trait"; version = "0.1.77"; @@ -463,7 +299,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.52"; + packageId = "syn"; features = [ "full" "visit-mut" ]; } ]; @@ -479,135 +315,7 @@ rec { ]; }; - "axum 0.6.20" = rec { - crateName = "axum"; - version = "0.6.20"; - edition = "2021"; - sha256 = "1gynqkg3dcy1zd7il69h8a3zax86v6qq5zpawqyn87mr6979x0iv"; - dependencies = [ - { - name = "async-trait"; - packageId = "async-trait"; - } - { - name = "axum-core"; - packageId = "axum-core 0.3.4"; - } - { - name = "bitflags"; - packageId = "bitflags 1.3.2"; - } - { - name = "bytes"; - packageId = "bytes"; - } - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - features = [ "alloc" ]; - } - { - name = "http"; - packageId = "http 0.2.12"; - } - { - name = "http-body"; - packageId = "http-body 0.4.6"; - } - { - name = "hyper"; - packageId = "hyper 0.14.28"; - features = [ "stream" ]; - } - { - name = "itoa"; - packageId = "itoa"; - } - { - name = "matchit"; - packageId = "matchit"; - } - { - name = "memchr"; - packageId = "memchr"; - } - { - name = "mime"; - packageId = "mime"; - } - { - name = "percent-encoding"; - packageId = "percent-encoding"; - } - { - name = "pin-project-lite"; - packageId = "pin-project-lite"; - } - { - name = "serde"; - packageId = "serde"; - } - { - name = "sync_wrapper"; - packageId = "sync_wrapper"; - } - { - name = "tower"; - packageId = "tower"; - usesDefaultFeatures = false; - features = [ "util" ]; - } - { - name = "tower-layer"; - packageId = "tower-layer"; - } - { - name = "tower-service"; - packageId = "tower-service"; - } - ]; - buildDependencies = [ - { - name = "rustversion"; - packageId = "rustversion"; - } - ]; - devDependencies = [ - { - name = "rustversion"; - packageId = "rustversion"; - } - { - name = "serde"; - packageId = "serde"; - features = [ "derive" ]; - } - { - name = "tower"; - packageId = "tower"; - rename = "tower"; - features = [ "util" "timeout" "limit" "load-shed" "steer" "filter" ]; - } - ]; - features = { - "__private_docs" = [ "tower/full" "dep:tower-http" ]; - "default" = [ "form" "http1" "json" "matched-path" "original-uri" "query" "tokio" "tower-log" ]; - "form" = [ "dep:serde_urlencoded" ]; - "headers" = [ "dep:headers" ]; - "http1" = [ "hyper/http1" ]; - "http2" = [ "hyper/http2" ]; - "json" = [ "dep:serde_json" "dep:serde_path_to_error" ]; - "macros" = [ "dep:axum-macros" ]; - "multipart" = [ "dep:multer" ]; - "query" = [ "dep:serde_urlencoded" ]; - "tokio" = [ "dep:tokio" "hyper/server" "hyper/tcp" "hyper/runtime" "tower/make" ]; - "tower-log" = [ "tower/log" ]; - "tracing" = [ "dep:tracing" "axum-core/tracing" ]; - "ws" = [ "tokio" "dep:tokio-tungstenite" "dep:sha1" "dep:base64" ]; - }; - }; - "axum 0.7.4" = rec { + "axum" = rec { crateName = "axum"; version = "0.7.4"; edition = "2021"; @@ -619,7 +327,7 @@ rec { } { name = "axum-core"; - packageId = "axum-core 0.4.3"; + packageId = "axum-core"; } { name = "bytes"; @@ -633,11 +341,11 @@ rec { } { name = "http"; - packageId = "http 1.1.0"; + packageId = "http"; } { name = "http-body"; - packageId = "http-body 1.0.0"; + packageId = "http-body"; } { name = "http-body-util"; @@ -645,7 +353,7 @@ rec { } { name = "hyper"; - packageId = "hyper 1.2.0"; + packageId = "hyper"; optional = true; } { @@ -784,67 +492,7 @@ rec { }; resolvedDefaultFeatures = [ "default" "form" "http1" "json" "matched-path" "original-uri" "query" "tokio" "tower-log" "tracing" ]; }; - "axum-core 0.3.4" = rec { - crateName = "axum-core"; - version = "0.3.4"; - edition = "2021"; - sha256 = "0b1d9nkqb8znaba4qqzxzc968qwj4ybn4vgpyz9lz4a7l9vsb7vm"; - dependencies = [ - { - name = "async-trait"; - packageId = "async-trait"; - } - { - name = "bytes"; - packageId = "bytes"; - } - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - features = [ "alloc" ]; - } - { - name = "http"; - packageId = "http 0.2.12"; - } - { - name = "http-body"; - packageId = "http-body 0.4.6"; - } - { - name = "mime"; - packageId = "mime"; - } - { - name = "tower-layer"; - packageId = "tower-layer"; - } - { - name = "tower-service"; - packageId = "tower-service"; - } - ]; - buildDependencies = [ - { - name = "rustversion"; - packageId = "rustversion"; - } - ]; - devDependencies = [ - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - features = [ "alloc" ]; - } - ]; - features = { - "__private_docs" = [ "dep:tower-http" ]; - "tracing" = [ "dep:tracing" ]; - }; - }; - "axum-core 0.4.3" = rec { + "axum-core" = rec { crateName = "axum-core"; version = "0.4.3"; edition = "2021"; @@ -866,11 +514,11 @@ rec { } { name = "http"; - packageId = "http 1.1.0"; + packageId = "http"; } { name = "http-body"; - packageId = "http-body 1.0.0"; + packageId = "http-body"; } { name = "http-body-util"; @@ -984,21 +632,6 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "base64" = rec { - crateName = "base64"; - version = "0.21.7"; - edition = "2018"; - sha256 = "0rw52yvsk75kar9wgqfwgb414kvil1gn7mqkrhn9zf1537mpsacx"; - authors = [ - "Alice Maz <alice@alicemaz.com>" - "Marshall Pierce <marshall@mpierce.org>" - ]; - features = { - "default" = [ "std" ]; - "std" = [ "alloc" ]; - }; - resolvedDefaultFeatures = [ "alloc" "default" "std" ]; - }; "base64ct" = rec { crateName = "base64ct"; version = "1.6.0"; @@ -1043,63 +676,6 @@ rec { "rustc-dep-of-std" = [ "core" "compiler_builtins" ]; "serde" = [ "dep:serde" ]; }; - resolvedDefaultFeatures = [ "std" ]; - }; - "blake3" = rec { - crateName = "blake3"; - version = "1.5.1"; - edition = "2021"; - sha256 = "0lpgw3gl564wqwf98v9jsa29lp6nv7w7nn1cvw6w75s5cz9sdk1h"; - authors = [ - "Jack O'Connor <oconnor663@gmail.com>" - "Samuel Neves" - ]; - dependencies = [ - { - name = "arrayref"; - packageId = "arrayref"; - } - { - name = "arrayvec"; - packageId = "arrayvec"; - usesDefaultFeatures = false; - } - { - name = "cfg-if"; - packageId = "cfg-if"; - } - { - name = "constant_time_eq"; - packageId = "constant_time_eq"; - } - { - name = "digest"; - packageId = "digest"; - optional = true; - features = [ "mac" ]; - } - { - name = "rayon"; - packageId = "rayon"; - optional = true; - } - ]; - buildDependencies = [ - { - name = "cc"; - packageId = "cc"; - } - ]; - features = { - "default" = [ "std" ]; - "digest" = [ "dep:digest" ]; - "mmap" = [ "std" "dep:memmap2" ]; - "rayon" = [ "dep:rayon" "std" ]; - "serde" = [ "dep:serde" ]; - "traits-preview" = [ "digest" ]; - "zeroize" = [ "dep:zeroize" "arrayvec/zeroize" ]; - }; - resolvedDefaultFeatures = [ "default" "digest" "rayon" "std" "traits-preview" ]; }; "block-buffer" = rec { crateName = "block-buffer"; @@ -1133,7 +709,7 @@ rec { } { name = "regex-automata"; - packageId = "regex-automata 0.4.6"; + packageId = "regex-automata"; optional = true; usesDefaultFeatures = false; features = [ "dfa-search" ]; @@ -1154,32 +730,6 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "serde" "std" "unicode" ]; }; - "bumpalo" = rec { - crateName = "bumpalo"; - version = "3.15.4"; - edition = "2021"; - sha256 = "1ahfhgw2lzlgv5j0h07z8mkdnk4kvl2grf8dkb32dm4zsjfrpxkz"; - authors = [ - "Nick Fitzgerald <fitzgen@gmail.com>" - ]; - features = { - "allocator-api2" = [ "dep:allocator-api2" ]; - }; - resolvedDefaultFeatures = [ "default" ]; - }; - "byteorder" = rec { - crateName = "byteorder"; - version = "1.5.0"; - edition = "2021"; - sha256 = "0jzncxyf404mwqdbspihyzpkndfgda450l0893pz5xj685cg5l0z"; - authors = [ - "Andrew Gallant <jamslam@gmail.com>" - ]; - features = { - "default" = [ "std" ]; - }; - resolvedDefaultFeatures = [ "default" "std" ]; - }; "bytes" = rec { crateName = "bytes"; version = "1.5.0"; @@ -1195,29 +745,6 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "caps" = rec { - crateName = "caps"; - version = "0.5.5"; - edition = "2018"; - sha256 = "02vk0w48rncgvfmj2mz2kpzvdgc14z225451w7lvvkwvaansl2qr"; - authors = [ - "Luca Bruno <lucab@lucabruno.net>" - ]; - dependencies = [ - { - name = "libc"; - packageId = "libc"; - } - { - name = "thiserror"; - packageId = "thiserror"; - } - ]; - features = { - "serde" = [ "dep:serde" ]; - "serde_support" = [ "serde" ]; - }; - }; "cc" = rec { crateName = "cc"; version = "1.0.90"; @@ -1226,27 +753,11 @@ rec { authors = [ "Alex Crichton <alex@alexcrichton.com>" ]; - dependencies = [ - { - name = "jobserver"; - packageId = "jobserver"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "libc"; - packageId = "libc"; - optional = true; - usesDefaultFeatures = false; - target = { target, features }: (target."unix" or false); - } - ]; features = { "jobserver" = [ "dep:jobserver" ]; "libc" = [ "dep:libc" ]; "parallel" = [ "libc" "jobserver" ]; }; - resolvedDefaultFeatures = [ "jobserver" "libc" "parallel" ]; }; "cfg-if" = rec { crateName = "cfg-if"; @@ -1262,68 +773,6 @@ rec { "rustc-dep-of-std" = [ "core" "compiler_builtins" ]; }; }; - "chrono" = rec { - crateName = "chrono"; - version = "0.4.35"; - edition = "2021"; - sha256 = "16k3caxzip1ql827pz5rj7aqfqy7yhpxyxzb5wqkj2mwvh1mkbwf"; - dependencies = [ - { - name = "android-tzdata"; - packageId = "android-tzdata"; - optional = true; - target = { target, features }: ("android" == target."os" or null); - } - { - name = "iana-time-zone"; - packageId = "iana-time-zone"; - optional = true; - target = { target, features }: (target."unix" or false); - features = [ "fallback" ]; - } - { - name = "num-traits"; - packageId = "num-traits"; - usesDefaultFeatures = false; - } - { - name = "serde"; - packageId = "serde"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "windows-targets"; - packageId = "windows-targets 0.52.4"; - optional = true; - target = { target, features }: (target."windows" or false); - } - ]; - features = { - "android-tzdata" = [ "dep:android-tzdata" ]; - "arbitrary" = [ "dep:arbitrary" ]; - "clock" = [ "winapi" "iana-time-zone" "android-tzdata" "now" ]; - "default" = [ "clock" "std" "oldtime" "wasmbind" ]; - "iana-time-zone" = [ "dep:iana-time-zone" ]; - "js-sys" = [ "dep:js-sys" ]; - "now" = [ "std" ]; - "pure-rust-locales" = [ "dep:pure-rust-locales" ]; - "rkyv" = [ "dep:rkyv" "rkyv/size_32" ]; - "rkyv-16" = [ "dep:rkyv" "rkyv?/size_16" ]; - "rkyv-32" = [ "dep:rkyv" "rkyv?/size_32" ]; - "rkyv-64" = [ "dep:rkyv" "rkyv?/size_64" ]; - "rkyv-validation" = [ "rkyv?/validation" ]; - "rustc-serialize" = [ "dep:rustc-serialize" ]; - "serde" = [ "dep:serde" ]; - "std" = [ "alloc" ]; - "unstable-locales" = [ "pure-rust-locales" ]; - "wasm-bindgen" = [ "dep:wasm-bindgen" ]; - "wasmbind" = [ "wasm-bindgen" "js-sys" ]; - "winapi" = [ "windows-targets" ]; - "windows-targets" = [ "dep:windows-targets" ]; - }; - resolvedDefaultFeatures = [ "alloc" "android-tzdata" "clock" "iana-time-zone" "now" "serde" "std" "winapi" "windows-targets" ]; - }; "clap" = rec { crateName = "clap"; version = "4.5.3"; @@ -1412,7 +861,7 @@ rec { dependencies = [ { name = "heck"; - packageId = "heck 0.5.0"; + packageId = "heck"; } { name = "proc-macro2"; @@ -1424,7 +873,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.52"; + packageId = "syn"; features = [ "full" ]; } ]; @@ -1460,75 +909,6 @@ rec { "arbitrary" = [ "dep:arbitrary" ]; }; }; - "constant_time_eq" = rec { - crateName = "constant_time_eq"; - version = "0.3.0"; - edition = "2021"; - sha256 = "1hl0y8frzlhpr58rh8rlg4bm53ax09ikj2i5fk7gpyphvhq4s57p"; - authors = [ - "Cesar Eduardo Barros <cesarb@cesarb.eti.br>" - ]; - features = { }; - }; - "core-foundation" = rec { - crateName = "core-foundation"; - version = "0.9.4"; - edition = "2018"; - sha256 = "13zvbbj07yk3b61b8fhwfzhy35535a583irf23vlcg59j7h9bqci"; - authors = [ - "The Servo Project Developers" - ]; - dependencies = [ - { - name = "core-foundation-sys"; - packageId = "core-foundation-sys"; - usesDefaultFeatures = false; - } - { - name = "libc"; - packageId = "libc"; - } - ]; - features = { - "chrono" = [ "dep:chrono" ]; - "default" = [ "link" ]; - "link" = [ "core-foundation-sys/link" ]; - "mac_os_10_7_support" = [ "core-foundation-sys/mac_os_10_7_support" ]; - "mac_os_10_8_features" = [ "core-foundation-sys/mac_os_10_8_features" ]; - "uuid" = [ "dep:uuid" ]; - "with-chrono" = [ "chrono" ]; - "with-uuid" = [ "uuid" ]; - }; - resolvedDefaultFeatures = [ "default" "link" ]; - }; - "core-foundation-sys" = rec { - crateName = "core-foundation-sys"; - version = "0.8.6"; - edition = "2018"; - sha256 = "13w6sdf06r0hn7bx2b45zxsg1mm2phz34jikm6xc5qrbr6djpsh6"; - authors = [ - "The Servo Project Developers" - ]; - features = { - "default" = [ "link" ]; - }; - resolvedDefaultFeatures = [ "default" "link" ]; - }; - "count-write" = rec { - crateName = "count-write"; - version = "0.1.0"; - edition = "2018"; - sha256 = "11bswmgr81s3jagdci1pr6qh9vnz9zsbbf2dqpi260daa2mhgmff"; - authors = [ - "SOFe <sofe2038@gmail.com>" - ]; - features = { - "futures" = [ "futures-io-preview" ]; - "futures-io-preview" = [ "dep:futures-io-preview" ]; - "tokio" = [ "tokio-io" ]; - "tokio-io" = [ "dep:tokio-io" ]; - }; - }; "cpufeatures" = rec { crateName = "cpufeatures"; version = "0.2.12"; @@ -1561,99 +941,6 @@ rec { ]; }; - "crc32fast" = rec { - crateName = "crc32fast"; - version = "1.4.0"; - edition = "2015"; - sha256 = "1ahy259ypc955l5ak24hdlgllb6vm6y2pvwr6qrlyisbg255m1dk"; - authors = [ - "Sam Rijs <srijs@airpost.net>" - "Alex Crichton <alex@alexcrichton.com>" - ]; - dependencies = [ - { - name = "cfg-if"; - packageId = "cfg-if"; - } - ]; - features = { - "default" = [ "std" ]; - }; - resolvedDefaultFeatures = [ "default" "std" ]; - }; - "crossbeam-channel" = rec { - crateName = "crossbeam-channel"; - version = "0.5.12"; - edition = "2021"; - sha256 = "159gp30clv2ci3r473659ii04pjznspb3g9gwkhj2lavkhmb0gdb"; - dependencies = [ - { - name = "crossbeam-utils"; - packageId = "crossbeam-utils"; - usesDefaultFeatures = false; - } - ]; - features = { - "default" = [ "std" ]; - "std" = [ "crossbeam-utils/std" ]; - }; - resolvedDefaultFeatures = [ "default" "std" ]; - }; - "crossbeam-deque" = rec { - crateName = "crossbeam-deque"; - version = "0.8.5"; - edition = "2021"; - sha256 = "03bp38ljx4wj6vvy4fbhx41q8f585zyqix6pncz1mkz93z08qgv1"; - dependencies = [ - { - name = "crossbeam-epoch"; - packageId = "crossbeam-epoch"; - usesDefaultFeatures = false; - } - { - name = "crossbeam-utils"; - packageId = "crossbeam-utils"; - usesDefaultFeatures = false; - } - ]; - features = { - "default" = [ "std" ]; - "std" = [ "crossbeam-epoch/std" "crossbeam-utils/std" ]; - }; - resolvedDefaultFeatures = [ "default" "std" ]; - }; - "crossbeam-epoch" = rec { - crateName = "crossbeam-epoch"; - version = "0.9.18"; - edition = "2021"; - sha256 = "03j2np8llwf376m3fxqx859mgp9f83hj1w34153c7a9c7i5ar0jv"; - dependencies = [ - { - name = "crossbeam-utils"; - packageId = "crossbeam-utils"; - usesDefaultFeatures = false; - } - ]; - features = { - "default" = [ "std" ]; - "loom" = [ "loom-crate" "crossbeam-utils/loom" ]; - "loom-crate" = [ "dep:loom-crate" ]; - "nightly" = [ "crossbeam-utils/nightly" ]; - "std" = [ "alloc" "crossbeam-utils/std" ]; - }; - resolvedDefaultFeatures = [ "alloc" "default" "std" ]; - }; - "crossbeam-utils" = rec { - crateName = "crossbeam-utils"; - version = "0.8.19"; - edition = "2021"; - sha256 = "0iakrb1b8fjqrag7wphl94d10irhbh2fw1g444xslsywqyn3p3i4"; - features = { - "default" = [ "std" ]; - "loom" = [ "dep:loom" ]; - }; - resolvedDefaultFeatures = [ "default" "std" ]; - }; "crypto-common" = rec { crateName = "crypto-common"; version = "0.1.6"; @@ -1767,7 +1054,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.52"; + packageId = "syn"; features = [ "full" ]; } ]; @@ -1840,12 +1127,6 @@ rec { name = "crypto-common"; packageId = "crypto-common"; } - { - name = "subtle"; - packageId = "subtle"; - optional = true; - usesDefaultFeatures = false; - } ]; features = { "blobby" = [ "dep:blobby" ]; @@ -1860,18 +1141,7 @@ rec { "std" = [ "alloc" "crypto-common/std" ]; "subtle" = [ "dep:subtle" ]; }; - resolvedDefaultFeatures = [ "alloc" "block-buffer" "core-api" "default" "mac" "std" "subtle" ]; - }; - "doc-comment" = rec { - crateName = "doc-comment"; - version = "0.3.3"; - edition = "2015"; - sha256 = "043sprsf3wl926zmck1bm7gw0jq50mb76lkpk49vasfr6ax1p97y"; - libName = "doc_comment"; - authors = [ - "Guillaume Gomez <guillaume1.gomez@gmail.com>" - ]; - features = { }; + resolvedDefaultFeatures = [ "alloc" "block-buffer" "core-api" "default" "std" ]; }; "document-features" = rec { crateName = "document-features"; @@ -2002,43 +1272,6 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "fast" "std" "zeroize" ]; }; - "either" = rec { - crateName = "either"; - version = "1.10.0"; - edition = "2018"; - sha256 = "0jiyq2mc1aa5b8whwl1bhm11i06xxcbk9ck7macxxggzjk07l58i"; - authors = [ - "bluss" - ]; - features = { - "default" = [ "use_std" ]; - "serde" = [ "dep:serde" ]; - }; - resolvedDefaultFeatures = [ "default" "use_std" ]; - }; - "encoding_rs" = rec { - crateName = "encoding_rs"; - version = "0.8.33"; - edition = "2018"; - sha256 = "1qa5k4a0ipdrxq4xg9amms9r9pnnfn7nfh2i9m3mw0ka563b6s3j"; - authors = [ - "Henri Sivonen <hsivonen@hsivonen.fi>" - ]; - dependencies = [ - { - name = "cfg-if"; - packageId = "cfg-if"; - } - ]; - features = { - "default" = [ "alloc" ]; - "fast-legacy-encode" = [ "fast-hangul-encode" "fast-hanja-encode" "fast-kanji-encode" "fast-gb-hanzi-encode" "fast-big5-hanzi-encode" ]; - "packed_simd" = [ "dep:packed_simd" ]; - "serde" = [ "dep:serde" ]; - "simd-accel" = [ "packed_simd" "packed_simd/into_bits" ]; - }; - resolvedDefaultFeatures = [ "alloc" "default" ]; - }; "enum-primitive-derive" = rec { crateName = "enum-primitive-derive"; version = "0.3.0"; @@ -2060,7 +1293,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.52"; + packageId = "syn"; } ]; @@ -2072,103 +1305,6 @@ rec { sha256 = "1malmx5f4lkfvqasz319lq6gb3ddg19yzf9s8cykfsgzdmyq0hsl"; }; - "errno" = rec { - crateName = "errno"; - version = "0.3.8"; - edition = "2018"; - sha256 = "0ia28ylfsp36i27g1qih875cyyy4by2grf80ki8vhgh6vinf8n52"; - authors = [ - "Chris Wong <lambda.fairy@gmail.com>" - ]; - dependencies = [ - { - name = "libc"; - packageId = "libc"; - usesDefaultFeatures = false; - target = { target, features }: ("hermit" == target."os" or null); - } - { - name = "libc"; - packageId = "libc"; - usesDefaultFeatures = false; - target = { target, features }: ("wasi" == target."os" or null); - } - { - name = "libc"; - packageId = "libc"; - usesDefaultFeatures = false; - target = { target, features }: (target."unix" or false); - } - { - name = "windows-sys"; - packageId = "windows-sys 0.52.0"; - target = { target, features }: (target."windows" or false); - features = [ "Win32_Foundation" "Win32_System_Diagnostics_Debug" ]; - } - ]; - features = { - "default" = [ "std" ]; - "std" = [ "libc/std" ]; - }; - resolvedDefaultFeatures = [ "std" ]; - }; - "fastcdc" = rec { - crateName = "fastcdc"; - version = "3.1.0"; - edition = "2018"; - sha256 = "1wi82qd58j3ysf8m2dhb092qga6rj1wwbppgsajabadzjz862457"; - authors = [ - "Nathan Fiedler <nathanfiedler@fastmail.fm>" - ]; - dependencies = [ - { - name = "async-stream"; - packageId = "async-stream"; - optional = true; - } - { - name = "tokio"; - packageId = "tokio"; - optional = true; - features = [ "io-util" ]; - } - { - name = "tokio-stream"; - packageId = "tokio-stream"; - optional = true; - } - ]; - devDependencies = [ - { - name = "tokio"; - packageId = "tokio"; - features = [ "fs" "io-util" "rt" "rt-multi-thread" "macros" ]; - } - ]; - features = { - "async-stream" = [ "dep:async-stream" ]; - "futures" = [ "dep:futures" ]; - "tokio" = [ "dep:tokio" "tokio-stream" "async-stream" ]; - "tokio-stream" = [ "dep:tokio-stream" ]; - }; - resolvedDefaultFeatures = [ "async-stream" "default" "tokio" "tokio-stream" ]; - }; - "fastrand" = rec { - crateName = "fastrand"; - version = "2.0.1"; - edition = "2018"; - sha256 = "19flpv5zbzpf0rk4x77z4zf25in0brg8l7m304d3yrf47qvwxjr5"; - authors = [ - "Stjepan Glavina <stjepang@gmail.com>" - ]; - features = { - "default" = [ "std" ]; - "getrandom" = [ "dep:getrandom" ]; - "js" = [ "std" "getrandom" ]; - "std" = [ "alloc" ]; - }; - resolvedDefaultFeatures = [ "alloc" "default" "std" ]; - }; "fiat-crypto" = rec { crateName = "fiat-crypto"; version = "0.2.6"; @@ -2181,19 +1317,6 @@ rec { "default" = [ "std" ]; }; }; - "fixedbitset" = rec { - crateName = "fixedbitset"; - version = "0.4.2"; - edition = "2015"; - sha256 = "101v41amgv5n9h4hcghvrbfk5vrncx1jwm35rn5szv4rk55i7rqc"; - authors = [ - "bluss" - ]; - features = { - "default" = [ "std" ]; - "serde" = [ "dep:serde" ]; - }; - }; "fnv" = rec { crateName = "fnv"; version = "1.0.7"; @@ -2230,186 +1353,6 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "std" ]; }; - "fs2" = rec { - crateName = "fs2"; - version = "0.4.3"; - edition = "2015"; - sha256 = "04v2hwk7035c088f19mfl5b1lz84gnvv2hv6m935n0hmirszqr4m"; - authors = [ - "Dan Burkert <dan@danburkert.com>" - ]; - dependencies = [ - { - name = "libc"; - packageId = "libc"; - target = { target, features }: (target."unix" or false); - } - { - name = "winapi"; - packageId = "winapi"; - target = { target, features }: (target."windows" or false); - features = [ "handleapi" "processthreadsapi" "winerror" "fileapi" "winbase" "std" ]; - } - ]; - - }; - "fuse-backend-rs" = rec { - crateName = "fuse-backend-rs"; - version = "0.11.0"; - edition = "2018"; - sha256 = "0jyldvp0kvjk21j5vqga42lkksaf7zg8jkj3l6h2dv20kyl66nif"; - authors = [ - "Liu Bo <bo.liu@linux.alibaba.com>" - "Liu Jiang <gerry@linux.alibaba.com>" - "Peng Tao <bergwolf@hyper.sh>" - ]; - dependencies = [ - { - name = "arc-swap"; - packageId = "arc-swap"; - } - { - name = "bitflags"; - packageId = "bitflags 1.3.2"; - } - { - name = "caps"; - packageId = "caps"; - optional = true; - target = { target, features }: ("linux" == target."os" or null); - } - { - name = "core-foundation-sys"; - packageId = "core-foundation-sys"; - optional = true; - target = { target, features }: ("macos" == target."os" or null); - } - { - name = "lazy_static"; - packageId = "lazy_static"; - } - { - name = "libc"; - packageId = "libc"; - } - { - name = "log"; - packageId = "log"; - } - { - name = "mio"; - packageId = "mio"; - features = [ "os-poll" "os-ext" ]; - } - { - name = "nix"; - packageId = "nix 0.24.3"; - } - { - name = "vm-memory"; - packageId = "vm-memory"; - features = [ "backend-mmap" ]; - } - { - name = "vmm-sys-util"; - packageId = "vmm-sys-util"; - optional = true; - } - ]; - devDependencies = [ - { - name = "vm-memory"; - packageId = "vm-memory"; - features = [ "backend-mmap" "backend-bitmap" ]; - } - { - name = "vmm-sys-util"; - packageId = "vmm-sys-util"; - } - ]; - features = { - "async-io" = [ "async-trait" "tokio-uring" "tokio/fs" "tokio/net" "tokio/sync" "tokio/rt" "tokio/macros" "io-uring" ]; - "async-trait" = [ "dep:async-trait" ]; - "caps" = [ "dep:caps" ]; - "core-foundation-sys" = [ "dep:core-foundation-sys" ]; - "dbs-snapshot" = [ "dep:dbs-snapshot" ]; - "default" = [ "fusedev" ]; - "fusedev" = [ "vmm-sys-util" "caps" "core-foundation-sys" ]; - "io-uring" = [ "dep:io-uring" ]; - "persist" = [ "dbs-snapshot" "versionize" "versionize_derive" ]; - "tokio" = [ "dep:tokio" ]; - "tokio-uring" = [ "dep:tokio-uring" ]; - "versionize" = [ "dep:versionize" ]; - "versionize_derive" = [ "dep:versionize_derive" ]; - "vhost" = [ "dep:vhost" ]; - "vhost-user-fs" = [ "virtiofs" "vhost" "caps" ]; - "virtio-queue" = [ "dep:virtio-queue" ]; - "virtiofs" = [ "virtio-queue" "caps" "vmm-sys-util" ]; - "vmm-sys-util" = [ "dep:vmm-sys-util" ]; - }; - resolvedDefaultFeatures = [ "caps" "core-foundation-sys" "default" "fusedev" "vmm-sys-util" ]; - }; - "futures" = rec { - crateName = "futures"; - version = "0.3.30"; - edition = "2018"; - sha256 = "1c04g14bccmprwsvx2j9m2blhwrynq7vhl151lsvcv4gi0b6jp34"; - dependencies = [ - { - name = "futures-channel"; - packageId = "futures-channel"; - usesDefaultFeatures = false; - features = [ "sink" ]; - } - { - name = "futures-core"; - packageId = "futures-core"; - usesDefaultFeatures = false; - } - { - name = "futures-executor"; - packageId = "futures-executor"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "futures-io"; - packageId = "futures-io"; - usesDefaultFeatures = false; - } - { - name = "futures-sink"; - packageId = "futures-sink"; - usesDefaultFeatures = false; - } - { - name = "futures-task"; - packageId = "futures-task"; - usesDefaultFeatures = false; - } - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - features = [ "sink" ]; - } - ]; - features = { - "alloc" = [ "futures-core/alloc" "futures-task/alloc" "futures-sink/alloc" "futures-channel/alloc" "futures-util/alloc" ]; - "async-await" = [ "futures-util/async-await" "futures-util/async-await-macro" ]; - "bilock" = [ "futures-util/bilock" ]; - "compat" = [ "std" "futures-util/compat" ]; - "default" = [ "std" "async-await" "executor" ]; - "executor" = [ "std" "futures-executor/std" ]; - "futures-executor" = [ "dep:futures-executor" ]; - "io-compat" = [ "compat" "futures-util/io-compat" ]; - "std" = [ "alloc" "futures-core/std" "futures-task/std" "futures-io/std" "futures-sink/std" "futures-util/std" "futures-util/io" "futures-util/channel" ]; - "thread-pool" = [ "executor" "futures-executor/thread-pool" ]; - "unstable" = [ "futures-core/unstable" "futures-task/unstable" "futures-channel/unstable" "futures-io/unstable" "futures-util/unstable" ]; - "write-all-vectored" = [ "futures-util/write-all-vectored" ]; - }; - resolvedDefaultFeatures = [ "alloc" "async-await" "default" "executor" "futures-executor" "std" ]; - }; "futures-channel" = rec { crateName = "futures-channel"; version = "0.3.30"; @@ -2421,12 +1364,6 @@ rec { packageId = "futures-core"; usesDefaultFeatures = false; } - { - name = "futures-sink"; - packageId = "futures-sink"; - optional = true; - usesDefaultFeatures = false; - } ]; features = { "alloc" = [ "futures-core/alloc" ]; @@ -2435,7 +1372,7 @@ rec { "sink" = [ "futures-sink" ]; "std" = [ "alloc" "futures-core/std" ]; }; - resolvedDefaultFeatures = [ "alloc" "default" "futures-sink" "sink" "std" ]; + resolvedDefaultFeatures = [ "alloc" "default" "std" ]; }; "futures-core" = rec { crateName = "futures-core"; @@ -2449,46 +1386,6 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "std" ]; }; - "futures-executor" = rec { - crateName = "futures-executor"; - version = "0.3.30"; - edition = "2018"; - sha256 = "07dh08gs9vfll2h36kq32q9xd86xm6lyl9xikmmwlkqnmrrgqxm5"; - dependencies = [ - { - name = "futures-core"; - packageId = "futures-core"; - usesDefaultFeatures = false; - } - { - name = "futures-task"; - packageId = "futures-task"; - usesDefaultFeatures = false; - } - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - } - ]; - features = { - "default" = [ "std" ]; - "num_cpus" = [ "dep:num_cpus" ]; - "std" = [ "futures-core/std" "futures-task/std" "futures-util/std" ]; - "thread-pool" = [ "std" "num_cpus" ]; - }; - resolvedDefaultFeatures = [ "default" "std" ]; - }; - "futures-io" = rec { - crateName = "futures-io"; - version = "0.3.30"; - edition = "2018"; - sha256 = "1hgh25isvsr4ybibywhr4dpys8mjnscw4wfxxwca70cn1gi26im4"; - features = { - "default" = [ "std" ]; - }; - resolvedDefaultFeatures = [ "default" "std" ]; - }; "futures-macro" = rec { crateName = "futures-macro"; version = "0.3.30"; @@ -2506,7 +1403,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.52"; + packageId = "syn"; features = [ "full" ]; } ]; @@ -2541,47 +1438,22 @@ rec { sha256 = "0j0xqhcir1zf2dcbpd421kgw6wvsk0rpxflylcysn1rlp3g02r1x"; dependencies = [ { - name = "futures-channel"; - packageId = "futures-channel"; - optional = true; - usesDefaultFeatures = false; - features = [ "std" ]; - } - { name = "futures-core"; packageId = "futures-core"; usesDefaultFeatures = false; } { - name = "futures-io"; - packageId = "futures-io"; - optional = true; - usesDefaultFeatures = false; - features = [ "std" ]; - } - { name = "futures-macro"; packageId = "futures-macro"; optional = true; usesDefaultFeatures = false; } { - name = "futures-sink"; - packageId = "futures-sink"; - optional = true; - usesDefaultFeatures = false; - } - { name = "futures-task"; packageId = "futures-task"; usesDefaultFeatures = false; } { - name = "memchr"; - packageId = "memchr"; - optional = true; - } - { name = "pin-project-lite"; packageId = "pin-project-lite"; } @@ -2617,24 +1489,7 @@ rec { "unstable" = [ "futures-core/unstable" "futures-task/unstable" ]; "write-all-vectored" = [ "io" ]; }; - resolvedDefaultFeatures = [ "alloc" "async-await" "async-await-macro" "channel" "default" "futures-channel" "futures-io" "futures-macro" "futures-sink" "io" "memchr" "sink" "slab" "std" ]; - }; - "fxhash" = rec { - crateName = "fxhash"; - version = "0.2.1"; - edition = "2015"; - sha256 = "037mb9ichariqi45xm6mz0b11pa92gj38ba0409z3iz239sns6y3"; - libPath = "lib.rs"; - authors = [ - "cbreeden <github@u.breeden.cc>" - ]; - dependencies = [ - { - name = "byteorder"; - packageId = "byteorder"; - } - ]; - + resolvedDefaultFeatures = [ "alloc" "async-await" "async-await-macro" "default" "futures-macro" "slab" "std" ]; }; "generic-array" = rec { crateName = "generic-array"; @@ -2727,79 +1582,7 @@ rec { ]; }; - "h2 0.3.24" = rec { - crateName = "h2"; - version = "0.3.24"; - edition = "2018"; - sha256 = "1jf9488b66nayxzp3iw3b2rb64y49hdbbywnv9wfwrsv14i48b5v"; - authors = [ - "Carl Lerche <me@carllerche.com>" - "Sean McArthur <sean@seanmonstar.com>" - ]; - dependencies = [ - { - name = "bytes"; - packageId = "bytes"; - } - { - name = "fnv"; - packageId = "fnv"; - } - { - name = "futures-core"; - packageId = "futures-core"; - usesDefaultFeatures = false; - } - { - name = "futures-sink"; - packageId = "futures-sink"; - usesDefaultFeatures = false; - } - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - } - { - name = "http"; - packageId = "http 0.2.12"; - } - { - name = "indexmap"; - packageId = "indexmap 2.2.5"; - features = [ "std" ]; - } - { - name = "slab"; - packageId = "slab"; - } - { - name = "tokio"; - packageId = "tokio"; - features = [ "io-util" ]; - } - { - name = "tokio-util"; - packageId = "tokio-util"; - features = [ "codec" "io" ]; - } - { - name = "tracing"; - packageId = "tracing"; - usesDefaultFeatures = false; - features = [ "std" ]; - } - ]; - devDependencies = [ - { - name = "tokio"; - packageId = "tokio"; - features = [ "rt-multi-thread" "macros" "sync" "net" ]; - } - ]; - features = { }; - }; - "h2 0.4.2" = rec { + "h2" = rec { crateName = "h2"; version = "0.4.2"; edition = "2018"; @@ -2834,11 +1617,11 @@ rec { } { name = "http"; - packageId = "http 1.1.0"; + packageId = "http"; } { name = "indexmap"; - packageId = "indexmap 2.2.5"; + packageId = "indexmap"; features = [ "std" ]; } { @@ -2871,29 +1654,7 @@ rec { ]; features = { }; }; - "hashbrown 0.12.3" = rec { - crateName = "hashbrown"; - version = "0.12.3"; - edition = "2021"; - sha256 = "1268ka4750pyg2pbgsr43f0289l5zah4arir2k4igx5a8c6fg7la"; - authors = [ - "Amanieu d'Antras <amanieu@gmail.com>" - ]; - features = { - "ahash" = [ "dep:ahash" ]; - "ahash-compile-time-rng" = [ "ahash/compile-time-rng" ]; - "alloc" = [ "dep:alloc" ]; - "bumpalo" = [ "dep:bumpalo" ]; - "compiler_builtins" = [ "dep:compiler_builtins" ]; - "core" = [ "dep:core" ]; - "default" = [ "ahash" "inline-more" ]; - "rayon" = [ "dep:rayon" ]; - "rustc-dep-of-std" = [ "nightly" "core" "compiler_builtins" "alloc" "rustc-internal-api" ]; - "serde" = [ "dep:serde" ]; - }; - resolvedDefaultFeatures = [ "raw" ]; - }; - "hashbrown 0.14.3" = rec { + "hashbrown" = rec { crateName = "hashbrown"; version = "0.14.3"; edition = "2021"; @@ -2917,21 +1678,7 @@ rec { }; resolvedDefaultFeatures = [ "raw" ]; }; - "heck 0.4.1" = rec { - crateName = "heck"; - version = "0.4.1"; - edition = "2018"; - sha256 = "1a7mqsnycv5z4z5vnv1k34548jzmc0ajic7c1j8jsaspnhw5ql4m"; - authors = [ - "Without Boats <woboats@gmail.com>" - ]; - features = { - "unicode" = [ "unicode-segmentation" ]; - "unicode-segmentation" = [ "dep:unicode-segmentation" ]; - }; - resolvedDefaultFeatures = [ "default" ]; - }; - "heck 0.5.0" = rec { + "heck" = rec { crateName = "heck"; version = "0.5.0"; edition = "2021"; @@ -2954,51 +1701,7 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; - "home" = rec { - crateName = "home"; - version = "0.5.9"; - edition = "2021"; - sha256 = "19grxyg35rqfd802pcc9ys1q3lafzlcjcv2pl2s5q8xpyr5kblg3"; - authors = [ - "Brian Anderson <andersrb@gmail.com>" - ]; - dependencies = [ - { - name = "windows-sys"; - packageId = "windows-sys 0.52.0"; - target = { target, features }: (target."windows" or false); - features = [ "Win32_Foundation" "Win32_UI_Shell" "Win32_System_Com" ]; - } - ]; - - }; - "http 0.2.12" = rec { - crateName = "http"; - version = "0.2.12"; - edition = "2018"; - sha256 = "1w81s4bcbmcj9bjp7mllm8jlz6b31wzvirz8bgpzbqkpwmbvn730"; - authors = [ - "Alex Crichton <alex@alexcrichton.com>" - "Carl Lerche <me@carllerche.com>" - "Sean McArthur <sean@seanmonstar.com>" - ]; - dependencies = [ - { - name = "bytes"; - packageId = "bytes"; - } - { - name = "fnv"; - packageId = "fnv"; - } - { - name = "itoa"; - packageId = "itoa"; - } - ]; - - }; - "http 1.1.0" = rec { + "http" = rec { crateName = "http"; version = "1.1.0"; edition = "2018"; @@ -3027,33 +1730,7 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "http-body 0.4.6" = rec { - crateName = "http-body"; - version = "0.4.6"; - edition = "2018"; - sha256 = "1lmyjfk6bqk6k9gkn1dxq770sb78pqbqshga241hr5p995bb5skw"; - authors = [ - "Carl Lerche <me@carllerche.com>" - "Lucio Franco <luciofranco14@gmail.com>" - "Sean McArthur <sean@seanmonstar.com>" - ]; - dependencies = [ - { - name = "bytes"; - packageId = "bytes"; - } - { - name = "http"; - packageId = "http 0.2.12"; - } - { - name = "pin-project-lite"; - packageId = "pin-project-lite"; - } - ]; - - }; - "http-body 1.0.0" = rec { + "http-body" = rec { crateName = "http-body"; version = "1.0.0"; edition = "2018"; @@ -3070,7 +1747,7 @@ rec { } { name = "http"; - packageId = "http 1.1.0"; + packageId = "http"; } ]; @@ -3096,11 +1773,11 @@ rec { } { name = "http"; - packageId = "http 1.1.0"; + packageId = "http"; } { name = "http-body"; - packageId = "http-body 1.0.0"; + packageId = "http-body"; } { name = "pin-project-lite"; @@ -3132,124 +1809,7 @@ rec { ]; }; - "humantime" = rec { - crateName = "humantime"; - version = "2.1.0"; - edition = "2018"; - sha256 = "1r55pfkkf5v0ji1x6izrjwdq9v6sc7bv99xj6srywcar37xmnfls"; - authors = [ - "Paul Colomiets <paul@colomiets.name>" - ]; - - }; - "hyper 0.14.28" = rec { - crateName = "hyper"; - version = "0.14.28"; - edition = "2018"; - sha256 = "107gkvqx4h9bl17d602zkm2dgpfq86l2dr36yzfsi8l3xcsy35mz"; - authors = [ - "Sean McArthur <sean@seanmonstar.com>" - ]; - dependencies = [ - { - name = "bytes"; - packageId = "bytes"; - } - { - name = "futures-channel"; - packageId = "futures-channel"; - } - { - name = "futures-core"; - packageId = "futures-core"; - usesDefaultFeatures = false; - } - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - } - { - name = "h2"; - packageId = "h2 0.3.24"; - optional = true; - } - { - name = "http"; - packageId = "http 0.2.12"; - } - { - name = "http-body"; - packageId = "http-body 0.4.6"; - } - { - name = "httparse"; - packageId = "httparse"; - } - { - name = "httpdate"; - packageId = "httpdate"; - } - { - name = "itoa"; - packageId = "itoa"; - } - { - name = "pin-project-lite"; - packageId = "pin-project-lite"; - } - { - name = "socket2"; - packageId = "socket2"; - optional = true; - features = [ "all" ]; - } - { - name = "tokio"; - packageId = "tokio"; - features = [ "sync" ]; - } - { - name = "tower-service"; - packageId = "tower-service"; - } - { - name = "tracing"; - packageId = "tracing"; - usesDefaultFeatures = false; - features = [ "std" ]; - } - { - name = "want"; - packageId = "want"; - } - ]; - devDependencies = [ - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - features = [ "alloc" ]; - } - { - name = "tokio"; - packageId = "tokio"; - features = [ "fs" "macros" "io-std" "io-util" "rt" "rt-multi-thread" "sync" "time" "test-util" ]; - } - ]; - features = { - "ffi" = [ "libc" ]; - "full" = [ "client" "http1" "http2" "server" "stream" "runtime" ]; - "h2" = [ "dep:h2" ]; - "http2" = [ "h2" ]; - "libc" = [ "dep:libc" ]; - "runtime" = [ "tcp" "tokio/rt" "tokio/time" ]; - "socket2" = [ "dep:socket2" ]; - "tcp" = [ "socket2" "tokio/net" "tokio/rt" "tokio/time" ]; - }; - resolvedDefaultFeatures = [ "client" "default" "full" "h2" "http1" "http2" "runtime" "server" "socket2" "stream" "tcp" ]; - }; - "hyper 1.2.0" = rec { + "hyper" = rec { crateName = "hyper"; version = "1.2.0"; edition = "2021"; @@ -3275,16 +1835,16 @@ rec { } { name = "h2"; - packageId = "h2 0.4.2"; + packageId = "h2"; optional = true; } { name = "http"; - packageId = "http 1.1.0"; + packageId = "http"; } { name = "http-body"; - packageId = "http-body 1.0.0"; + packageId = "http-body"; } { name = "httparse"; @@ -3347,116 +1907,6 @@ rec { }; resolvedDefaultFeatures = [ "default" "http1" "http2" "server" ]; }; - "hyper-rustls" = rec { - crateName = "hyper-rustls"; - version = "0.24.2"; - edition = "2021"; - sha256 = "1475j4a2nczz4aajzzsq3hpwg1zacmzbqg393a14j80ff8izsgpc"; - dependencies = [ - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - } - { - name = "http"; - packageId = "http 0.2.12"; - } - { - name = "hyper"; - packageId = "hyper 0.14.28"; - usesDefaultFeatures = false; - features = [ "client" ]; - } - { - name = "rustls"; - packageId = "rustls 0.21.10"; - usesDefaultFeatures = false; - } - { - name = "tokio"; - packageId = "tokio"; - } - { - name = "tokio-rustls"; - packageId = "tokio-rustls 0.24.1"; - usesDefaultFeatures = false; - } - ]; - devDependencies = [ - { - name = "hyper"; - packageId = "hyper 0.14.28"; - features = [ "full" ]; - } - { - name = "rustls"; - packageId = "rustls 0.21.10"; - usesDefaultFeatures = false; - features = [ "tls12" ]; - } - { - name = "tokio"; - packageId = "tokio"; - features = [ "io-std" "macros" "net" "rt-multi-thread" ]; - } - ]; - features = { - "acceptor" = [ "hyper/server" "tokio-runtime" ]; - "default" = [ "native-tokio" "http1" "tls12" "logging" "acceptor" ]; - "http1" = [ "hyper/http1" ]; - "http2" = [ "hyper/http2" ]; - "log" = [ "dep:log" ]; - "logging" = [ "log" "tokio-rustls/logging" "rustls/logging" ]; - "native-tokio" = [ "tokio-runtime" "rustls-native-certs" ]; - "rustls-native-certs" = [ "dep:rustls-native-certs" ]; - "tls12" = [ "tokio-rustls/tls12" "rustls/tls12" ]; - "tokio-runtime" = [ "hyper/runtime" ]; - "webpki-roots" = [ "dep:webpki-roots" ]; - "webpki-tokio" = [ "tokio-runtime" "webpki-roots" ]; - }; - }; - "hyper-timeout" = rec { - crateName = "hyper-timeout"; - version = "0.4.1"; - edition = "2018"; - sha256 = "1c8k3g8k2yh1gxvsx9p7amkimgxhl9kafwpj7jyf8ywc5r45ifdv"; - authors = [ - "Herman J. Radtke III <herman@hermanradtke.com>" - ]; - dependencies = [ - { - name = "hyper"; - packageId = "hyper 0.14.28"; - features = [ "client" ]; - } - { - name = "pin-project-lite"; - packageId = "pin-project-lite"; - } - { - name = "tokio"; - packageId = "tokio"; - } - { - name = "tokio-io-timeout"; - packageId = "tokio-io-timeout"; - } - ]; - devDependencies = [ - { - name = "hyper"; - packageId = "hyper 0.14.28"; - features = [ "client" "http1" "tcp" ]; - } - { - name = "tokio"; - packageId = "tokio"; - features = [ "io-std" "io-util" "macros" ]; - } - ]; - - }; "hyper-util" = rec { crateName = "hyper-util"; version = "0.1.3"; @@ -3477,15 +1927,15 @@ rec { } { name = "http"; - packageId = "http 1.1.0"; + packageId = "http"; } { name = "http-body"; - packageId = "http-body 1.0.0"; + packageId = "http-body"; } { name = "hyper"; - packageId = "hyper 1.2.0"; + packageId = "hyper"; } { name = "pin-project-lite"; @@ -3511,7 +1961,7 @@ rec { } { name = "hyper"; - packageId = "hyper 1.2.0"; + packageId = "hyper"; features = [ "full" ]; } { @@ -3533,123 +1983,7 @@ rec { }; resolvedDefaultFeatures = [ "default" "http1" "http2" "server" "server-auto" "tokio" ]; }; - "iana-time-zone" = rec { - crateName = "iana-time-zone"; - version = "0.1.60"; - edition = "2018"; - sha256 = "0hdid5xz3jznm04lysjm3vi93h3c523w0hcc3xba47jl3ddbpzz7"; - authors = [ - "Andrew Straw <strawman@astraw.com>" - "Renรฉ Kijewski <rene.kijewski@fu-berlin.de>" - "Ryan Lopopolo <rjl@hyperbo.la>" - ]; - dependencies = [ - { - name = "android_system_properties"; - packageId = "android_system_properties"; - target = { target, features }: ("android" == target."os" or null); - } - { - name = "core-foundation-sys"; - packageId = "core-foundation-sys"; - target = { target, features }: (("macos" == target."os" or null) || ("ios" == target."os" or null)); - } - { - name = "iana-time-zone-haiku"; - packageId = "iana-time-zone-haiku"; - target = { target, features }: ("haiku" == target."os" or null); - } - { - name = "js-sys"; - packageId = "js-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); - } - { - name = "wasm-bindgen"; - packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); - } - { - name = "windows-core"; - packageId = "windows-core"; - target = { target, features }: ("windows" == target."os" or null); - } - ]; - features = { }; - resolvedDefaultFeatures = [ "fallback" ]; - }; - "iana-time-zone-haiku" = rec { - crateName = "iana-time-zone-haiku"; - version = "0.1.2"; - edition = "2018"; - sha256 = "17r6jmj31chn7xs9698r122mapq85mfnv98bb4pg6spm0si2f67k"; - authors = [ - "Renรฉ Kijewski <crates.io@k6i.de>" - ]; - buildDependencies = [ - { - name = "cc"; - packageId = "cc"; - } - ]; - - }; - "idna" = rec { - crateName = "idna"; - version = "0.5.0"; - edition = "2018"; - sha256 = "1xhjrcjqq0l5bpzvdgylvpkgk94panxgsirzhjnnqfdgc4a9nkb3"; - authors = [ - "The rust-url developers" - ]; - dependencies = [ - { - name = "unicode-bidi"; - packageId = "unicode-bidi"; - usesDefaultFeatures = false; - features = [ "hardcoded-data" ]; - } - { - name = "unicode-normalization"; - packageId = "unicode-normalization"; - usesDefaultFeatures = false; - } - ]; - features = { - "default" = [ "std" ]; - "std" = [ "alloc" "unicode-bidi/std" "unicode-normalization/std" ]; - }; - resolvedDefaultFeatures = [ "alloc" "default" "std" ]; - }; - "indexmap 1.9.3" = rec { - crateName = "indexmap"; - version = "1.9.3"; - edition = "2021"; - sha256 = "16dxmy7yvk51wvnih3a3im6fp5lmx0wx76i03n06wyak6cwhw1xx"; - dependencies = [ - { - name = "hashbrown"; - packageId = "hashbrown 0.12.3"; - usesDefaultFeatures = false; - features = [ "raw" ]; - } - ]; - buildDependencies = [ - { - name = "autocfg"; - packageId = "autocfg"; - } - ]; - features = { - "arbitrary" = [ "dep:arbitrary" ]; - "quickcheck" = [ "dep:quickcheck" ]; - "rayon" = [ "dep:rayon" ]; - "rustc-rayon" = [ "dep:rustc-rayon" ]; - "serde" = [ "dep:serde" ]; - "serde-1" = [ "serde" ]; - }; - }; - "indexmap 2.2.5" = rec { + "indexmap" = rec { crateName = "indexmap"; version = "2.2.5"; edition = "2021"; @@ -3662,7 +1996,7 @@ rec { } { name = "hashbrown"; - packageId = "hashbrown 0.14.3"; + packageId = "hashbrown"; usesDefaultFeatures = false; features = [ "raw" ]; } @@ -3678,109 +2012,6 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "instant" = rec { - crateName = "instant"; - version = "0.1.12"; - edition = "2018"; - sha256 = "0b2bx5qdlwayriidhrag8vhy10kdfimfhmb3jnjmsz2h9j1bwnvs"; - authors = [ - "sebcrozet <developer@crozet.re>" - ]; - dependencies = [ - { - name = "cfg-if"; - packageId = "cfg-if"; - } - ]; - features = { - "js-sys" = [ "dep:js-sys" ]; - "stdweb" = [ "dep:stdweb" ]; - "wasm-bindgen" = [ "js-sys" "wasm-bindgen_rs" "web-sys" ]; - "wasm-bindgen_rs" = [ "dep:wasm-bindgen_rs" ]; - "web-sys" = [ "dep:web-sys" ]; - }; - }; - "ipnet" = rec { - crateName = "ipnet"; - version = "2.9.0"; - edition = "2018"; - sha256 = "1hzrcysgwf0knf83ahb3535hrkw63mil88iqc6kjaryfblrqylcg"; - authors = [ - "Kris Price <kris@krisprice.nz>" - ]; - features = { - "default" = [ "std" ]; - "heapless" = [ "dep:heapless" ]; - "json" = [ "serde" "schemars" ]; - "schemars" = [ "dep:schemars" ]; - "ser_as_str" = [ "heapless" ]; - "serde" = [ "dep:serde" ]; - }; - resolvedDefaultFeatures = [ "default" "std" ]; - }; - "itertools 0.10.5" = rec { - crateName = "itertools"; - version = "0.10.5"; - edition = "2018"; - sha256 = "0ww45h7nxx5kj6z2y6chlskxd1igvs4j507anr6dzg99x1h25zdh"; - authors = [ - "bluss" - ]; - dependencies = [ - { - name = "either"; - packageId = "either"; - usesDefaultFeatures = false; - } - ]; - features = { - "default" = [ "use_std" ]; - "use_std" = [ "use_alloc" "either/use_std" ]; - }; - resolvedDefaultFeatures = [ "use_alloc" ]; - }; - "itertools 0.11.0" = rec { - crateName = "itertools"; - version = "0.11.0"; - edition = "2018"; - sha256 = "0mzyqcc59azx9g5cg6fs8k529gvh4463smmka6jvzs3cd2jp7hdi"; - authors = [ - "bluss" - ]; - dependencies = [ - { - name = "either"; - packageId = "either"; - usesDefaultFeatures = false; - } - ]; - features = { - "default" = [ "use_std" ]; - "use_std" = [ "use_alloc" "either/use_std" ]; - }; - resolvedDefaultFeatures = [ "use_alloc" ]; - }; - "itertools 0.12.1" = rec { - crateName = "itertools"; - version = "0.12.1"; - edition = "2018"; - sha256 = "0s95jbb3ndj1lvfxyq5wanc0fm0r6hg6q4ngb92qlfdxvci10ads"; - authors = [ - "bluss" - ]; - dependencies = [ - { - name = "either"; - packageId = "either"; - usesDefaultFeatures = false; - } - ]; - features = { - "default" = [ "use_std" ]; - "use_std" = [ "use_alloc" "either/use_std" ]; - }; - resolvedDefaultFeatures = [ "default" "use_alloc" "use_std" ]; - }; "itoa" = rec { crateName = "itoa"; version = "1.0.10"; @@ -3793,39 +2024,6 @@ rec { "no-panic" = [ "dep:no-panic" ]; }; }; - "jobserver" = rec { - crateName = "jobserver"; - version = "0.1.28"; - edition = "2021"; - sha256 = "1mji1wis4w76v3issgpah2x3j1k0ybq0cz3qgypg7pkdablscimb"; - authors = [ - "Alex Crichton <alex@alexcrichton.com>" - ]; - dependencies = [ - { - name = "libc"; - packageId = "libc"; - target = { target, features }: (target."unix" or false); - } - ]; - - }; - "js-sys" = rec { - crateName = "js-sys"; - version = "0.3.69"; - edition = "2018"; - sha256 = "0v99rz97asnzapb0jsc3jjhvxpfxr7h7qd97yqyrf9i7viimbh99"; - authors = [ - "The wasm-bindgen Developers" - ]; - dependencies = [ - { - name = "wasm-bindgen"; - packageId = "wasm-bindgen"; - } - ]; - - }; "lazy_static" = rec { crateName = "lazy_static"; version = "1.4.0"; @@ -3855,22 +2053,6 @@ rec { }; resolvedDefaultFeatures = [ "default" "extra_traits" "std" ]; }; - "linux-raw-sys" = rec { - crateName = "linux-raw-sys"; - version = "0.4.13"; - edition = "2021"; - sha256 = "172k2c6422gsc914ig8rh99mb9yc7siw6ikc3d9xw1k7vx0s3k81"; - authors = [ - "Dan Gohman <dev@sunfishcode.online>" - ]; - features = { - "compiler_builtins" = [ "dep:compiler_builtins" ]; - "core" = [ "dep:core" ]; - "default" = [ "std" "general" "errno" ]; - "rustc-dep-of-std" = [ "core" "compiler_builtins" "no_std" ]; - }; - resolvedDefaultFeatures = [ "elf" "errno" "general" "ioctl" "no_std" ]; - }; "litrs" = rec { crateName = "litrs"; version = "0.4.1"; @@ -3937,49 +2119,6 @@ rec { }; resolvedDefaultFeatures = [ "std" ]; }; - "lzma-sys" = rec { - crateName = "lzma-sys"; - version = "0.1.20"; - edition = "2018"; - links = "lzma"; - sha256 = "09sxp20waxyglgn3cjz8qjkspb3ryz2fwx4rigkwvrk46ymh9njz"; - authors = [ - "Alex Crichton <alex@alexcrichton.com>" - ]; - dependencies = [ - { - name = "libc"; - packageId = "libc"; - } - ]; - buildDependencies = [ - { - name = "cc"; - packageId = "cc"; - } - { - name = "pkg-config"; - packageId = "pkg-config"; - } - ]; - features = { }; - }; - "matchers" = rec { - crateName = "matchers"; - version = "0.1.0"; - edition = "2018"; - sha256 = "0n2mbk7lg2vf962c8xwzdq96yrc9i0p8dbmm4wa1nnkcp1dhfqw2"; - authors = [ - "Eliza Weisman <eliza@buoyant.io>" - ]; - dependencies = [ - { - name = "regex-automata"; - packageId = "regex-automata 0.1.10"; - } - ]; - - }; "matchit" = rec { crateName = "matchit"; version = "0.7.3"; @@ -3991,40 +2130,6 @@ rec { features = { }; resolvedDefaultFeatures = [ "default" ]; }; - "md-5" = rec { - crateName = "md-5"; - version = "0.10.6"; - edition = "2018"; - sha256 = "1kvq5rnpm4fzwmyv5nmnxygdhhb2369888a06gdc9pxyrzh7x7nq"; - libName = "md5"; - authors = [ - "RustCrypto Developers" - ]; - dependencies = [ - { - name = "cfg-if"; - packageId = "cfg-if"; - } - { - name = "digest"; - packageId = "digest"; - } - ]; - devDependencies = [ - { - name = "digest"; - packageId = "digest"; - features = [ "dev" ]; - } - ]; - features = { - "asm" = [ "md5-asm" ]; - "default" = [ "std" ]; - "md5-asm" = [ "dep:md5-asm" ]; - "oid" = [ "digest/oid" ]; - "std" = [ "digest/std" ]; - }; - }; "memchr" = rec { crateName = "memchr"; version = "2.7.1"; @@ -4045,23 +2150,6 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "std" ]; }; - "memoffset" = rec { - crateName = "memoffset"; - version = "0.6.5"; - edition = "2015"; - sha256 = "1kkrzll58a3ayn5zdyy9i1f1v3mx0xgl29x0chq614zazba638ss"; - authors = [ - "Gilad Naaman <gilad.naaman@gmail.com>" - ]; - buildDependencies = [ - { - name = "autocfg"; - packageId = "autocfg"; - } - ]; - features = { }; - resolvedDefaultFeatures = [ "default" ]; - }; "mime" = rec { crateName = "mime"; version = "0.3.17"; @@ -4133,11 +2221,6 @@ rec { target = { target, features }: (target."unix" or false); } { - name = "log"; - packageId = "log"; - optional = true; - } - { name = "wasi"; packageId = "wasi"; target = { target, features }: ("wasi" == target."os" or null); @@ -4154,69 +2237,9 @@ rec { "log" = [ "dep:log" ]; "os-ext" = [ "os-poll" "windows-sys/Win32_System_Pipes" "windows-sys/Win32_Security" ]; }; - resolvedDefaultFeatures = [ "default" "log" "net" "os-ext" "os-poll" ]; - }; - "multimap" = rec { - crateName = "multimap"; - version = "0.8.3"; - edition = "2015"; - sha256 = "0sicyz4n500vdhgcxn4g8jz97cp1ijir1rnbgph3pmx9ckz4dkp5"; - authors = [ - "Hรฅvar Nรธvik <havar.novik@gmail.com>" - ]; - features = { - "default" = [ "serde_impl" ]; - "serde" = [ "dep:serde" ]; - "serde_impl" = [ "serde" ]; - }; - }; - "nix 0.24.3" = rec { - crateName = "nix"; - version = "0.24.3"; - edition = "2018"; - sha256 = "0sc0yzdl51b49bqd9l9cmimp1sw1hxb8iyv4d35ww6d7m5rfjlps"; - authors = [ - "The nix-rust Project Developers" - ]; - dependencies = [ - { - name = "bitflags"; - packageId = "bitflags 1.3.2"; - } - { - name = "cfg-if"; - packageId = "cfg-if"; - } - { - name = "libc"; - packageId = "libc"; - features = [ "extra_traits" ]; - } - { - name = "memoffset"; - packageId = "memoffset"; - optional = true; - target = { target, features }: (!("redox" == target."os" or null)); - } - ]; - features = { - "default" = [ "acct" "aio" "dir" "env" "event" "feature" "fs" "hostname" "inotify" "ioctl" "kmod" "mman" "mount" "mqueue" "net" "personality" "poll" "process" "pthread" "ptrace" "quota" "reboot" "resource" "sched" "signal" "socket" "term" "time" "ucontext" "uio" "user" "zerocopy" ]; - "dir" = [ "fs" ]; - "memoffset" = [ "dep:memoffset" ]; - "mount" = [ "uio" ]; - "mqueue" = [ "fs" ]; - "net" = [ "socket" ]; - "ptrace" = [ "process" ]; - "sched" = [ "process" ]; - "signal" = [ "process" ]; - "socket" = [ "memoffset" ]; - "ucontext" = [ "signal" ]; - "user" = [ "feature" ]; - "zerocopy" = [ "fs" "uio" ]; - }; - resolvedDefaultFeatures = [ "acct" "aio" "default" "dir" "env" "event" "feature" "fs" "hostname" "inotify" "ioctl" "kmod" "memoffset" "mman" "mount" "mqueue" "net" "personality" "poll" "process" "pthread" "ptrace" "quota" "reboot" "resource" "sched" "signal" "socket" "term" "time" "ucontext" "uio" "user" "zerocopy" ]; + resolvedDefaultFeatures = [ "net" "os-ext" "os-poll" ]; }; - "nix 0.26.4" = rec { + "nix" = rec { crateName = "nix"; version = "0.26.4"; edition = "2018"; @@ -4296,12 +2319,6 @@ rec { packageId = "enum-primitive-derive"; } { - name = "futures-util"; - packageId = "futures-util"; - optional = true; - features = [ "io" ]; - } - { name = "glob"; packageId = "glob"; } @@ -4314,6 +2331,11 @@ rec { packageId = "num-traits"; } { + name = "pin-project-lite"; + packageId = "pin-project-lite"; + optional = true; + } + { name = "serde"; packageId = "serde"; features = [ "derive" ]; @@ -4344,11 +2366,12 @@ rec { } ]; features = { - "async" = [ "futures-util" "tokio" ]; - "futures-util" = [ "dep:futures-util" ]; + "async" = [ "tokio" ]; + "pin-project-lite" = [ "dep:pin-project-lite" ]; "tokio" = [ "dep:tokio" ]; + "wire" = [ "tokio" "pin-project-lite" ]; }; - resolvedDefaultFeatures = [ "async" "futures-util" "tokio" ]; + resolvedDefaultFeatures = [ "pin-project-lite" "tokio" "wire" ]; }; "nom" = rec { crateName = "nom"; @@ -4478,165 +2501,6 @@ rec { }; resolvedDefaultFeatures = [ "archive" "coff" "elf" "macho" "pe" "read_core" "unaligned" ]; }; - "object_store" = rec { - crateName = "object_store"; - version = "0.9.1"; - edition = "2021"; - sha256 = "1cwx0xg57cp3z6xjgrqwp0gxgxsagls4h5cd212pmxpxcn5qywdq"; - dependencies = [ - { - name = "async-trait"; - packageId = "async-trait"; - } - { - name = "base64"; - packageId = "base64"; - optional = true; - usesDefaultFeatures = false; - features = [ "std" ]; - } - { - name = "bytes"; - packageId = "bytes"; - } - { - name = "chrono"; - packageId = "chrono"; - usesDefaultFeatures = false; - features = [ "clock" ]; - } - { - name = "futures"; - packageId = "futures"; - } - { - name = "humantime"; - packageId = "humantime"; - } - { - name = "hyper"; - packageId = "hyper 0.14.28"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "itertools"; - packageId = "itertools 0.12.1"; - } - { - name = "md-5"; - packageId = "md-5"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "parking_lot"; - packageId = "parking_lot 0.12.1"; - } - { - name = "percent-encoding"; - packageId = "percent-encoding"; - } - { - name = "quick-xml"; - packageId = "quick-xml"; - optional = true; - features = [ "serialize" "overlapped-lists" ]; - } - { - name = "rand"; - packageId = "rand"; - optional = true; - usesDefaultFeatures = false; - features = [ "std" "std_rng" ]; - } - { - name = "reqwest"; - packageId = "reqwest"; - optional = true; - usesDefaultFeatures = false; - features = [ "rustls-tls-native-roots" ]; - } - { - name = "ring"; - packageId = "ring"; - optional = true; - usesDefaultFeatures = false; - features = [ "std" ]; - } - { - name = "rustls-pemfile"; - packageId = "rustls-pemfile 2.1.1"; - optional = true; - usesDefaultFeatures = false; - features = [ "std" ]; - } - { - name = "serde"; - packageId = "serde"; - optional = true; - usesDefaultFeatures = false; - features = [ "derive" ]; - } - { - name = "serde_json"; - packageId = "serde_json"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "snafu"; - packageId = "snafu"; - } - { - name = "tokio"; - packageId = "tokio"; - features = [ "sync" "macros" "rt" "time" "io-util" ]; - } - { - name = "tracing"; - packageId = "tracing"; - } - { - name = "url"; - packageId = "url"; - } - { - name = "walkdir"; - packageId = "walkdir"; - } - ]; - devDependencies = [ - { - name = "hyper"; - packageId = "hyper 0.14.28"; - features = [ "server" ]; - } - { - name = "rand"; - packageId = "rand"; - } - ]; - features = { - "aws" = [ "cloud" "md-5" ]; - "azure" = [ "cloud" ]; - "base64" = [ "dep:base64" ]; - "cloud" = [ "serde" "serde_json" "quick-xml" "hyper" "reqwest" "reqwest/json" "reqwest/stream" "chrono/serde" "base64" "rand" "ring" ]; - "gcp" = [ "cloud" "rustls-pemfile" ]; - "http" = [ "cloud" ]; - "hyper" = [ "dep:hyper" ]; - "md-5" = [ "dep:md-5" ]; - "quick-xml" = [ "dep:quick-xml" ]; - "rand" = [ "dep:rand" ]; - "reqwest" = [ "dep:reqwest" ]; - "ring" = [ "dep:ring" ]; - "rustls-pemfile" = [ "dep:rustls-pemfile" ]; - "serde" = [ "dep:serde" ]; - "serde_json" = [ "dep:serde_json" ]; - "tls-webpki-roots" = [ "reqwest?/rustls-tls-webpki-roots" ]; - }; - resolvedDefaultFeatures = [ "aws" "azure" "base64" "cloud" "gcp" "http" "hyper" "md-5" "quick-xml" "rand" "reqwest" "ring" "rustls-pemfile" "serde" "serde_json" ]; - }; "once_cell" = rec { crateName = "once_cell"; version = "1.19.0"; @@ -4656,362 +2520,6 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "race" "std" ]; }; - "openssl-probe" = rec { - crateName = "openssl-probe"; - version = "0.1.5"; - edition = "2015"; - sha256 = "1kq18qm48rvkwgcggfkqq6pm948190czqc94d6bm2sir5hq1l0gz"; - authors = [ - "Alex Crichton <alex@alexcrichton.com>" - ]; - - }; - "opentelemetry" = rec { - crateName = "opentelemetry"; - version = "0.21.0"; - edition = "2021"; - sha256 = "12jfmyx8k9q2sjlx4wp76ddzaf94i7lnkliv1c9mj164bnd36chy"; - dependencies = [ - { - name = "futures-core"; - packageId = "futures-core"; - } - { - name = "futures-sink"; - packageId = "futures-sink"; - } - { - name = "indexmap"; - packageId = "indexmap 2.2.5"; - } - { - name = "js-sys"; - packageId = "js-sys"; - target = { target, features }: (("wasm32" == target."arch" or null) && (!("wasi" == target."os" or null))); - } - { - name = "once_cell"; - packageId = "once_cell"; - } - { - name = "pin-project-lite"; - packageId = "pin-project-lite"; - optional = true; - } - { - name = "thiserror"; - packageId = "thiserror"; - } - { - name = "urlencoding"; - packageId = "urlencoding"; - } - ]; - features = { - "default" = [ "trace" ]; - "logs_level_enabled" = [ "logs" ]; - "pin-project-lite" = [ "dep:pin-project-lite" ]; - "testing" = [ "trace" "metrics" ]; - "trace" = [ "pin-project-lite" ]; - }; - resolvedDefaultFeatures = [ "default" "metrics" "pin-project-lite" "trace" ]; - }; - "opentelemetry-otlp" = rec { - crateName = "opentelemetry-otlp"; - version = "0.14.0"; - edition = "2021"; - sha256 = "0c59bh4wa824mf89ayivsjqwipkg1y6r27r4d0y47lhfna1xlk7j"; - dependencies = [ - { - name = "async-trait"; - packageId = "async-trait"; - } - { - name = "futures-core"; - packageId = "futures-core"; - } - { - name = "http"; - packageId = "http 0.2.12"; - optional = true; - } - { - name = "opentelemetry"; - packageId = "opentelemetry"; - usesDefaultFeatures = false; - } - { - name = "opentelemetry-proto"; - packageId = "opentelemetry-proto"; - usesDefaultFeatures = false; - } - { - name = "opentelemetry-semantic-conventions"; - packageId = "opentelemetry-semantic-conventions"; - } - { - name = "opentelemetry_sdk"; - packageId = "opentelemetry_sdk"; - usesDefaultFeatures = false; - } - { - name = "prost"; - packageId = "prost 0.11.9"; - optional = true; - } - { - name = "thiserror"; - packageId = "thiserror"; - } - { - name = "tokio"; - packageId = "tokio"; - optional = true; - features = [ "sync" "rt" ]; - } - { - name = "tonic"; - packageId = "tonic 0.9.2"; - optional = true; - } - ]; - devDependencies = [ - { - name = "tokio"; - packageId = "tokio"; - features = [ "macros" "rt-multi-thread" ]; - } - ]; - features = { - "default" = [ "grpc-tonic" "trace" ]; - "grpc-sys" = [ "grpcio" "opentelemetry-proto/gen-grpcio" ]; - "grpc-tonic" = [ "tonic" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; - "grpcio" = [ "dep:grpcio" ]; - "gzip-tonic" = [ "tonic/gzip" ]; - "http" = [ "dep:http" ]; - "http-proto" = [ "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "http" "trace" "metrics" ]; - "integration-testing" = [ "tonic" "prost" "tokio/full" "trace" ]; - "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" "opentelemetry-proto/logs" ]; - "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" "opentelemetry-proto/metrics" ]; - "openssl" = [ "grpcio/openssl" ]; - "openssl-vendored" = [ "grpcio/openssl-vendored" ]; - "opentelemetry-http" = [ "dep:opentelemetry-http" ]; - "prost" = [ "dep:prost" ]; - "reqwest" = [ "dep:reqwest" ]; - "reqwest-blocking-client" = [ "reqwest/blocking" "opentelemetry-http/reqwest" ]; - "reqwest-client" = [ "reqwest" "opentelemetry-http/reqwest" ]; - "reqwest-rustls" = [ "reqwest" "reqwest/rustls-tls-native-roots" ]; - "serde" = [ "dep:serde" ]; - "serialize" = [ "serde" ]; - "surf" = [ "dep:surf" ]; - "surf-client" = [ "surf" "opentelemetry-http/surf" ]; - "tls" = [ "tonic/tls" ]; - "tls-roots" = [ "tls" "tonic/tls-roots" ]; - "tokio" = [ "dep:tokio" ]; - "tonic" = [ "dep:tonic" ]; - "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" "opentelemetry-proto/trace" ]; - }; - resolvedDefaultFeatures = [ "default" "grpc-tonic" "http" "prost" "tokio" "tonic" "trace" ]; - }; - "opentelemetry-proto" = rec { - crateName = "opentelemetry-proto"; - version = "0.4.0"; - edition = "2021"; - sha256 = "1qblsq0hkksdw3k60bc8yi5xwlynmqwibggz3lyyl4n8bk75bqd2"; - dependencies = [ - { - name = "opentelemetry"; - packageId = "opentelemetry"; - usesDefaultFeatures = false; - } - { - name = "opentelemetry_sdk"; - packageId = "opentelemetry_sdk"; - usesDefaultFeatures = false; - } - { - name = "prost"; - packageId = "prost 0.11.9"; - optional = true; - } - { - name = "tonic"; - packageId = "tonic 0.9.2"; - optional = true; - usesDefaultFeatures = false; - features = [ "codegen" "prost" ]; - } - ]; - features = { - "full" = [ "gen-tonic" "gen-grpcio" "trace" "logs" "metrics" "zpages" "with-serde" ]; - "gen-grpcio" = [ "grpcio" "prost" ]; - "gen-tonic" = [ "gen-tonic-messages" "tonic/transport" ]; - "gen-tonic-messages" = [ "tonic" "prost" ]; - "grpcio" = [ "dep:grpcio" ]; - "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" ]; - "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" ]; - "prost" = [ "dep:prost" ]; - "serde" = [ "dep:serde" ]; - "tonic" = [ "dep:tonic" ]; - "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" ]; - "with-serde" = [ "serde" ]; - "zpages" = [ "trace" ]; - }; - resolvedDefaultFeatures = [ "gen-tonic" "gen-tonic-messages" "prost" "tonic" "trace" ]; - }; - "opentelemetry-semantic-conventions" = rec { - crateName = "opentelemetry-semantic-conventions"; - version = "0.13.0"; - edition = "2021"; - sha256 = "115wbgk840dklyhpg3lwp4x1m643qd7f0vkz8hmfz0pry4g4yxzm"; - dependencies = [ - { - name = "opentelemetry"; - packageId = "opentelemetry"; - usesDefaultFeatures = false; - } - ]; - - }; - "opentelemetry_sdk" = rec { - crateName = "opentelemetry_sdk"; - version = "0.21.2"; - edition = "2021"; - sha256 = "1r7gw2j2n800rd0vdnga32yhlfmc3c4y0sadcr97licam74aw5ig"; - dependencies = [ - { - name = "async-trait"; - packageId = "async-trait"; - optional = true; - } - { - name = "crossbeam-channel"; - packageId = "crossbeam-channel"; - optional = true; - } - { - name = "futures-channel"; - packageId = "futures-channel"; - } - { - name = "futures-executor"; - packageId = "futures-executor"; - } - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - features = [ "std" "sink" "async-await-macro" ]; - } - { - name = "glob"; - packageId = "glob"; - optional = true; - } - { - name = "once_cell"; - packageId = "once_cell"; - } - { - name = "opentelemetry"; - packageId = "opentelemetry"; - } - { - name = "ordered-float"; - packageId = "ordered-float"; - } - { - name = "percent-encoding"; - packageId = "percent-encoding"; - optional = true; - } - { - name = "rand"; - packageId = "rand"; - optional = true; - usesDefaultFeatures = false; - features = [ "std" "std_rng" ]; - } - { - name = "thiserror"; - packageId = "thiserror"; - } - { - name = "tokio"; - packageId = "tokio"; - optional = true; - usesDefaultFeatures = false; - features = [ "rt" "time" ]; - } - { - name = "tokio-stream"; - packageId = "tokio-stream"; - optional = true; - } - ]; - features = { - "async-std" = [ "dep:async-std" ]; - "async-trait" = [ "dep:async-trait" ]; - "crossbeam-channel" = [ "dep:crossbeam-channel" ]; - "default" = [ "trace" ]; - "glob" = [ "dep:glob" ]; - "http" = [ "dep:http" ]; - "jaeger_remote_sampler" = [ "trace" "opentelemetry-http" "http" "serde" "serde_json" "url" ]; - "logs" = [ "opentelemetry/logs" "crossbeam-channel" "async-trait" "serde_json" ]; - "logs_level_enabled" = [ "logs" "opentelemetry/logs_level_enabled" ]; - "metrics" = [ "opentelemetry/metrics" "glob" "async-trait" ]; - "opentelemetry-http" = [ "dep:opentelemetry-http" ]; - "percent-encoding" = [ "dep:percent-encoding" ]; - "rand" = [ "dep:rand" ]; - "rt-async-std" = [ "async-std" ]; - "rt-tokio" = [ "tokio" "tokio-stream" ]; - "rt-tokio-current-thread" = [ "tokio" "tokio-stream" ]; - "serde" = [ "dep:serde" ]; - "serde_json" = [ "dep:serde_json" ]; - "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "rt-async-std" "rt-tokio" "rt-tokio-current-thread" "tokio/macros" "tokio/rt-multi-thread" ]; - "tokio" = [ "dep:tokio" ]; - "tokio-stream" = [ "dep:tokio-stream" ]; - "trace" = [ "opentelemetry/trace" "crossbeam-channel" "rand" "async-trait" "percent-encoding" ]; - "url" = [ "dep:url" ]; - }; - resolvedDefaultFeatures = [ "async-trait" "crossbeam-channel" "default" "glob" "metrics" "percent-encoding" "rand" "rt-tokio" "tokio" "tokio-stream" "trace" ]; - }; - "ordered-float" = rec { - crateName = "ordered-float"; - version = "4.2.0"; - edition = "2021"; - sha256 = "0kjqcvvbcsibbx3hnj7ag06bd9gv2zfi5ja6rgyh2kbxbh3zfvd7"; - authors = [ - "Jonathan Reem <jonathan.reem@gmail.com>" - "Matt Brubeck <mbrubeck@limpet.net>" - ]; - dependencies = [ - { - name = "num-traits"; - packageId = "num-traits"; - usesDefaultFeatures = false; - } - ]; - features = { - "arbitrary" = [ "dep:arbitrary" ]; - "borsh" = [ "dep:borsh" ]; - "bytemuck" = [ "dep:bytemuck" ]; - "default" = [ "std" ]; - "proptest" = [ "dep:proptest" ]; - "rand" = [ "dep:rand" ]; - "randtest" = [ "rand/std" "rand/std_rng" ]; - "rkyv" = [ "rkyv_32" ]; - "rkyv_16" = [ "dep:rkyv" "rkyv?/size_16" ]; - "rkyv_32" = [ "dep:rkyv" "rkyv?/size_32" ]; - "rkyv_64" = [ "dep:rkyv" "rkyv?/size_64" ]; - "rkyv_ck" = [ "rkyv?/validation" ]; - "schemars" = [ "dep:schemars" ]; - "serde" = [ "dep:serde" "rand?/serde1" ]; - "speedy" = [ "dep:speedy" ]; - "std" = [ "num-traits/std" ]; - }; - resolvedDefaultFeatures = [ "default" "std" ]; - }; "overload" = rec { crateName = "overload"; version = "0.1.1"; @@ -5022,40 +2530,7 @@ rec { ]; }; - "parking_lot 0.11.2" = rec { - crateName = "parking_lot"; - version = "0.11.2"; - edition = "2018"; - sha256 = "16gzf41bxmm10x82bla8d6wfppy9ym3fxsmdjyvn61m66s0bf5vx"; - authors = [ - "Amanieu d'Antras <amanieu@gmail.com>" - ]; - dependencies = [ - { - name = "instant"; - packageId = "instant"; - } - { - name = "lock_api"; - packageId = "lock_api"; - } - { - name = "parking_lot_core"; - packageId = "parking_lot_core 0.8.6"; - } - ]; - features = { - "arc_lock" = [ "lock_api/arc_lock" ]; - "deadlock_detection" = [ "parking_lot_core/deadlock_detection" ]; - "nightly" = [ "parking_lot_core/nightly" "lock_api/nightly" ]; - "owning_ref" = [ "lock_api/owning_ref" ]; - "serde" = [ "lock_api/serde" ]; - "stdweb" = [ "instant/stdweb" ]; - "wasm-bindgen" = [ "instant/wasm-bindgen" ]; - }; - resolvedDefaultFeatures = [ "default" ]; - }; - "parking_lot 0.12.1" = rec { + "parking_lot" = rec { crateName = "parking_lot"; version = "0.12.1"; edition = "2018"; @@ -5070,7 +2545,7 @@ rec { } { name = "parking_lot_core"; - packageId = "parking_lot_core 0.9.9"; + packageId = "parking_lot_core"; } ]; features = { @@ -5082,52 +2557,7 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; - "parking_lot_core 0.8.6" = rec { - crateName = "parking_lot_core"; - version = "0.8.6"; - edition = "2018"; - sha256 = "1p2nfcbr0b9lm9rglgm28k6mwyjwgm4knipsmqbgqaxdy3kcz8k0"; - authors = [ - "Amanieu d'Antras <amanieu@gmail.com>" - ]; - dependencies = [ - { - name = "cfg-if"; - packageId = "cfg-if"; - } - { - name = "instant"; - packageId = "instant"; - } - { - name = "libc"; - packageId = "libc"; - target = { target, features }: (target."unix" or false); - } - { - name = "redox_syscall"; - packageId = "redox_syscall 0.2.16"; - target = { target, features }: ("redox" == target."os" or null); - } - { - name = "smallvec"; - packageId = "smallvec"; - } - { - name = "winapi"; - packageId = "winapi"; - target = { target, features }: (target."windows" or false); - features = [ "winnt" "ntstatus" "minwindef" "winerror" "winbase" "errhandlingapi" "handleapi" ]; - } - ]; - features = { - "backtrace" = [ "dep:backtrace" ]; - "deadlock_detection" = [ "petgraph" "thread-id" "backtrace" ]; - "petgraph" = [ "dep:petgraph" ]; - "thread-id" = [ "dep:thread-id" ]; - }; - }; - "parking_lot_core 0.9.9" = rec { + "parking_lot_core" = rec { crateName = "parking_lot_core"; version = "0.9.9"; edition = "2018"; @@ -5147,7 +2577,7 @@ rec { } { name = "redox_syscall"; - packageId = "redox_syscall 0.4.1"; + packageId = "redox_syscall"; target = { target, features }: ("redox" == target."os" or null); } { @@ -5181,36 +2611,6 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "std" ]; }; - "petgraph" = rec { - crateName = "petgraph"; - version = "0.6.4"; - edition = "2018"; - sha256 = "1ac6wfq5f5pzcv0nvzzfgjbwg2kwslpnzsw5wcmxlscfcb9azlz1"; - authors = [ - "bluss" - "mitchmindtree" - ]; - dependencies = [ - { - name = "fixedbitset"; - packageId = "fixedbitset"; - usesDefaultFeatures = false; - } - { - name = "indexmap"; - packageId = "indexmap 2.2.5"; - } - ]; - features = { - "all" = [ "unstable" "quickcheck" "matrix_graph" "stable_graph" "graphmap" ]; - "default" = [ "graphmap" "stable_graph" "matrix_graph" ]; - "quickcheck" = [ "dep:quickcheck" ]; - "serde" = [ "dep:serde" ]; - "serde-1" = [ "serde" "serde_derive" ]; - "serde_derive" = [ "dep:serde_derive" ]; - "unstable" = [ "generate" ]; - }; - }; "pin-project" = rec { crateName = "pin-project"; version = "1.1.5"; @@ -5241,7 +2641,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.52"; + packageId = "syn"; features = [ "full" "visit-mut" ]; } ]; @@ -5298,16 +2698,6 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "std" ]; }; - "pkg-config" = rec { - crateName = "pkg-config"; - version = "0.3.30"; - edition = "2015"; - sha256 = "1v07557dj1sa0aly9c90wsygc0i8xv5vnmyv0g94lpkvj8qb4cfj"; - authors = [ - "Alex Crichton <alex@alexcrichton.com>" - ]; - - }; "platforms" = rec { crateName = "platforms"; version = "3.3.0"; @@ -5323,58 +2713,6 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "ppv-lite86" = rec { - crateName = "ppv-lite86"; - version = "0.2.17"; - edition = "2018"; - sha256 = "1pp6g52aw970adv3x2310n7glqnji96z0a9wiamzw89ibf0ayh2v"; - authors = [ - "The CryptoCorrosion Contributors" - ]; - features = { - "default" = [ "std" ]; - }; - resolvedDefaultFeatures = [ "simd" "std" ]; - }; - "prettyplease" = rec { - crateName = "prettyplease"; - version = "0.2.16"; - edition = "2021"; - links = "prettyplease02"; - sha256 = "1dfbq98rkq86l9g8w1l81bdvrz4spcfl48929n0pyz79clhzc754"; - authors = [ - "David Tolnay <dtolnay@gmail.com>" - ]; - dependencies = [ - { - name = "proc-macro2"; - packageId = "proc-macro2"; - usesDefaultFeatures = false; - } - { - name = "syn"; - packageId = "syn 2.0.52"; - usesDefaultFeatures = false; - features = [ "full" ]; - } - ]; - devDependencies = [ - { - name = "proc-macro2"; - packageId = "proc-macro2"; - usesDefaultFeatures = false; - } - { - name = "syn"; - packageId = "syn 2.0.52"; - usesDefaultFeatures = false; - features = [ "parsing" ]; - } - ]; - features = { - "verbatim" = [ "syn/parsing" ]; - }; - }; "proc-macro2" = rec { crateName = "proc-macro2"; version = "1.0.79"; @@ -5395,282 +2733,6 @@ rec { }; resolvedDefaultFeatures = [ "default" "proc-macro" ]; }; - "prost 0.11.9" = rec { - crateName = "prost"; - version = "0.11.9"; - edition = "2021"; - sha256 = "1kc1hva2h894hc0zf6r4r8fsxfpazf7xn5rj3jya9sbrsyhym0hb"; - authors = [ - "Dan Burkert <dan@danburkert.com>" - "Lucio Franco <luciofranco14@gmail.com" - "Tokio Contributors <team@tokio.rs>" - ]; - dependencies = [ - { - name = "bytes"; - packageId = "bytes"; - usesDefaultFeatures = false; - } - { - name = "prost-derive"; - packageId = "prost-derive 0.11.9"; - optional = true; - } - ]; - features = { - "default" = [ "prost-derive" "std" ]; - "prost-derive" = [ "dep:prost-derive" ]; - }; - resolvedDefaultFeatures = [ "default" "prost-derive" "std" ]; - }; - "prost 0.12.3" = rec { - crateName = "prost"; - version = "0.12.3"; - edition = "2021"; - sha256 = "0jmrhlb4jkiylz72xb14vlkfbmlq0jwv7j20ini9harhvaf2hv0l"; - authors = [ - "Dan Burkert <dan@danburkert.com>" - "Lucio Franco <luciofranco14@gmail.com" - "Tokio Contributors <team@tokio.rs>" - ]; - dependencies = [ - { - name = "bytes"; - packageId = "bytes"; - usesDefaultFeatures = false; - } - { - name = "prost-derive"; - packageId = "prost-derive 0.12.3"; - optional = true; - } - ]; - features = { - "default" = [ "prost-derive" "std" ]; - "prost-derive" = [ "dep:prost-derive" ]; - }; - resolvedDefaultFeatures = [ "default" "prost-derive" "std" ]; - }; - "prost-build" = rec { - crateName = "prost-build"; - version = "0.12.3"; - edition = "2021"; - sha256 = "1lp2l1l65l163yggk9nw5mjb2fqwzz12693af5phn1v0abih4pn5"; - authors = [ - "Dan Burkert <dan@danburkert.com>" - "Lucio Franco <luciofranco14@gmail.com>" - "Tokio Contributors <team@tokio.rs>" - ]; - dependencies = [ - { - name = "bytes"; - packageId = "bytes"; - usesDefaultFeatures = false; - } - { - name = "heck"; - packageId = "heck 0.4.1"; - } - { - name = "itertools"; - packageId = "itertools 0.11.0"; - usesDefaultFeatures = false; - features = [ "use_alloc" ]; - } - { - name = "log"; - packageId = "log"; - } - { - name = "multimap"; - packageId = "multimap"; - usesDefaultFeatures = false; - } - { - name = "once_cell"; - packageId = "once_cell"; - } - { - name = "petgraph"; - packageId = "petgraph"; - usesDefaultFeatures = false; - } - { - name = "prettyplease"; - packageId = "prettyplease"; - optional = true; - } - { - name = "prost"; - packageId = "prost 0.12.3"; - usesDefaultFeatures = false; - } - { - name = "prost-types"; - packageId = "prost-types"; - usesDefaultFeatures = false; - } - { - name = "regex"; - packageId = "regex"; - usesDefaultFeatures = false; - features = [ "std" "unicode-bool" ]; - } - { - name = "syn"; - packageId = "syn 2.0.52"; - optional = true; - features = [ "full" ]; - } - { - name = "tempfile"; - packageId = "tempfile"; - } - { - name = "which"; - packageId = "which"; - } - ]; - features = { - "cleanup-markdown" = [ "pulldown-cmark" "pulldown-cmark-to-cmark" ]; - "default" = [ "format" ]; - "format" = [ "prettyplease" "syn" ]; - "prettyplease" = [ "dep:prettyplease" ]; - "pulldown-cmark" = [ "dep:pulldown-cmark" ]; - "pulldown-cmark-to-cmark" = [ "dep:pulldown-cmark-to-cmark" ]; - "syn" = [ "dep:syn" ]; - }; - resolvedDefaultFeatures = [ "default" "format" "prettyplease" "syn" ]; - }; - "prost-derive 0.11.9" = rec { - crateName = "prost-derive"; - version = "0.11.9"; - edition = "2021"; - sha256 = "1d3mw2s2jba1f7wcjmjd6ha2a255p2rmynxhm1nysv9w1z8xilp5"; - procMacro = true; - authors = [ - "Dan Burkert <dan@danburkert.com>" - "Lucio Franco <luciofranco14@gmail.com>" - "Tokio Contributors <team@tokio.rs>" - ]; - dependencies = [ - { - name = "anyhow"; - packageId = "anyhow"; - } - { - name = "itertools"; - packageId = "itertools 0.10.5"; - usesDefaultFeatures = false; - features = [ "use_alloc" ]; - } - { - name = "proc-macro2"; - packageId = "proc-macro2"; - } - { - name = "quote"; - packageId = "quote"; - } - { - name = "syn"; - packageId = "syn 1.0.109"; - features = [ "extra-traits" ]; - } - ]; - - }; - "prost-derive 0.12.3" = rec { - crateName = "prost-derive"; - version = "0.12.3"; - edition = "2021"; - sha256 = "03l4yf6pdjvc4sgbvln2srq1avzm1ai86zni4hhqxvqxvnhwkdpg"; - procMacro = true; - authors = [ - "Dan Burkert <dan@danburkert.com>" - "Lucio Franco <luciofranco14@gmail.com>" - "Tokio Contributors <team@tokio.rs>" - ]; - dependencies = [ - { - name = "anyhow"; - packageId = "anyhow"; - } - { - name = "itertools"; - packageId = "itertools 0.11.0"; - usesDefaultFeatures = false; - features = [ "use_alloc" ]; - } - { - name = "proc-macro2"; - packageId = "proc-macro2"; - } - { - name = "quote"; - packageId = "quote"; - } - { - name = "syn"; - packageId = "syn 2.0.52"; - features = [ "extra-traits" ]; - } - ]; - - }; - "prost-types" = rec { - crateName = "prost-types"; - version = "0.12.3"; - edition = "2021"; - sha256 = "03j73llzljdxv9cdxp4m3vb9j3gh4y24rkbx48k3rx6wkvsrhf0r"; - authors = [ - "Dan Burkert <dan@danburkert.com>" - "Lucio Franco <luciofranco14@gmail.com" - "Tokio Contributors <team@tokio.rs>" - ]; - dependencies = [ - { - name = "prost"; - packageId = "prost 0.12.3"; - usesDefaultFeatures = false; - features = [ "prost-derive" ]; - } - ]; - features = { - "default" = [ "std" ]; - "std" = [ "prost/std" ]; - }; - resolvedDefaultFeatures = [ "default" "std" ]; - }; - "quick-xml" = rec { - crateName = "quick-xml"; - version = "0.31.0"; - edition = "2021"; - sha256 = "0cravqanylzh5cq2v6hzlfqgxcid5nrp2snnb3pf4m0and2a610h"; - dependencies = [ - { - name = "memchr"; - packageId = "memchr"; - } - { - name = "serde"; - packageId = "serde"; - optional = true; - } - ]; - features = { - "arbitrary" = [ "dep:arbitrary" ]; - "async-tokio" = [ "tokio" ]; - "document-features" = [ "dep:document-features" ]; - "encoding" = [ "encoding_rs" ]; - "encoding_rs" = [ "dep:encoding_rs" ]; - "serde" = [ "dep:serde" ]; - "serde-types" = [ "serde/derive" ]; - "serialize" = [ "serde" ]; - "tokio" = [ "dep:tokio" ]; - }; - resolvedDefaultFeatures = [ "default" "overlapped-lists" "serde" "serialize" ]; - }; "quote" = rec { crateName = "quote"; version = "1.0.35"; @@ -5692,80 +2754,6 @@ rec { }; resolvedDefaultFeatures = [ "default" "proc-macro" ]; }; - "rand" = rec { - crateName = "rand"; - version = "0.8.5"; - edition = "2018"; - sha256 = "013l6931nn7gkc23jz5mm3qdhf93jjf0fg64nz2lp4i51qd8vbrl"; - authors = [ - "The Rand Project Developers" - "The Rust Project Developers" - ]; - dependencies = [ - { - name = "libc"; - packageId = "libc"; - optional = true; - usesDefaultFeatures = false; - target = { target, features }: (target."unix" or false); - } - { - name = "rand_chacha"; - packageId = "rand_chacha"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "rand_core"; - packageId = "rand_core"; - } - ]; - features = { - "alloc" = [ "rand_core/alloc" ]; - "default" = [ "std" "std_rng" ]; - "getrandom" = [ "rand_core/getrandom" ]; - "libc" = [ "dep:libc" ]; - "log" = [ "dep:log" ]; - "packed_simd" = [ "dep:packed_simd" ]; - "rand_chacha" = [ "dep:rand_chacha" ]; - "serde" = [ "dep:serde" ]; - "serde1" = [ "serde" "rand_core/serde1" ]; - "simd_support" = [ "packed_simd" ]; - "std" = [ "rand_core/std" "rand_chacha/std" "alloc" "getrandom" "libc" ]; - "std_rng" = [ "rand_chacha" ]; - }; - resolvedDefaultFeatures = [ "alloc" "default" "getrandom" "libc" "rand_chacha" "small_rng" "std" "std_rng" ]; - }; - "rand_chacha" = rec { - crateName = "rand_chacha"; - version = "0.3.1"; - edition = "2018"; - sha256 = "123x2adin558xbhvqb8w4f6syjsdkmqff8cxwhmjacpsl1ihmhg6"; - authors = [ - "The Rand Project Developers" - "The Rust Project Developers" - "The CryptoCorrosion Contributors" - ]; - dependencies = [ - { - name = "ppv-lite86"; - packageId = "ppv-lite86"; - usesDefaultFeatures = false; - features = [ "simd" ]; - } - { - name = "rand_core"; - packageId = "rand_core"; - } - ]; - features = { - "default" = [ "std" ]; - "serde" = [ "dep:serde" ]; - "serde1" = [ "serde" ]; - "std" = [ "ppv-lite86/std" ]; - }; - resolvedDefaultFeatures = [ "std" ]; - }; "rand_core" = rec { crateName = "rand_core"; version = "0.6.4"; @@ -5790,72 +2778,7 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "getrandom" "std" ]; }; - "rayon" = rec { - crateName = "rayon"; - version = "1.9.0"; - edition = "2021"; - sha256 = "1gdk945j52vq3zx5vb4yzc3yyz19bf2vs8kh47pg7r46pk8kx5p4"; - authors = [ - "Niko Matsakis <niko@alum.mit.edu>" - "Josh Stone <cuviper@gmail.com>" - ]; - dependencies = [ - { - name = "either"; - packageId = "either"; - usesDefaultFeatures = false; - } - { - name = "rayon-core"; - packageId = "rayon-core"; - } - ]; - features = { - "web_spin_lock" = [ "dep:wasm_sync" "rayon-core/web_spin_lock" ]; - }; - }; - "rayon-core" = rec { - crateName = "rayon-core"; - version = "1.12.1"; - edition = "2021"; - links = "rayon-core"; - sha256 = "1qpwim68ai5h0j7axa8ai8z0payaawv3id0lrgkqmapx7lx8fr8l"; - authors = [ - "Niko Matsakis <niko@alum.mit.edu>" - "Josh Stone <cuviper@gmail.com>" - ]; - dependencies = [ - { - name = "crossbeam-deque"; - packageId = "crossbeam-deque"; - } - { - name = "crossbeam-utils"; - packageId = "crossbeam-utils"; - } - ]; - features = { - "web_spin_lock" = [ "dep:wasm_sync" ]; - }; - }; - "redox_syscall 0.2.16" = rec { - crateName = "redox_syscall"; - version = "0.2.16"; - edition = "2018"; - sha256 = "16jicm96kjyzm802cxdd1k9jmcph0db1a4lhslcnhjsvhp0mhnpv"; - libName = "syscall"; - authors = [ - "Jeremy Soller <jackpot51@gmail.com>" - ]; - dependencies = [ - { - name = "bitflags"; - packageId = "bitflags 1.3.2"; - } - ]; - - }; - "redox_syscall 0.4.1" = rec { + "redox_syscall" = rec { crateName = "redox_syscall"; version = "0.4.1"; edition = "2018"; @@ -5875,89 +2798,7 @@ rec { "rustc-dep-of-std" = [ "core" "bitflags/rustc-dep-of-std" ]; }; }; - "regex" = rec { - crateName = "regex"; - version = "1.10.3"; - edition = "2021"; - sha256 = "05cvihqy0wgnh9i8a9y2n803n5azg2h0b7nlqy6rsvxhy00vwbdn"; - authors = [ - "The Rust Project Developers" - "Andrew Gallant <jamslam@gmail.com>" - ]; - dependencies = [ - { - name = "aho-corasick"; - packageId = "aho-corasick"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "memchr"; - packageId = "memchr"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "regex-automata"; - packageId = "regex-automata 0.4.6"; - usesDefaultFeatures = false; - features = [ "alloc" "syntax" "meta" "nfa-pikevm" ]; - } - { - name = "regex-syntax"; - packageId = "regex-syntax 0.8.2"; - usesDefaultFeatures = false; - } - ]; - features = { - "default" = [ "std" "perf" "unicode" "regex-syntax/default" ]; - "logging" = [ "aho-corasick?/logging" "memchr?/logging" "regex-automata/logging" ]; - "perf" = [ "perf-cache" "perf-dfa" "perf-onepass" "perf-backtrack" "perf-inline" "perf-literal" ]; - "perf-backtrack" = [ "regex-automata/nfa-backtrack" ]; - "perf-dfa" = [ "regex-automata/hybrid" ]; - "perf-dfa-full" = [ "regex-automata/dfa-build" "regex-automata/dfa-search" ]; - "perf-inline" = [ "regex-automata/perf-inline" ]; - "perf-literal" = [ "dep:aho-corasick" "dep:memchr" "regex-automata/perf-literal" ]; - "perf-onepass" = [ "regex-automata/dfa-onepass" ]; - "std" = [ "aho-corasick?/std" "memchr?/std" "regex-automata/std" "regex-syntax/std" ]; - "unicode" = [ "unicode-age" "unicode-bool" "unicode-case" "unicode-gencat" "unicode-perl" "unicode-script" "unicode-segment" "regex-automata/unicode" "regex-syntax/unicode" ]; - "unicode-age" = [ "regex-automata/unicode-age" "regex-syntax/unicode-age" ]; - "unicode-bool" = [ "regex-automata/unicode-bool" "regex-syntax/unicode-bool" ]; - "unicode-case" = [ "regex-automata/unicode-case" "regex-syntax/unicode-case" ]; - "unicode-gencat" = [ "regex-automata/unicode-gencat" "regex-syntax/unicode-gencat" ]; - "unicode-perl" = [ "regex-automata/unicode-perl" "regex-automata/unicode-word-boundary" "regex-syntax/unicode-perl" ]; - "unicode-script" = [ "regex-automata/unicode-script" "regex-syntax/unicode-script" ]; - "unicode-segment" = [ "regex-automata/unicode-segment" "regex-syntax/unicode-segment" ]; - "unstable" = [ "pattern" ]; - "use_std" = [ "std" ]; - }; - resolvedDefaultFeatures = [ "std" "unicode-bool" "unicode-case" "unicode-perl" ]; - }; - "regex-automata 0.1.10" = rec { - crateName = "regex-automata"; - version = "0.1.10"; - edition = "2015"; - sha256 = "0ci1hvbzhrfby5fdpf4ganhf7kla58acad9i1ff1p34dzdrhs8vc"; - authors = [ - "Andrew Gallant <jamslam@gmail.com>" - ]; - dependencies = [ - { - name = "regex-syntax"; - packageId = "regex-syntax 0.6.29"; - optional = true; - } - ]; - features = { - "default" = [ "std" ]; - "fst" = [ "dep:fst" ]; - "regex-syntax" = [ "dep:regex-syntax" ]; - "std" = [ "regex-syntax" ]; - "transducer" = [ "std" "fst" ]; - }; - resolvedDefaultFeatures = [ "default" "regex-syntax" "std" ]; - }; - "regex-automata 0.4.6" = rec { + "regex-automata" = rec { crateName = "regex-automata"; version = "0.4.6"; edition = "2021"; @@ -5966,26 +2807,6 @@ rec { "The Rust Project Developers" "Andrew Gallant <jamslam@gmail.com>" ]; - dependencies = [ - { - name = "aho-corasick"; - packageId = "aho-corasick"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "memchr"; - packageId = "memchr"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "regex-syntax"; - packageId = "regex-syntax 0.8.2"; - optional = true; - usesDefaultFeatures = false; - } - ]; features = { "default" = [ "std" "syntax" "perf" "unicode" "meta" "nfa" "dfa" "hybrid" ]; "dfa" = [ "dfa-build" "dfa-search" "dfa-onepass" ]; @@ -6015,382 +2836,7 @@ rec { "unicode-script" = [ "regex-syntax?/unicode-script" ]; "unicode-segment" = [ "regex-syntax?/unicode-segment" ]; }; - resolvedDefaultFeatures = [ "alloc" "dfa-search" "meta" "nfa-pikevm" "nfa-thompson" "std" "syntax" "unicode-bool" "unicode-case" "unicode-perl" "unicode-word-boundary" ]; - }; - "regex-syntax 0.6.29" = rec { - crateName = "regex-syntax"; - version = "0.6.29"; - edition = "2018"; - sha256 = "1qgj49vm6y3zn1hi09x91jvgkl2b1fiaq402skj83280ggfwcqpi"; - authors = [ - "The Rust Project Developers" - ]; - features = { - "default" = [ "unicode" ]; - "unicode" = [ "unicode-age" "unicode-bool" "unicode-case" "unicode-gencat" "unicode-perl" "unicode-script" "unicode-segment" ]; - }; - resolvedDefaultFeatures = [ "default" "unicode" "unicode-age" "unicode-bool" "unicode-case" "unicode-gencat" "unicode-perl" "unicode-script" "unicode-segment" ]; - }; - "regex-syntax 0.8.2" = rec { - crateName = "regex-syntax"; - version = "0.8.2"; - edition = "2021"; - sha256 = "17rd2s8xbiyf6lb4aj2nfi44zqlj98g2ays8zzj2vfs743k79360"; - authors = [ - "The Rust Project Developers" - "Andrew Gallant <jamslam@gmail.com>" - ]; - features = { - "arbitrary" = [ "dep:arbitrary" ]; - "default" = [ "std" "unicode" ]; - "unicode" = [ "unicode-age" "unicode-bool" "unicode-case" "unicode-gencat" "unicode-perl" "unicode-script" "unicode-segment" ]; - }; - resolvedDefaultFeatures = [ "std" "unicode-bool" "unicode-case" "unicode-perl" ]; - }; - "reqwest" = rec { - crateName = "reqwest"; - version = "0.11.26"; - edition = "2021"; - sha256 = "1chhmkfdk0ji5dacwsjix05g2vrmjzkyn76mg7cbg2vsmz297gvq"; - authors = [ - "Sean McArthur <sean@seanmonstar.com>" - ]; - dependencies = [ - { - name = "base64"; - packageId = "base64"; - } - { - name = "bytes"; - packageId = "bytes"; - } - { - name = "encoding_rs"; - packageId = "encoding_rs"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "futures-core"; - packageId = "futures-core"; - usesDefaultFeatures = false; - } - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - } - { - name = "h2"; - packageId = "h2 0.3.24"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "http"; - packageId = "http 0.2.12"; - } - { - name = "http-body"; - packageId = "http-body 0.4.6"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "hyper"; - packageId = "hyper 0.14.28"; - usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "tcp" "http1" "http2" "client" "runtime" ]; - } - { - name = "hyper-rustls"; - packageId = "hyper-rustls"; - optional = true; - usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "ipnet"; - packageId = "ipnet"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "js-sys"; - packageId = "js-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); - } - { - name = "log"; - packageId = "log"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "mime"; - packageId = "mime"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "once_cell"; - packageId = "once_cell"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "percent-encoding"; - packageId = "percent-encoding"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "pin-project-lite"; - packageId = "pin-project-lite"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "rustls"; - packageId = "rustls 0.21.10"; - optional = true; - target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "dangerous_configuration" ]; - } - { - name = "rustls-native-certs"; - packageId = "rustls-native-certs 0.6.3"; - optional = true; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "rustls-pemfile"; - packageId = "rustls-pemfile 1.0.4"; - optional = true; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "serde"; - packageId = "serde"; - } - { - name = "serde_json"; - packageId = "serde_json"; - optional = true; - } - { - name = "serde_json"; - packageId = "serde_json"; - target = { target, features }: ("wasm32" == target."arch" or null); - } - { - name = "serde_urlencoded"; - packageId = "serde_urlencoded"; - } - { - name = "sync_wrapper"; - packageId = "sync_wrapper"; - } - { - name = "system-configuration"; - packageId = "system-configuration"; - target = { target, features }: ("macos" == target."os" or null); - } - { - name = "tokio"; - packageId = "tokio"; - usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "net" "time" ]; - } - { - name = "tokio-rustls"; - packageId = "tokio-rustls 0.24.1"; - optional = true; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "tokio-util"; - packageId = "tokio-util"; - optional = true; - usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "codec" "io" ]; - } - { - name = "tower-service"; - packageId = "tower-service"; - } - { - name = "url"; - packageId = "url"; - } - { - name = "wasm-bindgen"; - packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); - } - { - name = "wasm-bindgen-futures"; - packageId = "wasm-bindgen-futures"; - target = { target, features }: ("wasm32" == target."arch" or null); - } - { - name = "wasm-streams"; - packageId = "wasm-streams"; - optional = true; - target = { target, features }: ("wasm32" == target."arch" or null); - } - { - name = "web-sys"; - packageId = "web-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); - features = [ "AbortController" "AbortSignal" "Headers" "Request" "RequestInit" "RequestMode" "Response" "Window" "FormData" "Blob" "BlobPropertyBag" "ServiceWorkerGlobalScope" "RequestCredentials" "File" "ReadableStream" ]; - } - { - name = "winreg"; - packageId = "winreg"; - target = { target, features }: (target."windows" or false); - } - ]; - devDependencies = [ - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "std" "alloc" ]; - } - { - name = "hyper"; - packageId = "hyper 0.14.28"; - usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "tcp" "stream" "http1" "http2" "client" "server" "runtime" ]; - } - { - name = "serde"; - packageId = "serde"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "derive" ]; - } - { - name = "tokio"; - packageId = "tokio"; - usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "macros" "rt-multi-thread" ]; - } - { - name = "wasm-bindgen"; - packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); - features = [ "serde-serialize" ]; - } - ]; - features = { - "__rustls" = [ "hyper-rustls" "tokio-rustls" "rustls" "__tls" ]; - "__tls" = [ "dep:rustls-pemfile" ]; - "async-compression" = [ "dep:async-compression" ]; - "blocking" = [ "futures-util/io" "tokio/sync" ]; - "brotli" = [ "async-compression" "async-compression/brotli" "tokio-util" ]; - "cookie_crate" = [ "dep:cookie_crate" ]; - "cookie_store" = [ "dep:cookie_store" ]; - "cookies" = [ "cookie_crate" "cookie_store" ]; - "default" = [ "default-tls" ]; - "default-tls" = [ "hyper-tls" "native-tls-crate" "__tls" "tokio-native-tls" ]; - "deflate" = [ "async-compression" "async-compression/zlib" "tokio-util" ]; - "futures-channel" = [ "dep:futures-channel" ]; - "gzip" = [ "async-compression" "async-compression/gzip" "tokio-util" ]; - "h3" = [ "dep:h3" ]; - "h3-quinn" = [ "dep:h3-quinn" ]; - "http3" = [ "rustls-tls-manual-roots" "h3" "h3-quinn" "quinn" "futures-channel" ]; - "hyper-rustls" = [ "dep:hyper-rustls" ]; - "hyper-tls" = [ "dep:hyper-tls" ]; - "json" = [ "serde_json" ]; - "mime_guess" = [ "dep:mime_guess" ]; - "multipart" = [ "mime_guess" ]; - "native-tls" = [ "default-tls" ]; - "native-tls-alpn" = [ "native-tls" "native-tls-crate/alpn" ]; - "native-tls-crate" = [ "dep:native-tls-crate" ]; - "native-tls-vendored" = [ "native-tls" "native-tls-crate/vendored" ]; - "quinn" = [ "dep:quinn" ]; - "rustls" = [ "dep:rustls" ]; - "rustls-native-certs" = [ "dep:rustls-native-certs" ]; - "rustls-tls" = [ "rustls-tls-webpki-roots" ]; - "rustls-tls-manual-roots" = [ "__rustls" ]; - "rustls-tls-native-roots" = [ "rustls-native-certs" "__rustls" ]; - "rustls-tls-webpki-roots" = [ "webpki-roots" "__rustls" ]; - "serde_json" = [ "dep:serde_json" ]; - "socks" = [ "tokio-socks" ]; - "stream" = [ "tokio/fs" "tokio-util" "wasm-streams" ]; - "tokio-native-tls" = [ "dep:tokio-native-tls" ]; - "tokio-rustls" = [ "dep:tokio-rustls" ]; - "tokio-socks" = [ "dep:tokio-socks" ]; - "tokio-util" = [ "dep:tokio-util" ]; - "trust-dns" = [ "trust-dns-resolver" ]; - "trust-dns-resolver" = [ "dep:trust-dns-resolver" ]; - "wasm-streams" = [ "dep:wasm-streams" ]; - "webpki-roots" = [ "dep:webpki-roots" ]; - }; - resolvedDefaultFeatures = [ "__rustls" "__tls" "hyper-rustls" "json" "rustls" "rustls-native-certs" "rustls-tls-native-roots" "serde_json" "stream" "tokio-rustls" "tokio-util" "wasm-streams" ]; - }; - "ring" = rec { - crateName = "ring"; - version = "0.17.8"; - edition = "2021"; - links = "ring_core_0_17_8"; - sha256 = "03fwlb1ssrmfxdckvqv033pfmk01rhx9ynwi7r186dcfcp5s8zy1"; - authors = [ - "Brian Smith <brian@briansmith.org>" - ]; - dependencies = [ - { - name = "cfg-if"; - packageId = "cfg-if"; - usesDefaultFeatures = false; - } - { - name = "getrandom"; - packageId = "getrandom"; - } - { - name = "libc"; - packageId = "libc"; - usesDefaultFeatures = false; - target = { target, features }: ((("android" == target."os" or null) || ("linux" == target."os" or null)) && (("aarch64" == target."arch" or null) || ("arm" == target."arch" or null))); - } - { - name = "spin"; - packageId = "spin"; - usesDefaultFeatures = false; - target = { target, features }: (("aarch64" == target."arch" or null) || ("arm" == target."arch" or null) || ("x86" == target."arch" or null) || ("x86_64" == target."arch" or null)); - features = [ "once" ]; - } - { - name = "untrusted"; - packageId = "untrusted"; - } - { - name = "windows-sys"; - packageId = "windows-sys 0.52.0"; - target = { target, features }: (("aarch64" == target."arch" or null) && ("windows" == target."os" or null)); - features = [ "Win32_Foundation" "Win32_System_Threading" ]; - } - ]; - buildDependencies = [ - { - name = "cc"; - packageId = "cc"; - usesDefaultFeatures = false; - } - ]; - devDependencies = [ - { - name = "libc"; - packageId = "libc"; - usesDefaultFeatures = false; - target = { target, features }: ((target."unix" or false) || (target."windows" or false) || ("wasi" == target."os" or null)); - } - ]; - features = { - "default" = [ "alloc" "dev_urandom_fallback" ]; - "std" = [ "alloc" ]; - "wasm32_unknown_unknown_js" = [ "getrandom/js" ]; - }; - resolvedDefaultFeatures = [ "alloc" "default" "dev_urandom_fallback" "std" ]; + resolvedDefaultFeatures = [ "dfa-search" ]; }; "rustc-demangle" = rec { crateName = "rustc-demangle"; @@ -6423,378 +2869,6 @@ rec { ]; }; - "rustix" = rec { - crateName = "rustix"; - version = "0.38.31"; - edition = "2021"; - sha256 = "0jg9yj3i6qnzk1y82hng7rb1bwhslfbh57507dxcs9mgcakf38vf"; - authors = [ - "Dan Gohman <dev@sunfishcode.online>" - "Jakub Konka <kubkon@jakubkonka.com>" - ]; - dependencies = [ - { - name = "bitflags"; - packageId = "bitflags 2.4.2"; - usesDefaultFeatures = false; - } - { - name = "errno"; - packageId = "errno"; - rename = "libc_errno"; - optional = true; - usesDefaultFeatures = false; - target = { target, features }: ((!(target."rustix_use_libc" or false)) && (!(target."miri" or false)) && ("linux" == target."os" or null) && ("little" == target."endian" or null) && (("arm" == target."arch" or null) || (("aarch64" == target."arch" or null) && ("64" == target."pointer_width" or null)) || ("riscv64" == target."arch" or null) || ((target."rustix_use_experimental_asm" or false) && ("powerpc64" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips32r6" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips64" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips64r6" == target."arch" or null)) || ("x86" == target."arch" or null) || (("x86_64" == target."arch" or null) && ("64" == target."pointer_width" or null)))); - } - { - name = "errno"; - packageId = "errno"; - rename = "libc_errno"; - usesDefaultFeatures = false; - target = { target, features }: ((!(target."windows" or false)) && ((target."rustix_use_libc" or false) || (target."miri" or false) || (!(("linux" == target."os" or null) && ("little" == target."endian" or null) && (("arm" == target."arch" or null) || (("aarch64" == target."arch" or null) && ("64" == target."pointer_width" or null)) || ("riscv64" == target."arch" or null) || ((target."rustix_use_experimental_asm" or false) && ("powerpc64" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips32r6" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips64" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips64r6" == target."arch" or null)) || ("x86" == target."arch" or null) || (("x86_64" == target."arch" or null) && ("64" == target."pointer_width" or null))))))); - } - { - name = "errno"; - packageId = "errno"; - rename = "libc_errno"; - usesDefaultFeatures = false; - target = { target, features }: (target."windows" or false); - } - { - name = "libc"; - packageId = "libc"; - optional = true; - usesDefaultFeatures = false; - target = { target, features }: ((!(target."rustix_use_libc" or false)) && (!(target."miri" or false)) && ("linux" == target."os" or null) && ("little" == target."endian" or null) && (("arm" == target."arch" or null) || (("aarch64" == target."arch" or null) && ("64" == target."pointer_width" or null)) || ("riscv64" == target."arch" or null) || ((target."rustix_use_experimental_asm" or false) && ("powerpc64" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips32r6" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips64" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips64r6" == target."arch" or null)) || ("x86" == target."arch" or null) || (("x86_64" == target."arch" or null) && ("64" == target."pointer_width" or null)))); - features = [ "extra_traits" ]; - } - { - name = "libc"; - packageId = "libc"; - usesDefaultFeatures = false; - target = { target, features }: ((!(target."windows" or false)) && ((target."rustix_use_libc" or false) || (target."miri" or false) || (!(("linux" == target."os" or null) && ("little" == target."endian" or null) && (("arm" == target."arch" or null) || (("aarch64" == target."arch" or null) && ("64" == target."pointer_width" or null)) || ("riscv64" == target."arch" or null) || ((target."rustix_use_experimental_asm" or false) && ("powerpc64" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips32r6" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips64" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips64r6" == target."arch" or null)) || ("x86" == target."arch" or null) || (("x86_64" == target."arch" or null) && ("64" == target."pointer_width" or null))))))); - features = [ "extra_traits" ]; - } - { - name = "linux-raw-sys"; - packageId = "linux-raw-sys"; - usesDefaultFeatures = false; - target = { target, features }: ((("android" == target."os" or null) || ("linux" == target."os" or null)) && ((target."rustix_use_libc" or false) || (target."miri" or false) || (!(("linux" == target."os" or null) && ("little" == target."endian" or null) && (("arm" == target."arch" or null) || (("aarch64" == target."arch" or null) && ("64" == target."pointer_width" or null)) || ("riscv64" == target."arch" or null) || ((target."rustix_use_experimental_asm" or false) && ("powerpc64" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips32r6" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips64" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips64r6" == target."arch" or null)) || ("x86" == target."arch" or null) || (("x86_64" == target."arch" or null) && ("64" == target."pointer_width" or null))))))); - features = [ "general" "ioctl" "no_std" ]; - } - { - name = "linux-raw-sys"; - packageId = "linux-raw-sys"; - usesDefaultFeatures = false; - target = { target, features }: ((!(target."rustix_use_libc" or false)) && (!(target."miri" or false)) && ("linux" == target."os" or null) && ("little" == target."endian" or null) && (("arm" == target."arch" or null) || (("aarch64" == target."arch" or null) && ("64" == target."pointer_width" or null)) || ("riscv64" == target."arch" or null) || ((target."rustix_use_experimental_asm" or false) && ("powerpc64" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips32r6" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips64" == target."arch" or null)) || ((target."rustix_use_experimental_asm" or false) && ("mips64r6" == target."arch" or null)) || ("x86" == target."arch" or null) || (("x86_64" == target."arch" or null) && ("64" == target."pointer_width" or null)))); - features = [ "general" "errno" "ioctl" "no_std" "elf" ]; - } - { - name = "windows-sys"; - packageId = "windows-sys 0.52.0"; - target = { target, features }: (target."windows" or false); - features = [ "Win32_Foundation" "Win32_Networking_WinSock" "Win32_NetworkManagement_IpHelper" "Win32_System_Threading" ]; - } - ]; - devDependencies = [ - { - name = "errno"; - packageId = "errno"; - rename = "libc_errno"; - usesDefaultFeatures = false; - } - { - name = "libc"; - packageId = "libc"; - } - ]; - features = { - "all-apis" = [ "event" "fs" "io_uring" "mm" "mount" "net" "param" "pipe" "process" "procfs" "pty" "rand" "runtime" "shm" "stdio" "system" "termios" "thread" "time" ]; - "default" = [ "std" "use-libc-auxv" ]; - "io_uring" = [ "event" "fs" "net" "linux-raw-sys/io_uring" ]; - "itoa" = [ "dep:itoa" ]; - "libc" = [ "dep:libc" ]; - "libc_errno" = [ "dep:libc_errno" ]; - "linux_latest" = [ "linux_4_11" ]; - "net" = [ "linux-raw-sys/net" "linux-raw-sys/netlink" "linux-raw-sys/if_ether" "linux-raw-sys/xdp" ]; - "once_cell" = [ "dep:once_cell" ]; - "param" = [ "fs" ]; - "process" = [ "linux-raw-sys/prctl" ]; - "procfs" = [ "once_cell" "itoa" "fs" ]; - "pty" = [ "itoa" "fs" ]; - "runtime" = [ "linux-raw-sys/prctl" ]; - "rustc-dep-of-std" = [ "dep:core" "dep:alloc" "dep:compiler_builtins" "linux-raw-sys/rustc-dep-of-std" "bitflags/rustc-dep-of-std" "compiler_builtins?/rustc-dep-of-std" ]; - "shm" = [ "fs" ]; - "std" = [ "bitflags/std" "alloc" "libc?/std" "libc_errno?/std" ]; - "system" = [ "linux-raw-sys/system" ]; - "thread" = [ "linux-raw-sys/prctl" ]; - "use-libc" = [ "libc_errno" "libc" ]; - }; - resolvedDefaultFeatures = [ "alloc" "default" "fs" "std" "use-libc-auxv" ]; - }; - "rustls 0.21.10" = rec { - crateName = "rustls"; - version = "0.21.10"; - edition = "2021"; - sha256 = "1fmpzk3axnhkd99saqkvraifdfms4pkyi56lkihf8n877j0sdmgr"; - dependencies = [ - { - name = "log"; - packageId = "log"; - optional = true; - } - { - name = "ring"; - packageId = "ring"; - } - { - name = "rustls-webpki"; - packageId = "rustls-webpki 0.101.7"; - rename = "webpki"; - features = [ "alloc" "std" ]; - } - { - name = "sct"; - packageId = "sct"; - } - ]; - devDependencies = [ - { - name = "log"; - packageId = "log"; - } - ]; - features = { - "default" = [ "logging" "tls12" ]; - "log" = [ "dep:log" ]; - "logging" = [ "log" ]; - "read_buf" = [ "rustversion" ]; - "rustversion" = [ "dep:rustversion" ]; - }; - resolvedDefaultFeatures = [ "dangerous_configuration" "default" "log" "logging" "tls12" ]; - }; - "rustls 0.22.2" = rec { - crateName = "rustls"; - version = "0.22.2"; - edition = "2021"; - sha256 = "0hcxyhq6ynvws9v5b2h81s1nwmijmya7a3vyyyhsy1wqpmb9jz78"; - dependencies = [ - { - name = "log"; - packageId = "log"; - optional = true; - } - { - name = "ring"; - packageId = "ring"; - optional = true; - } - { - name = "rustls-pki-types"; - packageId = "rustls-pki-types"; - rename = "pki-types"; - features = [ "std" ]; - } - { - name = "rustls-webpki"; - packageId = "rustls-webpki 0.102.2"; - rename = "webpki"; - usesDefaultFeatures = false; - features = [ "std" ]; - } - { - name = "subtle"; - packageId = "subtle"; - usesDefaultFeatures = false; - } - { - name = "zeroize"; - packageId = "zeroize"; - } - ]; - devDependencies = [ - { - name = "log"; - packageId = "log"; - } - ]; - features = { - "aws_lc_rs" = [ "dep:aws-lc-rs" "webpki/aws_lc_rs" ]; - "default" = [ "logging" "ring" "tls12" ]; - "log" = [ "dep:log" ]; - "logging" = [ "log" ]; - "read_buf" = [ "rustversion" ]; - "ring" = [ "dep:ring" "webpki/ring" ]; - "rustversion" = [ "dep:rustversion" ]; - }; - resolvedDefaultFeatures = [ "log" "logging" "ring" "tls12" ]; - }; - "rustls-native-certs 0.6.3" = rec { - crateName = "rustls-native-certs"; - version = "0.6.3"; - edition = "2021"; - sha256 = "007zind70rd5rfsrkdcfm8vn09j8sg02phg9334kark6rdscxam9"; - dependencies = [ - { - name = "openssl-probe"; - packageId = "openssl-probe"; - target = { target, features }: ((target."unix" or false) && (!("macos" == target."os" or null))); - } - { - name = "rustls-pemfile"; - packageId = "rustls-pemfile 1.0.4"; - } - { - name = "schannel"; - packageId = "schannel"; - target = { target, features }: (target."windows" or false); - } - { - name = "security-framework"; - packageId = "security-framework"; - target = { target, features }: ("macos" == target."os" or null); - } - ]; - - }; - "rustls-native-certs 0.7.0" = rec { - crateName = "rustls-native-certs"; - version = "0.7.0"; - edition = "2021"; - sha256 = "14ip15dcr6fmjzi12lla9cpln7mmkdid4a7wsp344v4kz9gbh7wg"; - dependencies = [ - { - name = "openssl-probe"; - packageId = "openssl-probe"; - target = { target, features }: ((target."unix" or false) && (!("macos" == target."os" or null))); - } - { - name = "rustls-pemfile"; - packageId = "rustls-pemfile 2.1.1"; - } - { - name = "rustls-pki-types"; - packageId = "rustls-pki-types"; - rename = "pki-types"; - } - { - name = "schannel"; - packageId = "schannel"; - target = { target, features }: (target."windows" or false); - } - { - name = "security-framework"; - packageId = "security-framework"; - target = { target, features }: ("macos" == target."os" or null); - } - ]; - - }; - "rustls-pemfile 1.0.4" = rec { - crateName = "rustls-pemfile"; - version = "1.0.4"; - edition = "2018"; - sha256 = "1324n5bcns0rnw6vywr5agff3rwfvzphi7rmbyzwnv6glkhclx0w"; - dependencies = [ - { - name = "base64"; - packageId = "base64"; - } - ]; - - }; - "rustls-pemfile 2.1.1" = rec { - crateName = "rustls-pemfile"; - version = "2.1.1"; - edition = "2018"; - sha256 = "1awxak91qgraqrsk7bwxyn2aijhzyrs7flmaddajmxbgbrl750gl"; - dependencies = [ - { - name = "base64"; - packageId = "base64"; - usesDefaultFeatures = false; - features = [ "alloc" ]; - } - { - name = "rustls-pki-types"; - packageId = "rustls-pki-types"; - rename = "pki-types"; - } - ]; - features = { - "default" = [ "std" ]; - "std" = [ "base64/std" ]; - }; - resolvedDefaultFeatures = [ "default" "std" ]; - }; - "rustls-pki-types" = rec { - crateName = "rustls-pki-types"; - version = "1.3.1"; - edition = "2021"; - sha256 = "1a0g7453h07701vyxjj05gv903a0shi43mf7hl3cdd08hsr6gpjy"; - features = { - "default" = [ "alloc" ]; - "std" = [ "alloc" ]; - }; - resolvedDefaultFeatures = [ "alloc" "default" "std" ]; - }; - "rustls-webpki 0.101.7" = rec { - crateName = "rustls-webpki"; - version = "0.101.7"; - edition = "2021"; - sha256 = "0rapfhpkqp75552i8r0y7f4vq7csb4k7gjjans0df73sxv8paqlb"; - libName = "webpki"; - dependencies = [ - { - name = "ring"; - packageId = "ring"; - usesDefaultFeatures = false; - } - { - name = "untrusted"; - packageId = "untrusted"; - } - ]; - features = { - "alloc" = [ "ring/alloc" ]; - "default" = [ "std" ]; - "std" = [ "alloc" ]; - }; - resolvedDefaultFeatures = [ "alloc" "default" "std" ]; - }; - "rustls-webpki 0.102.2" = rec { - crateName = "rustls-webpki"; - version = "0.102.2"; - edition = "2021"; - sha256 = "041ncshpw8wsvi8p74a3yw9c0r17lhyk1yjsxyrbkv8bfii0maps"; - libName = "webpki"; - dependencies = [ - { - name = "ring"; - packageId = "ring"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "rustls-pki-types"; - packageId = "rustls-pki-types"; - rename = "pki-types"; - usesDefaultFeatures = false; - } - { - name = "untrusted"; - packageId = "untrusted"; - } - ]; - features = { - "alloc" = [ "ring?/alloc" "pki-types/alloc" ]; - "aws_lc_rs" = [ "dep:aws-lc-rs" ]; - "default" = [ "std" "ring" ]; - "ring" = [ "dep:ring" ]; - "std" = [ "alloc" "pki-types/std" ]; - }; - resolvedDefaultFeatures = [ "alloc" "ring" "std" ]; - }; "rustversion" = rec { crateName = "rustversion"; version = "1.0.14"; @@ -6819,48 +2893,6 @@ rec { "no-panic" = [ "dep:no-panic" ]; }; }; - "same-file" = rec { - crateName = "same-file"; - version = "1.0.6"; - edition = "2018"; - sha256 = "00h5j1w87dmhnvbv9l8bic3y7xxsnjmssvifw2ayvgx9mb1ivz4k"; - authors = [ - "Andrew Gallant <jamslam@gmail.com>" - ]; - dependencies = [ - { - name = "winapi-util"; - packageId = "winapi-util"; - target = { target, features }: (target."windows" or false); - } - ]; - - }; - "schannel" = rec { - crateName = "schannel"; - version = "0.1.23"; - edition = "2018"; - sha256 = "0d1m156bsjrws6xzzr1wyfyih9i22mb2csb5pc5kmkrvci2ibjgv"; - authors = [ - "Steven Fackler <sfackler@gmail.com>" - "Steffen Butzer <steffen.butzer@outlook.com>" - ]; - dependencies = [ - { - name = "windows-sys"; - packageId = "windows-sys 0.52.0"; - features = [ "Win32_Foundation" "Win32_Security_Cryptography" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_System_Memory" ]; - } - ]; - devDependencies = [ - { - name = "windows-sys"; - packageId = "windows-sys 0.52.0"; - features = [ "Win32_System_SystemInformation" "Win32_System_Time" ]; - } - ]; - - }; "scopeguard" = rec { crateName = "scopeguard"; version = "1.2.0"; @@ -6873,102 +2905,6 @@ rec { "default" = [ "use_std" ]; }; }; - "sct" = rec { - crateName = "sct"; - version = "0.7.1"; - edition = "2021"; - sha256 = "056lmi2xkzdg1dbai6ha3n57s18cbip4pnmpdhyljli3m99n216s"; - authors = [ - "Joseph Birr-Pixton <jpixton@gmail.com>" - ]; - dependencies = [ - { - name = "ring"; - packageId = "ring"; - } - { - name = "untrusted"; - packageId = "untrusted"; - } - ]; - - }; - "security-framework" = rec { - crateName = "security-framework"; - version = "2.9.2"; - edition = "2021"; - sha256 = "1pplxk15s5yxvi2m1sz5xfmjibp96cscdcl432w9jzbk0frlzdh5"; - authors = [ - "Steven Fackler <sfackler@gmail.com>" - "Kornel <kornel@geekhood.net>" - ]; - dependencies = [ - { - name = "bitflags"; - packageId = "bitflags 1.3.2"; - } - { - name = "core-foundation"; - packageId = "core-foundation"; - } - { - name = "core-foundation-sys"; - packageId = "core-foundation-sys"; - } - { - name = "libc"; - packageId = "libc"; - } - { - name = "security-framework-sys"; - packageId = "security-framework-sys"; - usesDefaultFeatures = false; - } - ]; - features = { - "OSX_10_10" = [ "OSX_10_9" "security-framework-sys/OSX_10_10" ]; - "OSX_10_11" = [ "OSX_10_10" "security-framework-sys/OSX_10_11" ]; - "OSX_10_12" = [ "OSX_10_11" "security-framework-sys/OSX_10_12" ]; - "OSX_10_13" = [ "OSX_10_12" "security-framework-sys/OSX_10_13" "alpn" "session-tickets" "serial-number-bigint" ]; - "OSX_10_14" = [ "OSX_10_13" "security-framework-sys/OSX_10_14" ]; - "OSX_10_15" = [ "OSX_10_14" "security-framework-sys/OSX_10_15" ]; - "OSX_10_9" = [ "security-framework-sys/OSX_10_9" ]; - "default" = [ "OSX_10_9" ]; - "log" = [ "dep:log" ]; - "serial-number-bigint" = [ "dep:num-bigint" ]; - }; - resolvedDefaultFeatures = [ "OSX_10_9" "default" ]; - }; - "security-framework-sys" = rec { - crateName = "security-framework-sys"; - version = "2.9.1"; - edition = "2021"; - sha256 = "0yhciwlsy9dh0ps1gw3197kvyqx1bvc4knrhiznhid6kax196cp9"; - authors = [ - "Steven Fackler <sfackler@gmail.com>" - "Kornel <kornel@geekhood.net>" - ]; - dependencies = [ - { - name = "core-foundation-sys"; - packageId = "core-foundation-sys"; - } - { - name = "libc"; - packageId = "libc"; - } - ]; - features = { - "OSX_10_10" = [ "OSX_10_9" ]; - "OSX_10_11" = [ "OSX_10_10" ]; - "OSX_10_12" = [ "OSX_10_11" ]; - "OSX_10_13" = [ "OSX_10_12" ]; - "OSX_10_14" = [ "OSX_10_13" ]; - "OSX_10_15" = [ "OSX_10_14" ]; - "default" = [ "OSX_10_9" ]; - }; - resolvedDefaultFeatures = [ "OSX_10_9" ]; - }; "semver" = rec { crateName = "semver"; version = "1.0.22"; @@ -7042,7 +2978,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.52"; + packageId = "syn"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" "proc-macro" ]; } @@ -7257,69 +3193,6 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "sled" = rec { - crateName = "sled"; - version = "0.34.7"; - edition = "2018"; - sha256 = "0dcr2s7cylj5mb33ci3kpx7fz797jwvysnl5airrir9cgirv95kz"; - authors = [ - "Tyler Neely <t@jujit.su>" - ]; - dependencies = [ - { - name = "crc32fast"; - packageId = "crc32fast"; - } - { - name = "crossbeam-epoch"; - packageId = "crossbeam-epoch"; - } - { - name = "crossbeam-utils"; - packageId = "crossbeam-utils"; - } - { - name = "fs2"; - packageId = "fs2"; - target = { target, features }: (("linux" == target."os" or null) || ("macos" == target."os" or null) || ("windows" == target."os" or null)); - } - { - name = "fxhash"; - packageId = "fxhash"; - } - { - name = "libc"; - packageId = "libc"; - } - { - name = "log"; - packageId = "log"; - } - { - name = "parking_lot"; - packageId = "parking_lot 0.11.2"; - } - ]; - devDependencies = [ - { - name = "log"; - packageId = "log"; - } - ]; - features = { - "backtrace" = [ "dep:backtrace" ]; - "color-backtrace" = [ "dep:color-backtrace" ]; - "compression" = [ "zstd" ]; - "default" = [ "no_metrics" ]; - "io_uring" = [ "rio" ]; - "no_logs" = [ "log/max_level_off" ]; - "pretty_backtrace" = [ "color-backtrace" ]; - "rio" = [ "dep:rio" ]; - "testing" = [ "event_log" "lock_free_delays" "compression" "failpoints" "backtrace" ]; - "zstd" = [ "dep:zstd" ]; - }; - resolvedDefaultFeatures = [ "default" "no_metrics" ]; - }; "smallvec" = rec { crateName = "smallvec"; version = "1.13.1"; @@ -7336,74 +3209,6 @@ rec { }; resolvedDefaultFeatures = [ "const_generics" "const_new" ]; }; - "snafu" = rec { - crateName = "snafu"; - version = "0.7.5"; - edition = "2018"; - sha256 = "1mj2j2gfbf8mm1hr02zrbrqrh2zp01f61xgkx0lpln2w0ankgpp4"; - authors = [ - "Jake Goulding <jake.goulding@gmail.com>" - ]; - dependencies = [ - { - name = "doc-comment"; - packageId = "doc-comment"; - usesDefaultFeatures = false; - } - { - name = "snafu-derive"; - packageId = "snafu-derive"; - } - ]; - features = { - "backtrace" = [ "dep:backtrace" ]; - "backtraces" = [ "std" "backtrace" ]; - "backtraces-impl-backtrace-crate" = [ "backtraces" ]; - "default" = [ "std" "rust_1_46" ]; - "futures" = [ "futures-core-crate" "pin-project" ]; - "futures-core-crate" = [ "dep:futures-core-crate" ]; - "futures-crate" = [ "dep:futures-crate" ]; - "internal-dev-dependencies" = [ "futures-crate" ]; - "pin-project" = [ "dep:pin-project" ]; - "rust_1_39" = [ "snafu-derive/rust_1_39" ]; - "rust_1_46" = [ "rust_1_39" "snafu-derive/rust_1_46" ]; - "rust_1_61" = [ "rust_1_46" "snafu-derive/rust_1_61" ]; - "unstable-backtraces-impl-std" = [ "backtraces-impl-std" "snafu-derive/unstable-backtraces-impl-std" ]; - "unstable-provider-api" = [ "snafu-derive/unstable-provider-api" ]; - }; - resolvedDefaultFeatures = [ "default" "rust_1_39" "rust_1_46" "std" ]; - }; - "snafu-derive" = rec { - crateName = "snafu-derive"; - version = "0.7.5"; - edition = "2018"; - sha256 = "1gzy9rzggs090zf7hfvgp4lm1glrmg9qzh796686jnq7bxk7j04r"; - procMacro = true; - authors = [ - "Jake Goulding <jake.goulding@gmail.com>" - ]; - dependencies = [ - { - name = "heck"; - packageId = "heck 0.4.1"; - } - { - name = "proc-macro2"; - packageId = "proc-macro2"; - } - { - name = "quote"; - packageId = "quote"; - } - { - name = "syn"; - packageId = "syn 1.0.109"; - features = [ "full" ]; - } - ]; - features = { }; - resolvedDefaultFeatures = [ "rust_1_39" "rust_1_46" ]; - }; "socket2" = rec { crateName = "socket2"; version = "0.5.6"; @@ -7429,31 +3234,6 @@ rec { features = { }; resolvedDefaultFeatures = [ "all" ]; }; - "spin" = rec { - crateName = "spin"; - version = "0.9.8"; - edition = "2015"; - sha256 = "0rvam5r0p3a6qhc18scqpvpgb3ckzyqxpgdfyjnghh8ja7byi039"; - authors = [ - "Mathijs van de Nes <git@mathijs.vd-nes.nl>" - "John Ericson <git@JohnEricson.me>" - "Joshua Barretto <joshua.s.barretto@gmail.com>" - ]; - features = { - "barrier" = [ "mutex" ]; - "default" = [ "lock_api" "mutex" "spin_mutex" "rwlock" "once" "lazy" "barrier" ]; - "fair_mutex" = [ "mutex" ]; - "lazy" = [ "once" ]; - "lock_api" = [ "lock_api_crate" ]; - "lock_api_crate" = [ "dep:lock_api_crate" ]; - "portable-atomic" = [ "dep:portable-atomic" ]; - "portable_atomic" = [ "portable-atomic" ]; - "spin_mutex" = [ "mutex" ]; - "ticket_mutex" = [ "mutex" ]; - "use_ticket_mutex" = [ "mutex" "ticket_mutex" ]; - }; - resolvedDefaultFeatures = [ "once" ]; - }; "spki" = rec { crateName = "spki"; version = "0.7.3"; @@ -7510,41 +3290,7 @@ rec { "default" = [ "std" "i128" ]; }; }; - "syn 1.0.109" = rec { - crateName = "syn"; - version = "1.0.109"; - edition = "2018"; - sha256 = "0ds2if4600bd59wsv7jjgfkayfzy3hnazs394kz6zdkmna8l3dkj"; - authors = [ - "David Tolnay <dtolnay@gmail.com>" - ]; - dependencies = [ - { - name = "proc-macro2"; - packageId = "proc-macro2"; - usesDefaultFeatures = false; - } - { - name = "quote"; - packageId = "quote"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "unicode-ident"; - packageId = "unicode-ident"; - } - ]; - features = { - "default" = [ "derive" "parsing" "printing" "clone-impls" "proc-macro" ]; - "printing" = [ "quote" ]; - "proc-macro" = [ "proc-macro2/proc-macro" "quote/proc-macro" ]; - "quote" = [ "dep:quote" ]; - "test" = [ "syn-test-suite/all-features" ]; - }; - resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "full" "parsing" "printing" "proc-macro" "quote" ]; - }; - "syn 2.0.52" = rec { + "syn" = rec { crateName = "syn"; version = "2.0.52"; edition = "2021"; @@ -7576,7 +3322,7 @@ rec { "quote" = [ "dep:quote" ]; "test" = [ "syn-test-suite/all-features" ]; }; - resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "full" "parsing" "printing" "proc-macro" "quote" "visit" "visit-mut" ]; + resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "full" "parsing" "printing" "proc-macro" "quote" "visit-mut" ]; }; "sync_wrapper" = rec { crateName = "sync_wrapper"; @@ -7591,85 +3337,6 @@ rec { "futures-core" = [ "dep:futures-core" ]; }; }; - "system-configuration" = rec { - crateName = "system-configuration"; - version = "0.5.1"; - edition = "2021"; - sha256 = "1rz0r30xn7fiyqay2dvzfy56cvaa3km74hnbz2d72p97bkf3lfms"; - authors = [ - "Mullvad VPN" - ]; - dependencies = [ - { - name = "bitflags"; - packageId = "bitflags 1.3.2"; - } - { - name = "core-foundation"; - packageId = "core-foundation"; - } - { - name = "system-configuration-sys"; - packageId = "system-configuration-sys"; - } - ]; - - }; - "system-configuration-sys" = rec { - crateName = "system-configuration-sys"; - version = "0.5.0"; - edition = "2021"; - sha256 = "1jckxvdr37bay3i9v52izgy52dg690x5xfg3hd394sv2xf4b2px7"; - authors = [ - "Mullvad VPN" - ]; - dependencies = [ - { - name = "core-foundation-sys"; - packageId = "core-foundation-sys"; - } - { - name = "libc"; - packageId = "libc"; - } - ]; - - }; - "tempfile" = rec { - crateName = "tempfile"; - version = "3.10.1"; - edition = "2021"; - sha256 = "1wdzz35ri168jn9al4s1g2rnsrr5ci91khgarc2rvpb3nappzdw5"; - authors = [ - "Steven Allen <steven@stebalien.com>" - "The Rust Project Developers" - "Ashley Mannix <ashleymannix@live.com.au>" - "Jason White <me@jasonwhite.io>" - ]; - dependencies = [ - { - name = "cfg-if"; - packageId = "cfg-if"; - } - { - name = "fastrand"; - packageId = "fastrand"; - } - { - name = "rustix"; - packageId = "rustix"; - target = { target, features }: ((target."unix" or false) || ("wasi" == target."os" or null)); - features = [ "fs" ]; - } - { - name = "windows-sys"; - packageId = "windows-sys 0.52.0"; - target = { target, features }: (target."windows" or false); - features = [ "Win32_Storage_FileSystem" "Win32_Foundation" ]; - } - ]; - features = { }; - }; "thiserror" = rec { crateName = "thiserror"; version = "1.0.58"; @@ -7706,7 +3373,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.52"; + packageId = "syn"; } ]; @@ -7731,43 +3398,6 @@ rec { ]; features = { }; }; - "tinyvec" = rec { - crateName = "tinyvec"; - version = "1.6.0"; - edition = "2018"; - sha256 = "0l6bl2h62a5m44jdnpn7lmj14rd44via8180i7121fvm73mmrk47"; - authors = [ - "Lokathor <zefria@gmail.com>" - ]; - dependencies = [ - { - name = "tinyvec_macros"; - packageId = "tinyvec_macros"; - optional = true; - } - ]; - features = { - "alloc" = [ "tinyvec_macros" ]; - "arbitrary" = [ "dep:arbitrary" ]; - "real_blackbox" = [ "criterion/real_blackbox" ]; - "rustc_1_55" = [ "rustc_1_40" ]; - "rustc_1_57" = [ "rustc_1_55" ]; - "serde" = [ "dep:serde" ]; - "std" = [ "alloc" ]; - "tinyvec_macros" = [ "dep:tinyvec_macros" ]; - }; - resolvedDefaultFeatures = [ "alloc" "default" "tinyvec_macros" ]; - }; - "tinyvec_macros" = rec { - crateName = "tinyvec_macros"; - version = "0.1.1"; - edition = "2018"; - sha256 = "081gag86208sc3y6sdkshgw3vysm5d34p431dzw0bshz66ncng0z"; - authors = [ - "Soveu <marx.tomasz@gmail.com>" - ]; - - }; "tokio" = rec { crateName = "tokio"; version = "1.36.0"; @@ -7806,7 +3436,7 @@ rec { } { name = "parking_lot"; - packageId = "parking_lot 0.12.1"; + packageId = "parking_lot"; optional = true; } { @@ -7878,34 +3508,6 @@ rec { }; resolvedDefaultFeatures = [ "bytes" "default" "fs" "full" "io-std" "io-util" "libc" "macros" "mio" "net" "num_cpus" "parking_lot" "process" "rt" "rt-multi-thread" "signal" "signal-hook-registry" "socket2" "sync" "test-util" "time" "tokio-macros" "windows-sys" ]; }; - "tokio-io-timeout" = rec { - crateName = "tokio-io-timeout"; - version = "1.2.0"; - edition = "2018"; - sha256 = "1gx84f92q1491vj4pkn81j8pz1s3pgwnbrsdhfsa2556mli41drh"; - authors = [ - "Steven Fackler <sfackler@gmail.com>" - ]; - dependencies = [ - { - name = "pin-project-lite"; - packageId = "pin-project-lite"; - } - { - name = "tokio"; - packageId = "tokio"; - features = [ "time" ]; - } - ]; - devDependencies = [ - { - name = "tokio"; - packageId = "tokio"; - features = [ "full" ]; - } - ]; - - }; "tokio-listener" = rec { crateName = "tokio-listener"; version = "0.3.2"; @@ -7914,7 +3516,7 @@ rec { dependencies = [ { name = "axum"; - packageId = "axum 0.7.4"; + packageId = "axum"; rename = "axum07"; } { @@ -7932,7 +3534,7 @@ rec { } { name = "nix"; - packageId = "nix 0.26.4"; + packageId = "nix"; optional = true; usesDefaultFeatures = false; target = { target, features }: (target."unix" or false); @@ -7960,12 +3562,6 @@ rec { features = [ "net" "codec" ]; } { - name = "tonic"; - packageId = "tonic 0.11.0"; - rename = "tonic"; - optional = true; - } - { name = "tracing"; packageId = "tracing"; } @@ -7994,7 +3590,7 @@ rec { "unix_path_tools" = [ "nix" ]; "user_facing_default" = [ "inetd" "unix" "unix_path_tools" "sd_listen" "socket_options" ]; }; - resolvedDefaultFeatures = [ "default" "inetd" "nix" "sd_listen" "socket2" "socket_options" "tokio-util" "tonic011" "unix" "unix_path_tools" "user_facing_default" ]; + resolvedDefaultFeatures = [ "default" "inetd" "nix" "sd_listen" "socket2" "socket_options" "tokio-util" "unix" "unix_path_tools" "user_facing_default" ]; }; "tokio-macros" = rec { crateName = "tokio-macros"; @@ -8016,80 +3612,12 @@ rec { } { name = "syn"; - packageId = "syn 2.0.52"; + packageId = "syn"; features = [ "full" ]; } ]; }; - "tokio-rustls 0.24.1" = rec { - crateName = "tokio-rustls"; - version = "0.24.1"; - edition = "2018"; - sha256 = "10bhibg57mqir7xjhb2xmf24xgfpx6fzpyw720a4ih8a737jg0y2"; - dependencies = [ - { - name = "rustls"; - packageId = "rustls 0.21.10"; - usesDefaultFeatures = false; - } - { - name = "tokio"; - packageId = "tokio"; - } - ]; - devDependencies = [ - { - name = "tokio"; - packageId = "tokio"; - features = [ "full" ]; - } - ]; - features = { - "dangerous_configuration" = [ "rustls/dangerous_configuration" ]; - "default" = [ "logging" "tls12" ]; - "logging" = [ "rustls/logging" ]; - "secret_extraction" = [ "rustls/secret_extraction" ]; - "tls12" = [ "rustls/tls12" ]; - }; - resolvedDefaultFeatures = [ "default" "logging" "tls12" ]; - }; - "tokio-rustls 0.25.0" = rec { - crateName = "tokio-rustls"; - version = "0.25.0"; - edition = "2021"; - sha256 = "03w6d5aqqf084rmcmrsyq5grhydl53blaiqcl0i2yfnv187hqpkp"; - dependencies = [ - { - name = "rustls"; - packageId = "rustls 0.22.2"; - usesDefaultFeatures = false; - } - { - name = "rustls-pki-types"; - packageId = "rustls-pki-types"; - rename = "pki-types"; - } - { - name = "tokio"; - packageId = "tokio"; - } - ]; - devDependencies = [ - { - name = "tokio"; - packageId = "tokio"; - features = [ "full" ]; - } - ]; - features = { - "default" = [ "logging" "tls12" "ring" ]; - "logging" = [ "rustls/logging" ]; - "ring" = [ "rustls/ring" ]; - "tls12" = [ "rustls/tls12" ]; - }; - resolvedDefaultFeatures = [ "default" "logging" "ring" "tls12" ]; - }; "tokio-stream" = rec { crateName = "tokio-stream"; version = "0.1.14"; @@ -8131,7 +3659,7 @@ rec { "time" = [ "tokio/time" ]; "tokio-util" = [ "dep:tokio-util" ]; }; - resolvedDefaultFeatures = [ "default" "fs" "net" "time" ]; + resolvedDefaultFeatures = [ "default" "time" ]; }; "tokio-test" = rec { crateName = "tokio-test"; @@ -8191,11 +3719,6 @@ rec { packageId = "futures-core"; } { - name = "futures-io"; - packageId = "futures-io"; - optional = true; - } - { name = "futures-sink"; packageId = "futures-sink"; } @@ -8238,378 +3761,7 @@ rec { "time" = [ "tokio/time" "slab" ]; "tracing" = [ "dep:tracing" ]; }; - resolvedDefaultFeatures = [ "codec" "compat" "default" "futures-io" "io" "io-util" "net" "tracing" ]; - }; - "tonic 0.11.0" = rec { - crateName = "tonic"; - version = "0.11.0"; - edition = "2021"; - sha256 = "04qsr527i256i3dk9dp1g2jr42q7yl91y5h06rvd9ycy9rxfpi3n"; - authors = [ - "Lucio Franco <luciofranco14@gmail.com>" - ]; - dependencies = [ - { - name = "async-stream"; - packageId = "async-stream"; - optional = true; - } - { - name = "async-trait"; - packageId = "async-trait"; - optional = true; - } - { - name = "axum"; - packageId = "axum 0.6.20"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "base64"; - packageId = "base64"; - } - { - name = "bytes"; - packageId = "bytes"; - } - { - name = "h2"; - packageId = "h2 0.3.24"; - optional = true; - } - { - name = "http"; - packageId = "http 0.2.12"; - } - { - name = "http-body"; - packageId = "http-body 0.4.6"; - } - { - name = "hyper"; - packageId = "hyper 0.14.28"; - optional = true; - features = [ "full" ]; - } - { - name = "hyper-timeout"; - packageId = "hyper-timeout"; - optional = true; - } - { - name = "percent-encoding"; - packageId = "percent-encoding"; - } - { - name = "pin-project"; - packageId = "pin-project"; - } - { - name = "prost"; - packageId = "prost 0.12.3"; - optional = true; - usesDefaultFeatures = false; - features = [ "std" ]; - } - { - name = "rustls-native-certs"; - packageId = "rustls-native-certs 0.7.0"; - optional = true; - } - { - name = "rustls-pemfile"; - packageId = "rustls-pemfile 2.1.1"; - optional = true; - } - { - name = "rustls-pki-types"; - packageId = "rustls-pki-types"; - optional = true; - } - { - name = "tokio"; - packageId = "tokio"; - } - { - name = "tokio-rustls"; - packageId = "tokio-rustls 0.25.0"; - optional = true; - } - { - name = "tokio-stream"; - packageId = "tokio-stream"; - } - { - name = "tower"; - packageId = "tower"; - optional = true; - usesDefaultFeatures = false; - features = [ "balance" "buffer" "discover" "limit" "load" "make" "timeout" "util" ]; - } - { - name = "tower-layer"; - packageId = "tower-layer"; - } - { - name = "tower-service"; - packageId = "tower-service"; - } - { - name = "tracing"; - packageId = "tracing"; - } - ]; - devDependencies = [ - { - name = "tokio"; - packageId = "tokio"; - features = [ "rt" "macros" ]; - } - { - name = "tower"; - packageId = "tower"; - features = [ "full" ]; - } - ]; - features = { - "codegen" = [ "dep:async-trait" ]; - "default" = [ "transport" "codegen" "prost" ]; - "gzip" = [ "dep:flate2" ]; - "prost" = [ "dep:prost" ]; - "tls" = [ "dep:rustls-pki-types" "dep:rustls-pemfile" "transport" "dep:tokio-rustls" "tokio/rt" "tokio/macros" ]; - "tls-roots" = [ "tls-roots-common" "dep:rustls-native-certs" ]; - "tls-roots-common" = [ "tls" ]; - "tls-webpki-roots" = [ "tls-roots-common" "dep:webpki-roots" ]; - "transport" = [ "dep:async-stream" "dep:axum" "channel" "dep:h2" "dep:hyper" "tokio/net" "tokio/time" "dep:tower" "dep:hyper-timeout" ]; - "zstd" = [ "dep:zstd" ]; - }; - resolvedDefaultFeatures = [ "channel" "codegen" "default" "prost" "tls" "tls-roots" "tls-roots-common" "transport" ]; - }; - "tonic 0.9.2" = rec { - crateName = "tonic"; - version = "0.9.2"; - edition = "2021"; - sha256 = "0nlx35lvah5hdcp6lg1d6dlprq0zz8ijj6f727szfcv479m6d0ih"; - authors = [ - "Lucio Franco <luciofranco14@gmail.com>" - ]; - dependencies = [ - { - name = "async-trait"; - packageId = "async-trait"; - optional = true; - } - { - name = "axum"; - packageId = "axum 0.6.20"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "base64"; - packageId = "base64"; - } - { - name = "bytes"; - packageId = "bytes"; - } - { - name = "futures-core"; - packageId = "futures-core"; - usesDefaultFeatures = false; - } - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - } - { - name = "h2"; - packageId = "h2 0.3.24"; - optional = true; - } - { - name = "http"; - packageId = "http 0.2.12"; - } - { - name = "http-body"; - packageId = "http-body 0.4.6"; - } - { - name = "hyper"; - packageId = "hyper 0.14.28"; - optional = true; - features = [ "full" ]; - } - { - name = "hyper-timeout"; - packageId = "hyper-timeout"; - optional = true; - } - { - name = "percent-encoding"; - packageId = "percent-encoding"; - } - { - name = "pin-project"; - packageId = "pin-project"; - } - { - name = "prost"; - packageId = "prost 0.11.9"; - optional = true; - usesDefaultFeatures = false; - features = [ "std" ]; - } - { - name = "tokio"; - packageId = "tokio"; - optional = true; - features = [ "net" "time" "macros" ]; - } - { - name = "tokio-stream"; - packageId = "tokio-stream"; - } - { - name = "tower"; - packageId = "tower"; - optional = true; - usesDefaultFeatures = false; - features = [ "balance" "buffer" "discover" "limit" "load" "make" "timeout" "util" ]; - } - { - name = "tower-layer"; - packageId = "tower-layer"; - } - { - name = "tower-service"; - packageId = "tower-service"; - } - { - name = "tracing"; - packageId = "tracing"; - } - ]; - devDependencies = [ - { - name = "tokio"; - packageId = "tokio"; - features = [ "rt" "macros" ]; - } - { - name = "tower"; - packageId = "tower"; - features = [ "full" ]; - } - ]; - features = { - "channel" = [ "dep:h2" "dep:hyper" "dep:tokio" "dep:tower" "dep:hyper-timeout" ]; - "codegen" = [ "dep:async-trait" ]; - "default" = [ "transport" "codegen" "prost" ]; - "gzip" = [ "dep:flate2" ]; - "prost" = [ "dep:prost" ]; - "tls" = [ "dep:rustls-pemfile" "transport" "dep:tokio-rustls" "dep:async-stream" ]; - "tls-roots" = [ "tls-roots-common" "dep:rustls-native-certs" ]; - "tls-roots-common" = [ "tls" ]; - "tls-webpki-roots" = [ "tls-roots-common" "dep:webpki-roots" ]; - "transport" = [ "dep:axum" "channel" ]; - }; - resolvedDefaultFeatures = [ "channel" "codegen" "default" "prost" "transport" ]; - }; - "tonic-build" = rec { - crateName = "tonic-build"; - version = "0.11.0"; - edition = "2021"; - sha256 = "1hm99ckaw0pzq8h22bdjy6gpbg06kpvs0f73nj60f456f3fzckmy"; - authors = [ - "Lucio Franco <luciofranco14@gmail.com>" - ]; - dependencies = [ - { - name = "prettyplease"; - packageId = "prettyplease"; - } - { - name = "proc-macro2"; - packageId = "proc-macro2"; - } - { - name = "prost-build"; - packageId = "prost-build"; - optional = true; - } - { - name = "quote"; - packageId = "quote"; - } - { - name = "syn"; - packageId = "syn 2.0.52"; - } - ]; - features = { - "cleanup-markdown" = [ "prost" "prost-build/cleanup-markdown" ]; - "default" = [ "transport" "prost" ]; - "prost" = [ "prost-build" ]; - "prost-build" = [ "dep:prost-build" ]; - }; - resolvedDefaultFeatures = [ "default" "prost" "prost-build" "transport" ]; - }; - "tonic-reflection" = rec { - crateName = "tonic-reflection"; - version = "0.11.0"; - edition = "2021"; - sha256 = "19xn3kyg062d7y1cnw537v9cxkzzcrnfri0jb29fbyn0smxj532l"; - authors = [ - "James Nugent <james@jen20.com>" - "Samani G. Gikandi <samani@gojulas.com>" - ]; - dependencies = [ - { - name = "prost"; - packageId = "prost 0.12.3"; - } - { - name = "prost-types"; - packageId = "prost-types"; - optional = true; - } - { - name = "tokio"; - packageId = "tokio"; - optional = true; - features = [ "sync" "rt" ]; - } - { - name = "tokio-stream"; - packageId = "tokio-stream"; - optional = true; - features = [ "net" ]; - } - { - name = "tonic"; - packageId = "tonic 0.11.0"; - usesDefaultFeatures = false; - features = [ "codegen" "prost" ]; - } - ]; - devDependencies = [ - { - name = "tonic"; - packageId = "tonic 0.11.0"; - usesDefaultFeatures = false; - features = [ "transport" ]; - } - ]; - features = { - "default" = [ "server" ]; - "prost-types" = [ "dep:prost-types" ]; - "server" = [ "prost-types" "dep:tokio" "dep:tokio-stream" ]; - }; - resolvedDefaultFeatures = [ "default" "prost-types" "server" ]; + resolvedDefaultFeatures = [ "codec" "default" "io" "net" "tracing" ]; }; "tower" = rec { crateName = "tower"; @@ -8633,11 +3785,6 @@ rec { features = [ "alloc" ]; } { - name = "indexmap"; - packageId = "indexmap 1.9.3"; - optional = true; - } - { name = "pin-project"; packageId = "pin-project"; optional = true; @@ -8648,29 +3795,12 @@ rec { optional = true; } { - name = "rand"; - packageId = "rand"; - optional = true; - features = [ "small_rng" ]; - } - { - name = "slab"; - packageId = "slab"; - optional = true; - } - { name = "tokio"; packageId = "tokio"; optional = true; features = [ "sync" ]; } { - name = "tokio-util"; - packageId = "tokio-util"; - optional = true; - usesDefaultFeatures = false; - } - { name = "tower-layer"; packageId = "tower-layer"; } @@ -8730,7 +3860,7 @@ rec { "tracing" = [ "dep:tracing" ]; "util" = [ "__common" "futures-util" "pin-project" ]; }; - resolvedDefaultFeatures = [ "__common" "balance" "buffer" "default" "discover" "futures-core" "futures-util" "indexmap" "limit" "load" "log" "make" "pin-project" "pin-project-lite" "rand" "ready-cache" "slab" "timeout" "tokio" "tokio-util" "tracing" "util" ]; + resolvedDefaultFeatures = [ "__common" "futures-core" "futures-util" "log" "make" "pin-project" "pin-project-lite" "tokio" "tracing" "util" ]; }; "tower-layer" = rec { crateName = "tower-layer"; @@ -8821,7 +3951,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.52"; + packageId = "syn"; usesDefaultFeatures = false; features = [ "full" "parsing" "printing" "visit-mut" "clone-impls" "extra-traits" "proc-macro" ]; } @@ -8889,129 +4019,6 @@ rec { }; resolvedDefaultFeatures = [ "log-tracer" "std" ]; }; - "tracing-opentelemetry" = rec { - crateName = "tracing-opentelemetry"; - version = "0.22.0"; - edition = "2018"; - sha256 = "15jmwmbp6wz15bx20bfsmabx53wmlnd7wvzwz9hvkrq7aifc4yn6"; - authors = [ - "Julian Tescher <julian@tescher.me>" - "Tokio Contributors <team@tokio.rs>" - ]; - dependencies = [ - { - name = "js-sys"; - packageId = "js-sys"; - target = { target, features }: (("wasm32" == target."arch" or null) && (!("wasi" == target."os" or null))); - } - { - name = "once_cell"; - packageId = "once_cell"; - } - { - name = "opentelemetry"; - packageId = "opentelemetry"; - usesDefaultFeatures = false; - features = [ "trace" ]; - } - { - name = "opentelemetry_sdk"; - packageId = "opentelemetry_sdk"; - usesDefaultFeatures = false; - features = [ "trace" ]; - } - { - name = "smallvec"; - packageId = "smallvec"; - optional = true; - } - { - name = "tracing"; - packageId = "tracing"; - usesDefaultFeatures = false; - features = [ "std" ]; - } - { - name = "tracing-core"; - packageId = "tracing-core"; - } - { - name = "tracing-log"; - packageId = "tracing-log"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "tracing-subscriber"; - packageId = "tracing-subscriber"; - usesDefaultFeatures = false; - features = [ "registry" "std" ]; - } - { - name = "web-time"; - packageId = "web-time"; - target = { target, features }: (("wasm32" == target."arch" or null) && (!("wasi" == target."os" or null))); - } - ]; - devDependencies = [ - { - name = "opentelemetry"; - packageId = "opentelemetry"; - features = [ "trace" "metrics" ]; - } - { - name = "opentelemetry_sdk"; - packageId = "opentelemetry_sdk"; - usesDefaultFeatures = false; - features = [ "trace" "rt-tokio" ]; - } - { - name = "tracing"; - packageId = "tracing"; - usesDefaultFeatures = false; - features = [ "std" "attributes" ]; - } - { - name = "tracing-subscriber"; - packageId = "tracing-subscriber"; - usesDefaultFeatures = false; - features = [ "registry" "std" "fmt" ]; - } - ]; - features = { - "async-trait" = [ "dep:async-trait" ]; - "default" = [ "tracing-log" "metrics" ]; - "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" "smallvec" ]; - "smallvec" = [ "dep:smallvec" ]; - "thiserror" = [ "dep:thiserror" ]; - "tracing-log" = [ "dep:tracing-log" ]; - }; - resolvedDefaultFeatures = [ "default" "metrics" "smallvec" "tracing-log" ]; - }; - "tracing-serde" = rec { - crateName = "tracing-serde"; - version = "0.1.3"; - edition = "2018"; - sha256 = "1qfr0va69djvxqvjrx4vqq7p6myy414lx4w1f6amcn0hfwqj2sxw"; - authors = [ - "Tokio Contributors <team@tokio.rs>" - ]; - dependencies = [ - { - name = "serde"; - packageId = "serde"; - } - { - name = "tracing-core"; - packageId = "tracing-core"; - } - ]; - features = { - "valuable" = [ "valuable_crate" "valuable-serde" "tracing-core/valuable" ]; - "valuable-serde" = [ "dep:valuable-serde" ]; - "valuable_crate" = [ "dep:valuable_crate" ]; - }; - }; "tracing-subscriber" = rec { crateName = "tracing-subscriber"; version = "0.3.18"; @@ -9024,38 +4031,11 @@ rec { ]; dependencies = [ { - name = "matchers"; - packageId = "matchers"; - optional = true; - } - { name = "nu-ansi-term"; packageId = "nu-ansi-term"; optional = true; } { - name = "once_cell"; - packageId = "once_cell"; - optional = true; - } - { - name = "regex"; - packageId = "regex"; - optional = true; - usesDefaultFeatures = false; - features = [ "std" "unicode-case" "unicode-perl" ]; - } - { - name = "serde"; - packageId = "serde"; - optional = true; - } - { - name = "serde_json"; - packageId = "serde_json"; - optional = true; - } - { name = "sharded-slab"; packageId = "sharded-slab"; optional = true; @@ -9071,12 +4051,6 @@ rec { optional = true; } { - name = "tracing"; - packageId = "tracing"; - optional = true; - usesDefaultFeatures = false; - } - { name = "tracing-core"; packageId = "tracing-core"; usesDefaultFeatures = false; @@ -9088,24 +4062,9 @@ rec { usesDefaultFeatures = false; features = [ "log-tracer" "std" ]; } - { - name = "tracing-serde"; - packageId = "tracing-serde"; - optional = true; - } ]; devDependencies = [ { - name = "regex"; - packageId = "regex"; - usesDefaultFeatures = false; - features = [ "std" ]; - } - { - name = "tracing"; - packageId = "tracing"; - } - { name = "tracing-log"; packageId = "tracing-log"; } @@ -9138,169 +4097,7 @@ rec { "valuable-serde" = [ "dep:valuable-serde" ]; "valuable_crate" = [ "dep:valuable_crate" ]; }; - resolvedDefaultFeatures = [ "alloc" "ansi" "default" "env-filter" "fmt" "json" "matchers" "nu-ansi-term" "once_cell" "regex" "registry" "serde" "serde_json" "sharded-slab" "smallvec" "std" "thread_local" "tracing" "tracing-log" "tracing-serde" ]; - }; - "try-lock" = rec { - crateName = "try-lock"; - version = "0.2.5"; - edition = "2015"; - sha256 = "0jqijrrvm1pyq34zn1jmy2vihd4jcrjlvsh4alkjahhssjnsn8g4"; - authors = [ - "Sean McArthur <sean@seanmonstar.com>" - ]; - - }; - "tvix-castore" = rec { - crateName = "tvix-castore"; - version = "0.1.0"; - edition = "2021"; - # We can't filter paths with references in Nix 2.4 - # See https://github.com/NixOS/nix/issues/5410 - src = - if ((lib.versionOlder builtins.nixVersion "2.4pre20211007") || (lib.versionOlder "2.5" builtins.nixVersion)) - then lib.cleanSourceWith { filter = sourceFilter; src = ../../../tvix/castore; } - else ../../../tvix/castore; - dependencies = [ - { - name = "async-stream"; - packageId = "async-stream"; - } - { - name = "async-tempfile"; - packageId = "async-tempfile"; - } - { - name = "blake3"; - packageId = "blake3"; - features = [ "rayon" "std" "traits-preview" ]; - } - { - name = "bstr"; - packageId = "bstr"; - } - { - name = "bytes"; - packageId = "bytes"; - } - { - name = "data-encoding"; - packageId = "data-encoding"; - } - { - name = "digest"; - packageId = "digest"; - } - { - name = "fastcdc"; - packageId = "fastcdc"; - features = [ "tokio" ]; - } - { - name = "fuse-backend-rs"; - packageId = "fuse-backend-rs"; - optional = true; - } - { - name = "futures"; - packageId = "futures"; - } - { - name = "lazy_static"; - packageId = "lazy_static"; - } - { - name = "libc"; - packageId = "libc"; - optional = true; - } - { - name = "object_store"; - packageId = "object_store"; - features = [ "http" ]; - } - { - name = "parking_lot"; - packageId = "parking_lot 0.12.1"; - } - { - name = "pin-project-lite"; - packageId = "pin-project-lite"; - } - { - name = "prost"; - packageId = "prost 0.12.3"; - } - { - name = "sled"; - packageId = "sled"; - } - { - name = "thiserror"; - packageId = "thiserror"; - } - { - name = "tokio"; - packageId = "tokio"; - features = [ "fs" "macros" "net" "rt" "rt-multi-thread" "signal" ]; - } - { - name = "tokio-stream"; - packageId = "tokio-stream"; - features = [ "fs" "net" ]; - } - { - name = "tokio-util"; - packageId = "tokio-util"; - features = [ "io" "io-util" ]; - } - { - name = "tonic"; - packageId = "tonic 0.11.0"; - } - { - name = "tonic-reflection"; - packageId = "tonic-reflection"; - optional = true; - } - { - name = "tower"; - packageId = "tower"; - } - { - name = "tracing"; - packageId = "tracing"; - } - { - name = "url"; - packageId = "url"; - } - { - name = "walkdir"; - packageId = "walkdir"; - } - { - name = "zstd"; - packageId = "zstd"; - } - ]; - buildDependencies = [ - { - name = "prost-build"; - packageId = "prost-build"; - } - { - name = "tonic-build"; - packageId = "tonic-build"; - } - ]; - features = { - "cloud" = [ "object_store/aws" "object_store/azure" "object_store/gcp" ]; - "fs" = [ "dep:libc" "dep:fuse-backend-rs" ]; - "fuse" = [ "fs" ]; - "tonic-reflection" = [ "dep:tonic-reflection" ]; - "virtiofs" = [ "fs" "dep:vhost" "dep:vhost-user-backend" "dep:virtio-queue" "dep:vm-memory" "dep:vmm-sys-util" "dep:virtio-bindings" "fuse-backend-rs?/vhost-user-fs" "fuse-backend-rs?/virtiofs" ]; - }; - resolvedDefaultFeatures = [ "cloud" "default" "fs" "fuse" "tonic-reflection" ]; + resolvedDefaultFeatures = [ "alloc" "ansi" "default" "fmt" "nu-ansi-term" "registry" "sharded-slab" "smallvec" "std" "thread_local" "tracing-log" ]; }; "tvix-daemon" = rec { crateName = "tvix-daemon"; @@ -9321,17 +4118,14 @@ rec { else ./.; dependencies = [ { - name = "anyhow"; - packageId = "anyhow"; - } - { name = "clap"; packageId = "clap"; - features = [ "derive" ]; + features = [ "derive" "env" ]; } { name = "nix-compat"; packageId = "nix-compat"; + features = [ "wire" ]; } { name = "tokio"; @@ -9350,10 +4144,6 @@ rec { name = "tracing-subscriber"; packageId = "tracing-subscriber"; } - { - name = "tvix-store"; - packageId = "tvix-store"; - } ]; devDependencies = [ { @@ -9363,195 +4153,6 @@ rec { ]; }; - "tvix-store" = rec { - crateName = "tvix-store"; - version = "0.1.0"; - edition = "2021"; - crateBin = [ ]; - # We can't filter paths with references in Nix 2.4 - # See https://github.com/NixOS/nix/issues/5410 - src = - if ((lib.versionOlder builtins.nixVersion "2.4pre20211007") || (lib.versionOlder "2.5" builtins.nixVersion)) - then lib.cleanSourceWith { filter = sourceFilter; src = ../../../tvix/store; } - else ../../../tvix/store; - dependencies = [ - { - name = "anyhow"; - packageId = "anyhow"; - } - { - name = "async-recursion"; - packageId = "async-recursion"; - } - { - name = "async-stream"; - packageId = "async-stream"; - } - { - name = "blake3"; - packageId = "blake3"; - features = [ "rayon" "std" ]; - } - { - name = "bstr"; - packageId = "bstr"; - } - { - name = "bytes"; - packageId = "bytes"; - } - { - name = "clap"; - packageId = "clap"; - features = [ "derive" "env" ]; - } - { - name = "count-write"; - packageId = "count-write"; - } - { - name = "data-encoding"; - packageId = "data-encoding"; - } - { - name = "futures"; - packageId = "futures"; - } - { - name = "lazy_static"; - packageId = "lazy_static"; - } - { - name = "nix-compat"; - packageId = "nix-compat"; - features = [ "async" ]; - } - { - name = "opentelemetry"; - packageId = "opentelemetry"; - optional = true; - } - { - name = "opentelemetry-otlp"; - packageId = "opentelemetry-otlp"; - optional = true; - } - { - name = "opentelemetry_sdk"; - packageId = "opentelemetry_sdk"; - optional = true; - features = [ "rt-tokio" ]; - } - { - name = "pin-project-lite"; - packageId = "pin-project-lite"; - } - { - name = "prost"; - packageId = "prost 0.12.3"; - } - { - name = "reqwest"; - packageId = "reqwest"; - usesDefaultFeatures = false; - features = [ "rustls-tls-native-roots" "stream" ]; - } - { - name = "sha2"; - packageId = "sha2"; - } - { - name = "sled"; - packageId = "sled"; - } - { - name = "thiserror"; - packageId = "thiserror"; - } - { - name = "tokio"; - packageId = "tokio"; - features = [ "fs" "macros" "net" "rt" "rt-multi-thread" "signal" ]; - } - { - name = "tokio-listener"; - packageId = "tokio-listener"; - features = [ "tonic011" ]; - } - { - name = "tokio-stream"; - packageId = "tokio-stream"; - features = [ "fs" ]; - } - { - name = "tokio-util"; - packageId = "tokio-util"; - features = [ "io" "io-util" "compat" ]; - } - { - name = "tonic"; - packageId = "tonic 0.11.0"; - features = [ "tls" "tls-roots" ]; - } - { - name = "tonic-reflection"; - packageId = "tonic-reflection"; - optional = true; - } - { - name = "tower"; - packageId = "tower"; - } - { - name = "tracing"; - packageId = "tracing"; - } - { - name = "tracing-opentelemetry"; - packageId = "tracing-opentelemetry"; - } - { - name = "tracing-subscriber"; - packageId = "tracing-subscriber"; - features = [ "env-filter" "json" ]; - } - { - name = "tvix-castore"; - packageId = "tvix-castore"; - } - { - name = "url"; - packageId = "url"; - } - { - name = "walkdir"; - packageId = "walkdir"; - } - { - name = "xz2"; - packageId = "xz2"; - } - ]; - buildDependencies = [ - { - name = "prost-build"; - packageId = "prost-build"; - } - { - name = "tonic-build"; - packageId = "tonic-build"; - } - ]; - features = { - "cloud" = [ "tvix-castore/cloud" ]; - "default" = [ "cloud" "fuse" "otlp" "tonic-reflection" ]; - "fuse" = [ "tvix-castore/fuse" ]; - "otlp" = [ "dep:opentelemetry" "dep:opentelemetry-otlp" "dep:opentelemetry_sdk" ]; - "tonic-reflection" = [ "dep:tonic-reflection" "tvix-castore/tonic-reflection" ]; - "virtiofs" = [ "tvix-castore/virtiofs" ]; - }; - resolvedDefaultFeatures = [ "cloud" "default" "fuse" "otlp" "tonic-reflection" ]; - }; "typenum" = rec { crateName = "typenum"; version = "1.17.0"; @@ -9567,25 +4168,6 @@ rec { "scale_info" = [ "scale-info/derive" ]; }; }; - "unicode-bidi" = rec { - crateName = "unicode-bidi"; - version = "0.3.15"; - edition = "2018"; - sha256 = "0xcdxm7h0ydyprwpcbh436rbs6s6lph7f3gr527lzgv6lw053y88"; - libName = "unicode_bidi"; - authors = [ - "The Servo Project Developers" - ]; - features = { - "default" = [ "std" "hardcoded-data" ]; - "flame" = [ "dep:flame" ]; - "flame_it" = [ "flame" "flamer" ]; - "flamer" = [ "dep:flamer" ]; - "serde" = [ "dep:serde" ]; - "with_serde" = [ "serde" ]; - }; - resolvedDefaultFeatures = [ "hardcoded-data" "std" ]; - }; "unicode-ident" = rec { crateName = "unicode-ident"; version = "1.0.12"; @@ -9596,75 +4178,6 @@ rec { ]; }; - "unicode-normalization" = rec { - crateName = "unicode-normalization"; - version = "0.1.23"; - edition = "2018"; - sha256 = "1x81a50h2zxigj74b9bqjsirxxbyhmis54kg600xj213vf31cvd5"; - authors = [ - "kwantam <kwantam@gmail.com>" - "Manish Goregaokar <manishsmail@gmail.com>" - ]; - dependencies = [ - { - name = "tinyvec"; - packageId = "tinyvec"; - features = [ "alloc" ]; - } - ]; - features = { - "default" = [ "std" ]; - }; - resolvedDefaultFeatures = [ "std" ]; - }; - "untrusted" = rec { - crateName = "untrusted"; - version = "0.9.0"; - edition = "2018"; - sha256 = "1ha7ib98vkc538x0z60gfn0fc5whqdd85mb87dvisdcaifi6vjwf"; - authors = [ - "Brian Smith <brian@briansmith.org>" - ]; - - }; - "url" = rec { - crateName = "url"; - version = "2.5.0"; - edition = "2018"; - sha256 = "0cs65961miawncdg2z20171w0vqrmraswv2ihdpd8lxp7cp31rii"; - authors = [ - "The rust-url developers" - ]; - dependencies = [ - { - name = "form_urlencoded"; - packageId = "form_urlencoded"; - } - { - name = "idna"; - packageId = "idna"; - } - { - name = "percent-encoding"; - packageId = "percent-encoding"; - } - ]; - features = { - "serde" = [ "dep:serde" ]; - }; - resolvedDefaultFeatures = [ "default" ]; - }; - "urlencoding" = rec { - crateName = "urlencoding"; - version = "2.1.3"; - edition = "2021"; - sha256 = "1nj99jp37k47n0hvaz5fvz7z6jd0sb4ppvfy3nphr1zbnyixpy6s"; - authors = [ - "Kornel <kornel@geekhood.net>" - "Bertram Truong <b@bertramtruong.com>" - ]; - - }; "utf8parse" = rec { crateName = "utf8parse"; version = "0.2.1"; @@ -9677,48 +4190,6 @@ rec { features = { }; resolvedDefaultFeatures = [ "default" ]; }; - "uuid" = rec { - crateName = "uuid"; - version = "1.7.0"; - edition = "2018"; - sha256 = "0aivp5ys7sg2izlj2sn6rr8p43vdcwg64naj8n0kqbd15iqcj37h"; - authors = [ - "Ashley Mannix<ashleymannix@live.com.au>" - "Christopher Armstrong" - "Dylan DPC<dylan.dpc@gmail.com>" - "Hunar Roop Kahlon<hunar.roop@gmail.com>" - ]; - dependencies = [ - { - name = "getrandom"; - packageId = "getrandom"; - optional = true; - } - ]; - features = { - "arbitrary" = [ "dep:arbitrary" ]; - "atomic" = [ "dep:atomic" ]; - "borsh" = [ "dep:borsh" "dep:borsh-derive" ]; - "bytemuck" = [ "dep:bytemuck" ]; - "default" = [ "std" ]; - "fast-rng" = [ "rng" "dep:rand" ]; - "js" = [ "dep:wasm-bindgen" "getrandom?/js" ]; - "macro-diagnostics" = [ "dep:uuid-macro-internal" ]; - "md5" = [ "dep:md-5" ]; - "rng" = [ "dep:getrandom" ]; - "serde" = [ "dep:serde" ]; - "sha1" = [ "dep:sha1_smol" ]; - "slog" = [ "dep:slog" ]; - "v1" = [ "atomic" ]; - "v3" = [ "md5" ]; - "v4" = [ "rng" ]; - "v5" = [ "sha1" ]; - "v6" = [ "atomic" ]; - "v7" = [ "atomic" "rng" ]; - "zerocopy" = [ "dep:zerocopy" ]; - }; - resolvedDefaultFeatures = [ "default" "rng" "std" "v4" ]; - }; "valuable" = rec { crateName = "valuable"; version = "0.1.0"; @@ -9742,94 +4213,6 @@ rec { ]; }; - "vm-memory" = rec { - crateName = "vm-memory"; - version = "0.10.0"; - edition = "2021"; - sha256 = "0z423a8i4s3addq4yjad4ar5l6qwarjwdn94lismbd0mcqv712k8"; - authors = [ - "Liu Jiang <gerry@linux.alibaba.com>" - ]; - dependencies = [ - { - name = "libc"; - packageId = "libc"; - } - { - name = "winapi"; - packageId = "winapi"; - target = { target, features }: (target."windows" or false); - features = [ "errhandlingapi" "sysinfoapi" ]; - } - ]; - features = { - "arc-swap" = [ "dep:arc-swap" ]; - "backend-atomic" = [ "arc-swap" ]; - }; - resolvedDefaultFeatures = [ "backend-mmap" "default" ]; - }; - "vmm-sys-util" = rec { - crateName = "vmm-sys-util"; - version = "0.11.2"; - edition = "2021"; - sha256 = "0a9azxk6wsahwkggshbdga4jdryzfw6j5r21f11gf50j4f2b1ds8"; - authors = [ - "Intel Virtualization Team <vmm-maintainers@intel.com>" - ]; - dependencies = [ - { - name = "bitflags"; - packageId = "bitflags 1.3.2"; - target = { target, features }: (("linux" == target."os" or null) || ("android" == target."os" or null)); - } - { - name = "libc"; - packageId = "libc"; - } - ]; - features = { - "serde" = [ "dep:serde" ]; - "serde_derive" = [ "dep:serde_derive" ]; - "with-serde" = [ "serde" "serde_derive" ]; - }; - }; - "walkdir" = rec { - crateName = "walkdir"; - version = "2.5.0"; - edition = "2018"; - sha256 = "0jsy7a710qv8gld5957ybrnc07gavppp963gs32xk4ag8130jy99"; - authors = [ - "Andrew Gallant <jamslam@gmail.com>" - ]; - dependencies = [ - { - name = "same-file"; - packageId = "same-file"; - } - { - name = "winapi-util"; - packageId = "winapi-util"; - target = { target, features }: (target."windows" or false); - } - ]; - - }; - "want" = rec { - crateName = "want"; - version = "0.3.1"; - edition = "2018"; - sha256 = "03hbfrnvqqdchb5kgxyavb9jabwza0dmh2vw5kg0dq8rxl57d9xz"; - authors = [ - "Sean McArthur <sean@seanmonstar.com>" - ]; - dependencies = [ - { - name = "try-lock"; - packageId = "try-lock"; - } - ]; - - }; "wasi" = rec { crateName = "wasi"; version = "0.11.0+wasi-snapshot-preview1"; @@ -9847,753 +4230,6 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "wasm-bindgen" = rec { - crateName = "wasm-bindgen"; - version = "0.2.92"; - edition = "2018"; - sha256 = "1a4mcw13nsk3fr8fxjzf9kk1wj88xkfsmnm0pjraw01ryqfm7qjb"; - authors = [ - "The wasm-bindgen Developers" - ]; - dependencies = [ - { - name = "cfg-if"; - packageId = "cfg-if"; - } - { - name = "wasm-bindgen-macro"; - packageId = "wasm-bindgen-macro"; - } - ]; - features = { - "default" = [ "spans" "std" ]; - "enable-interning" = [ "std" ]; - "gg-alloc" = [ "wasm-bindgen-test/gg-alloc" ]; - "serde" = [ "dep:serde" ]; - "serde-serialize" = [ "serde" "serde_json" "std" ]; - "serde_json" = [ "dep:serde_json" ]; - "spans" = [ "wasm-bindgen-macro/spans" ]; - "strict-macro" = [ "wasm-bindgen-macro/strict-macro" ]; - "xxx_debug_only_print_generated_code" = [ "wasm-bindgen-macro/xxx_debug_only_print_generated_code" ]; - }; - resolvedDefaultFeatures = [ "default" "spans" "std" ]; - }; - "wasm-bindgen-backend" = rec { - crateName = "wasm-bindgen-backend"; - version = "0.2.92"; - edition = "2018"; - sha256 = "1nj7wxbi49f0rw9d44rjzms26xlw6r76b2mrggx8jfbdjrxphkb1"; - authors = [ - "The wasm-bindgen Developers" - ]; - dependencies = [ - { - name = "bumpalo"; - packageId = "bumpalo"; - } - { - name = "log"; - packageId = "log"; - } - { - name = "once_cell"; - packageId = "once_cell"; - } - { - name = "proc-macro2"; - packageId = "proc-macro2"; - } - { - name = "quote"; - packageId = "quote"; - } - { - name = "syn"; - packageId = "syn 2.0.52"; - features = [ "full" ]; - } - { - name = "wasm-bindgen-shared"; - packageId = "wasm-bindgen-shared"; - } - ]; - features = { - "extra-traits" = [ "syn/extra-traits" ]; - }; - resolvedDefaultFeatures = [ "spans" ]; - }; - "wasm-bindgen-futures" = rec { - crateName = "wasm-bindgen-futures"; - version = "0.4.42"; - edition = "2018"; - sha256 = "1h322zjvpjllcpj7dahfxjsv6inkr6y0baw7nkdwivr1c4v19g3n"; - authors = [ - "The wasm-bindgen Developers" - ]; - dependencies = [ - { - name = "cfg-if"; - packageId = "cfg-if"; - } - { - name = "js-sys"; - packageId = "js-sys"; - } - { - name = "wasm-bindgen"; - packageId = "wasm-bindgen"; - } - { - name = "web-sys"; - packageId = "web-sys"; - target = { target, features }: (builtins.elem "atomics" targetFeatures); - features = [ "MessageEvent" "Worker" ]; - } - ]; - features = { - "futures-core" = [ "dep:futures-core" ]; - "futures-core-03-stream" = [ "futures-core" ]; - }; - }; - "wasm-bindgen-macro" = rec { - crateName = "wasm-bindgen-macro"; - version = "0.2.92"; - edition = "2018"; - sha256 = "09npa1srjjabd6nfph5yc03jb26sycjlxhy0c2a1pdrpx4yq5y51"; - procMacro = true; - authors = [ - "The wasm-bindgen Developers" - ]; - dependencies = [ - { - name = "quote"; - packageId = "quote"; - } - { - name = "wasm-bindgen-macro-support"; - packageId = "wasm-bindgen-macro-support"; - } - ]; - features = { - "spans" = [ "wasm-bindgen-macro-support/spans" ]; - "strict-macro" = [ "wasm-bindgen-macro-support/strict-macro" ]; - }; - resolvedDefaultFeatures = [ "spans" ]; - }; - "wasm-bindgen-macro-support" = rec { - crateName = "wasm-bindgen-macro-support"; - version = "0.2.92"; - edition = "2018"; - sha256 = "1dqv2xs8zcyw4kjgzj84bknp2h76phmsb3n7j6hn396h4ssifkz9"; - authors = [ - "The wasm-bindgen Developers" - ]; - dependencies = [ - { - name = "proc-macro2"; - packageId = "proc-macro2"; - } - { - name = "quote"; - packageId = "quote"; - } - { - name = "syn"; - packageId = "syn 2.0.52"; - features = [ "visit" "full" ]; - } - { - name = "wasm-bindgen-backend"; - packageId = "wasm-bindgen-backend"; - } - { - name = "wasm-bindgen-shared"; - packageId = "wasm-bindgen-shared"; - } - ]; - features = { - "extra-traits" = [ "syn/extra-traits" ]; - "spans" = [ "wasm-bindgen-backend/spans" ]; - }; - resolvedDefaultFeatures = [ "spans" ]; - }; - "wasm-bindgen-shared" = rec { - crateName = "wasm-bindgen-shared"; - version = "0.2.92"; - edition = "2018"; - links = "wasm_bindgen"; - sha256 = "15kyavsrna2cvy30kg03va257fraf9x00ny554vxngvpyaa0q6dg"; - authors = [ - "The wasm-bindgen Developers" - ]; - - }; - "wasm-streams" = rec { - crateName = "wasm-streams"; - version = "0.4.0"; - edition = "2021"; - sha256 = "0ad17c59xb8fffsnbrqbyqz93hb66nzxhizpii31icb31g4w8pdn"; - type = [ "cdylib" "rlib" ]; - authors = [ - "Mattias Buelens <mattias@buelens.com>" - ]; - dependencies = [ - { - name = "futures-util"; - packageId = "futures-util"; - features = [ "io" "sink" ]; - } - { - name = "js-sys"; - packageId = "js-sys"; - } - { - name = "wasm-bindgen"; - packageId = "wasm-bindgen"; - } - { - name = "wasm-bindgen-futures"; - packageId = "wasm-bindgen-futures"; - } - { - name = "web-sys"; - packageId = "web-sys"; - features = [ "AbortSignal" "QueuingStrategy" "ReadableStream" "ReadableStreamType" "ReadableWritablePair" "ReadableStreamByobReader" "ReadableStreamReaderMode" "ReadableStreamReadResult" "ReadableStreamByobRequest" "ReadableStreamDefaultReader" "ReadableByteStreamController" "ReadableStreamGetReaderOptions" "ReadableStreamDefaultController" "StreamPipeOptions" "TransformStream" "TransformStreamDefaultController" "Transformer" "UnderlyingSink" "UnderlyingSource" "WritableStream" "WritableStreamDefaultController" "WritableStreamDefaultWriter" ]; - } - ]; - devDependencies = [ - { - name = "web-sys"; - packageId = "web-sys"; - features = [ "console" "AbortSignal" "Response" "ReadableStream" "Window" ]; - } - ]; - - }; - "web-sys" = rec { - crateName = "web-sys"; - version = "0.3.69"; - edition = "2018"; - sha256 = "1vqkxk935xa8zcnsi4bd88sb267ly2i24xl1yiq26d1n32hskbvp"; - authors = [ - "The wasm-bindgen Developers" - ]; - dependencies = [ - { - name = "js-sys"; - packageId = "js-sys"; - } - { - name = "wasm-bindgen"; - packageId = "wasm-bindgen"; - } - ]; - features = { - "AbortSignal" = [ "EventTarget" ]; - "AnalyserNode" = [ "AudioNode" "EventTarget" ]; - "Animation" = [ "EventTarget" ]; - "AnimationEvent" = [ "Event" ]; - "AnimationPlaybackEvent" = [ "Event" ]; - "Attr" = [ "EventTarget" "Node" ]; - "AudioBufferSourceNode" = [ "AudioNode" "AudioScheduledSourceNode" "EventTarget" ]; - "AudioContext" = [ "BaseAudioContext" "EventTarget" ]; - "AudioDestinationNode" = [ "AudioNode" "EventTarget" ]; - "AudioNode" = [ "EventTarget" ]; - "AudioProcessingEvent" = [ "Event" ]; - "AudioScheduledSourceNode" = [ "AudioNode" "EventTarget" ]; - "AudioStreamTrack" = [ "EventTarget" "MediaStreamTrack" ]; - "AudioTrackList" = [ "EventTarget" ]; - "AudioWorklet" = [ "Worklet" ]; - "AudioWorkletGlobalScope" = [ "WorkletGlobalScope" ]; - "AudioWorkletNode" = [ "AudioNode" "EventTarget" ]; - "AuthenticatorAssertionResponse" = [ "AuthenticatorResponse" ]; - "AuthenticatorAttestationResponse" = [ "AuthenticatorResponse" ]; - "BaseAudioContext" = [ "EventTarget" ]; - "BatteryManager" = [ "EventTarget" ]; - "BeforeUnloadEvent" = [ "Event" ]; - "BiquadFilterNode" = [ "AudioNode" "EventTarget" ]; - "BlobEvent" = [ "Event" ]; - "Bluetooth" = [ "EventTarget" ]; - "BluetoothAdvertisingEvent" = [ "Event" ]; - "BluetoothDevice" = [ "EventTarget" ]; - "BluetoothPermissionResult" = [ "EventTarget" "PermissionStatus" ]; - "BluetoothRemoteGattCharacteristic" = [ "EventTarget" ]; - "BluetoothRemoteGattService" = [ "EventTarget" ]; - "BroadcastChannel" = [ "EventTarget" ]; - "CanvasCaptureMediaStream" = [ "EventTarget" "MediaStream" ]; - "CanvasCaptureMediaStreamTrack" = [ "EventTarget" "MediaStreamTrack" ]; - "CdataSection" = [ "CharacterData" "EventTarget" "Node" "Text" ]; - "ChannelMergerNode" = [ "AudioNode" "EventTarget" ]; - "ChannelSplitterNode" = [ "AudioNode" "EventTarget" ]; - "CharacterData" = [ "EventTarget" "Node" ]; - "ChromeWorker" = [ "EventTarget" "Worker" ]; - "Clipboard" = [ "EventTarget" ]; - "ClipboardEvent" = [ "Event" ]; - "CloseEvent" = [ "Event" ]; - "Comment" = [ "CharacterData" "EventTarget" "Node" ]; - "CompositionEvent" = [ "Event" "UiEvent" ]; - "ConstantSourceNode" = [ "AudioNode" "AudioScheduledSourceNode" "EventTarget" ]; - "ConvolverNode" = [ "AudioNode" "EventTarget" ]; - "CssAnimation" = [ "Animation" "EventTarget" ]; - "CssConditionRule" = [ "CssGroupingRule" "CssRule" ]; - "CssCounterStyleRule" = [ "CssRule" ]; - "CssFontFaceRule" = [ "CssRule" ]; - "CssFontFeatureValuesRule" = [ "CssRule" ]; - "CssGroupingRule" = [ "CssRule" ]; - "CssImportRule" = [ "CssRule" ]; - "CssKeyframeRule" = [ "CssRule" ]; - "CssKeyframesRule" = [ "CssRule" ]; - "CssMediaRule" = [ "CssConditionRule" "CssGroupingRule" "CssRule" ]; - "CssNamespaceRule" = [ "CssRule" ]; - "CssPageRule" = [ "CssRule" ]; - "CssStyleRule" = [ "CssRule" ]; - "CssStyleSheet" = [ "StyleSheet" ]; - "CssSupportsRule" = [ "CssConditionRule" "CssGroupingRule" "CssRule" ]; - "CssTransition" = [ "Animation" "EventTarget" ]; - "CustomEvent" = [ "Event" ]; - "DedicatedWorkerGlobalScope" = [ "EventTarget" "WorkerGlobalScope" ]; - "DelayNode" = [ "AudioNode" "EventTarget" ]; - "DeviceLightEvent" = [ "Event" ]; - "DeviceMotionEvent" = [ "Event" ]; - "DeviceOrientationEvent" = [ "Event" ]; - "DeviceProximityEvent" = [ "Event" ]; - "Document" = [ "EventTarget" "Node" ]; - "DocumentFragment" = [ "EventTarget" "Node" ]; - "DocumentTimeline" = [ "AnimationTimeline" ]; - "DocumentType" = [ "EventTarget" "Node" ]; - "DomMatrix" = [ "DomMatrixReadOnly" ]; - "DomPoint" = [ "DomPointReadOnly" ]; - "DomRect" = [ "DomRectReadOnly" ]; - "DomRequest" = [ "EventTarget" ]; - "DragEvent" = [ "Event" "MouseEvent" "UiEvent" ]; - "DynamicsCompressorNode" = [ "AudioNode" "EventTarget" ]; - "Element" = [ "EventTarget" "Node" ]; - "ErrorEvent" = [ "Event" ]; - "EventSource" = [ "EventTarget" ]; - "ExtendableEvent" = [ "Event" ]; - "ExtendableMessageEvent" = [ "Event" "ExtendableEvent" ]; - "FetchEvent" = [ "Event" "ExtendableEvent" ]; - "FetchObserver" = [ "EventTarget" ]; - "File" = [ "Blob" ]; - "FileReader" = [ "EventTarget" ]; - "FileSystemDirectoryEntry" = [ "FileSystemEntry" ]; - "FileSystemDirectoryHandle" = [ "FileSystemHandle" ]; - "FileSystemFileEntry" = [ "FileSystemEntry" ]; - "FileSystemFileHandle" = [ "FileSystemHandle" ]; - "FileSystemWritableFileStream" = [ "WritableStream" ]; - "FocusEvent" = [ "Event" "UiEvent" ]; - "FontFaceSet" = [ "EventTarget" ]; - "FontFaceSetLoadEvent" = [ "Event" ]; - "GainNode" = [ "AudioNode" "EventTarget" ]; - "GamepadAxisMoveEvent" = [ "Event" "GamepadEvent" ]; - "GamepadButtonEvent" = [ "Event" "GamepadEvent" ]; - "GamepadEvent" = [ "Event" ]; - "GpuDevice" = [ "EventTarget" ]; - "GpuInternalError" = [ "GpuError" ]; - "GpuOutOfMemoryError" = [ "GpuError" ]; - "GpuPipelineError" = [ "DomException" ]; - "GpuUncapturedErrorEvent" = [ "Event" ]; - "GpuValidationError" = [ "GpuError" ]; - "HashChangeEvent" = [ "Event" ]; - "Hid" = [ "EventTarget" ]; - "HidConnectionEvent" = [ "Event" ]; - "HidDevice" = [ "EventTarget" ]; - "HidInputReportEvent" = [ "Event" ]; - "HtmlAnchorElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlAreaElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlAudioElement" = [ "Element" "EventTarget" "HtmlElement" "HtmlMediaElement" "Node" ]; - "HtmlBaseElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlBodyElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlBrElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlButtonElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlCanvasElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlDListElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlDataElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlDataListElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlDetailsElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlDialogElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlDirectoryElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlDivElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlDocument" = [ "Document" "EventTarget" "Node" ]; - "HtmlElement" = [ "Element" "EventTarget" "Node" ]; - "HtmlEmbedElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlFieldSetElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlFontElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlFormControlsCollection" = [ "HtmlCollection" ]; - "HtmlFormElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlFrameElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlFrameSetElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlHeadElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlHeadingElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlHrElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlHtmlElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlIFrameElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlImageElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlInputElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlLabelElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlLegendElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlLiElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlLinkElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlMapElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlMediaElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlMenuElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlMenuItemElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlMetaElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlMeterElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlModElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlOListElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlObjectElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlOptGroupElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlOptionElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlOptionsCollection" = [ "HtmlCollection" ]; - "HtmlOutputElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlParagraphElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlParamElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlPictureElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlPreElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlProgressElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlQuoteElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlScriptElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlSelectElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlSlotElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlSourceElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlSpanElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlStyleElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlTableCaptionElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlTableCellElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlTableColElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlTableElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlTableRowElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlTableSectionElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlTemplateElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlTextAreaElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlTimeElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlTitleElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlTrackElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlUListElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlUnknownElement" = [ "Element" "EventTarget" "HtmlElement" "Node" ]; - "HtmlVideoElement" = [ "Element" "EventTarget" "HtmlElement" "HtmlMediaElement" "Node" ]; - "IdbCursorWithValue" = [ "IdbCursor" ]; - "IdbDatabase" = [ "EventTarget" ]; - "IdbFileHandle" = [ "EventTarget" ]; - "IdbFileRequest" = [ "DomRequest" "EventTarget" ]; - "IdbLocaleAwareKeyRange" = [ "IdbKeyRange" ]; - "IdbMutableFile" = [ "EventTarget" ]; - "IdbOpenDbRequest" = [ "EventTarget" "IdbRequest" ]; - "IdbRequest" = [ "EventTarget" ]; - "IdbTransaction" = [ "EventTarget" ]; - "IdbVersionChangeEvent" = [ "Event" ]; - "IirFilterNode" = [ "AudioNode" "EventTarget" ]; - "ImageCaptureErrorEvent" = [ "Event" ]; - "ImageTrack" = [ "EventTarget" ]; - "InputEvent" = [ "Event" "UiEvent" ]; - "KeyboardEvent" = [ "Event" "UiEvent" ]; - "KeyframeEffect" = [ "AnimationEffect" ]; - "LocalMediaStream" = [ "EventTarget" "MediaStream" ]; - "MediaDevices" = [ "EventTarget" ]; - "MediaElementAudioSourceNode" = [ "AudioNode" "EventTarget" ]; - "MediaEncryptedEvent" = [ "Event" ]; - "MediaKeyError" = [ "Event" ]; - "MediaKeyMessageEvent" = [ "Event" ]; - "MediaKeySession" = [ "EventTarget" ]; - "MediaQueryList" = [ "EventTarget" ]; - "MediaQueryListEvent" = [ "Event" ]; - "MediaRecorder" = [ "EventTarget" ]; - "MediaRecorderErrorEvent" = [ "Event" ]; - "MediaSource" = [ "EventTarget" ]; - "MediaStream" = [ "EventTarget" ]; - "MediaStreamAudioDestinationNode" = [ "AudioNode" "EventTarget" ]; - "MediaStreamAudioSourceNode" = [ "AudioNode" "EventTarget" ]; - "MediaStreamEvent" = [ "Event" ]; - "MediaStreamTrack" = [ "EventTarget" ]; - "MediaStreamTrackEvent" = [ "Event" ]; - "MediaStreamTrackGenerator" = [ "EventTarget" "MediaStreamTrack" ]; - "MessageEvent" = [ "Event" ]; - "MessagePort" = [ "EventTarget" ]; - "MidiAccess" = [ "EventTarget" ]; - "MidiConnectionEvent" = [ "Event" ]; - "MidiInput" = [ "EventTarget" "MidiPort" ]; - "MidiMessageEvent" = [ "Event" ]; - "MidiOutput" = [ "EventTarget" "MidiPort" ]; - "MidiPort" = [ "EventTarget" ]; - "MouseEvent" = [ "Event" "UiEvent" ]; - "MouseScrollEvent" = [ "Event" "MouseEvent" "UiEvent" ]; - "MutationEvent" = [ "Event" ]; - "NetworkInformation" = [ "EventTarget" ]; - "Node" = [ "EventTarget" ]; - "Notification" = [ "EventTarget" ]; - "NotificationEvent" = [ "Event" "ExtendableEvent" ]; - "OfflineAudioCompletionEvent" = [ "Event" ]; - "OfflineAudioContext" = [ "BaseAudioContext" "EventTarget" ]; - "OfflineResourceList" = [ "EventTarget" ]; - "OffscreenCanvas" = [ "EventTarget" ]; - "OscillatorNode" = [ "AudioNode" "AudioScheduledSourceNode" "EventTarget" ]; - "PageTransitionEvent" = [ "Event" ]; - "PaintWorkletGlobalScope" = [ "WorkletGlobalScope" ]; - "PannerNode" = [ "AudioNode" "EventTarget" ]; - "PaymentMethodChangeEvent" = [ "Event" "PaymentRequestUpdateEvent" ]; - "PaymentRequestUpdateEvent" = [ "Event" ]; - "Performance" = [ "EventTarget" ]; - "PerformanceMark" = [ "PerformanceEntry" ]; - "PerformanceMeasure" = [ "PerformanceEntry" ]; - "PerformanceNavigationTiming" = [ "PerformanceEntry" "PerformanceResourceTiming" ]; - "PerformanceResourceTiming" = [ "PerformanceEntry" ]; - "PermissionStatus" = [ "EventTarget" ]; - "PointerEvent" = [ "Event" "MouseEvent" "UiEvent" ]; - "PopStateEvent" = [ "Event" ]; - "PopupBlockedEvent" = [ "Event" ]; - "PresentationAvailability" = [ "EventTarget" ]; - "PresentationConnection" = [ "EventTarget" ]; - "PresentationConnectionAvailableEvent" = [ "Event" ]; - "PresentationConnectionCloseEvent" = [ "Event" ]; - "PresentationConnectionList" = [ "EventTarget" ]; - "PresentationRequest" = [ "EventTarget" ]; - "ProcessingInstruction" = [ "CharacterData" "EventTarget" "Node" ]; - "ProgressEvent" = [ "Event" ]; - "PromiseRejectionEvent" = [ "Event" ]; - "PublicKeyCredential" = [ "Credential" ]; - "PushEvent" = [ "Event" "ExtendableEvent" ]; - "RadioNodeList" = [ "NodeList" ]; - "RtcDataChannel" = [ "EventTarget" ]; - "RtcDataChannelEvent" = [ "Event" ]; - "RtcPeerConnection" = [ "EventTarget" ]; - "RtcPeerConnectionIceErrorEvent" = [ "Event" ]; - "RtcPeerConnectionIceEvent" = [ "Event" ]; - "RtcTrackEvent" = [ "Event" ]; - "RtcdtmfSender" = [ "EventTarget" ]; - "RtcdtmfToneChangeEvent" = [ "Event" ]; - "Screen" = [ "EventTarget" ]; - "ScreenOrientation" = [ "EventTarget" ]; - "ScriptProcessorNode" = [ "AudioNode" "EventTarget" ]; - "ScrollAreaEvent" = [ "Event" "UiEvent" ]; - "SecurityPolicyViolationEvent" = [ "Event" ]; - "Serial" = [ "EventTarget" ]; - "SerialPort" = [ "EventTarget" ]; - "ServiceWorker" = [ "EventTarget" ]; - "ServiceWorkerContainer" = [ "EventTarget" ]; - "ServiceWorkerGlobalScope" = [ "EventTarget" "WorkerGlobalScope" ]; - "ServiceWorkerRegistration" = [ "EventTarget" ]; - "ShadowRoot" = [ "DocumentFragment" "EventTarget" "Node" ]; - "SharedWorker" = [ "EventTarget" ]; - "SharedWorkerGlobalScope" = [ "EventTarget" "WorkerGlobalScope" ]; - "SourceBuffer" = [ "EventTarget" ]; - "SourceBufferList" = [ "EventTarget" ]; - "SpeechRecognition" = [ "EventTarget" ]; - "SpeechRecognitionError" = [ "Event" ]; - "SpeechRecognitionEvent" = [ "Event" ]; - "SpeechSynthesis" = [ "EventTarget" ]; - "SpeechSynthesisErrorEvent" = [ "Event" "SpeechSynthesisEvent" ]; - "SpeechSynthesisEvent" = [ "Event" ]; - "SpeechSynthesisUtterance" = [ "EventTarget" ]; - "StereoPannerNode" = [ "AudioNode" "EventTarget" ]; - "StorageEvent" = [ "Event" ]; - "SubmitEvent" = [ "Event" ]; - "SvgAnimateElement" = [ "Element" "EventTarget" "Node" "SvgAnimationElement" "SvgElement" ]; - "SvgAnimateMotionElement" = [ "Element" "EventTarget" "Node" "SvgAnimationElement" "SvgElement" ]; - "SvgAnimateTransformElement" = [ "Element" "EventTarget" "Node" "SvgAnimationElement" "SvgElement" ]; - "SvgAnimationElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgCircleElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGeometryElement" "SvgGraphicsElement" ]; - "SvgClipPathElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgComponentTransferFunctionElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgDefsElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGraphicsElement" ]; - "SvgDescElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgElement" = [ "Element" "EventTarget" "Node" ]; - "SvgEllipseElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGeometryElement" "SvgGraphicsElement" ]; - "SvgFilterElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgForeignObjectElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGraphicsElement" ]; - "SvgGeometryElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGraphicsElement" ]; - "SvgGradientElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgGraphicsElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgImageElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGraphicsElement" ]; - "SvgLineElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGeometryElement" "SvgGraphicsElement" ]; - "SvgLinearGradientElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGradientElement" ]; - "SvgMarkerElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgMaskElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgMetadataElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgPathElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGeometryElement" "SvgGraphicsElement" ]; - "SvgPathSegArcAbs" = [ "SvgPathSeg" ]; - "SvgPathSegArcRel" = [ "SvgPathSeg" ]; - "SvgPathSegClosePath" = [ "SvgPathSeg" ]; - "SvgPathSegCurvetoCubicAbs" = [ "SvgPathSeg" ]; - "SvgPathSegCurvetoCubicRel" = [ "SvgPathSeg" ]; - "SvgPathSegCurvetoCubicSmoothAbs" = [ "SvgPathSeg" ]; - "SvgPathSegCurvetoCubicSmoothRel" = [ "SvgPathSeg" ]; - "SvgPathSegCurvetoQuadraticAbs" = [ "SvgPathSeg" ]; - "SvgPathSegCurvetoQuadraticRel" = [ "SvgPathSeg" ]; - "SvgPathSegCurvetoQuadraticSmoothAbs" = [ "SvgPathSeg" ]; - "SvgPathSegCurvetoQuadraticSmoothRel" = [ "SvgPathSeg" ]; - "SvgPathSegLinetoAbs" = [ "SvgPathSeg" ]; - "SvgPathSegLinetoHorizontalAbs" = [ "SvgPathSeg" ]; - "SvgPathSegLinetoHorizontalRel" = [ "SvgPathSeg" ]; - "SvgPathSegLinetoRel" = [ "SvgPathSeg" ]; - "SvgPathSegLinetoVerticalAbs" = [ "SvgPathSeg" ]; - "SvgPathSegLinetoVerticalRel" = [ "SvgPathSeg" ]; - "SvgPathSegMovetoAbs" = [ "SvgPathSeg" ]; - "SvgPathSegMovetoRel" = [ "SvgPathSeg" ]; - "SvgPatternElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgPolygonElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGeometryElement" "SvgGraphicsElement" ]; - "SvgPolylineElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGeometryElement" "SvgGraphicsElement" ]; - "SvgRadialGradientElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGradientElement" ]; - "SvgRectElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGeometryElement" "SvgGraphicsElement" ]; - "SvgScriptElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgSetElement" = [ "Element" "EventTarget" "Node" "SvgAnimationElement" "SvgElement" ]; - "SvgStopElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgStyleElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgSwitchElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGraphicsElement" ]; - "SvgSymbolElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgTextContentElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGraphicsElement" ]; - "SvgTextElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGraphicsElement" "SvgTextContentElement" "SvgTextPositioningElement" ]; - "SvgTextPathElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGraphicsElement" "SvgTextContentElement" ]; - "SvgTextPositioningElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGraphicsElement" "SvgTextContentElement" ]; - "SvgTitleElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgUseElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGraphicsElement" ]; - "SvgViewElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgaElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGraphicsElement" ]; - "SvgfeBlendElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeColorMatrixElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeComponentTransferElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeCompositeElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeConvolveMatrixElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeDiffuseLightingElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeDisplacementMapElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeDistantLightElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeDropShadowElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeFloodElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeFuncAElement" = [ "Element" "EventTarget" "Node" "SvgComponentTransferFunctionElement" "SvgElement" ]; - "SvgfeFuncBElement" = [ "Element" "EventTarget" "Node" "SvgComponentTransferFunctionElement" "SvgElement" ]; - "SvgfeFuncGElement" = [ "Element" "EventTarget" "Node" "SvgComponentTransferFunctionElement" "SvgElement" ]; - "SvgfeFuncRElement" = [ "Element" "EventTarget" "Node" "SvgComponentTransferFunctionElement" "SvgElement" ]; - "SvgfeGaussianBlurElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeImageElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeMergeElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeMergeNodeElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeMorphologyElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeOffsetElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfePointLightElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeSpecularLightingElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeSpotLightElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeTileElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgfeTurbulenceElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvggElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGraphicsElement" ]; - "SvgmPathElement" = [ "Element" "EventTarget" "Node" "SvgElement" ]; - "SvgsvgElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGraphicsElement" ]; - "SvgtSpanElement" = [ "Element" "EventTarget" "Node" "SvgElement" "SvgGraphicsElement" "SvgTextContentElement" "SvgTextPositioningElement" ]; - "TaskController" = [ "AbortController" ]; - "TaskPriorityChangeEvent" = [ "Event" ]; - "TaskSignal" = [ "AbortSignal" "EventTarget" ]; - "TcpServerSocket" = [ "EventTarget" ]; - "TcpServerSocketEvent" = [ "Event" ]; - "TcpSocket" = [ "EventTarget" ]; - "TcpSocketErrorEvent" = [ "Event" ]; - "TcpSocketEvent" = [ "Event" ]; - "Text" = [ "CharacterData" "EventTarget" "Node" ]; - "TextTrack" = [ "EventTarget" ]; - "TextTrackCue" = [ "EventTarget" ]; - "TextTrackList" = [ "EventTarget" ]; - "TimeEvent" = [ "Event" ]; - "TouchEvent" = [ "Event" "UiEvent" ]; - "TrackEvent" = [ "Event" ]; - "TransitionEvent" = [ "Event" ]; - "UiEvent" = [ "Event" ]; - "Usb" = [ "EventTarget" ]; - "UsbConnectionEvent" = [ "Event" ]; - "UsbPermissionResult" = [ "EventTarget" "PermissionStatus" ]; - "UserProximityEvent" = [ "Event" ]; - "ValueEvent" = [ "Event" ]; - "VideoStreamTrack" = [ "EventTarget" "MediaStreamTrack" ]; - "VideoTrackList" = [ "EventTarget" ]; - "VrDisplay" = [ "EventTarget" ]; - "VttCue" = [ "EventTarget" "TextTrackCue" ]; - "WakeLockSentinel" = [ "EventTarget" ]; - "WaveShaperNode" = [ "AudioNode" "EventTarget" ]; - "WebGlContextEvent" = [ "Event" ]; - "WebKitCssMatrix" = [ "DomMatrix" "DomMatrixReadOnly" ]; - "WebSocket" = [ "EventTarget" ]; - "WebTransportError" = [ "DomException" ]; - "WebTransportReceiveStream" = [ "ReadableStream" ]; - "WebTransportSendStream" = [ "WritableStream" ]; - "WheelEvent" = [ "Event" "MouseEvent" "UiEvent" ]; - "Window" = [ "EventTarget" ]; - "WindowClient" = [ "Client" ]; - "Worker" = [ "EventTarget" ]; - "WorkerDebuggerGlobalScope" = [ "EventTarget" ]; - "WorkerGlobalScope" = [ "EventTarget" ]; - "XmlDocument" = [ "Document" "EventTarget" "Node" ]; - "XmlHttpRequest" = [ "EventTarget" "XmlHttpRequestEventTarget" ]; - "XmlHttpRequestEventTarget" = [ "EventTarget" ]; - "XmlHttpRequestUpload" = [ "EventTarget" "XmlHttpRequestEventTarget" ]; - "XrBoundedReferenceSpace" = [ "EventTarget" "XrReferenceSpace" "XrSpace" ]; - "XrInputSourceEvent" = [ "Event" ]; - "XrInputSourcesChangeEvent" = [ "Event" ]; - "XrJointPose" = [ "XrPose" ]; - "XrJointSpace" = [ "EventTarget" "XrSpace" ]; - "XrLayer" = [ "EventTarget" ]; - "XrPermissionStatus" = [ "EventTarget" "PermissionStatus" ]; - "XrReferenceSpace" = [ "EventTarget" "XrSpace" ]; - "XrReferenceSpaceEvent" = [ "Event" ]; - "XrSession" = [ "EventTarget" ]; - "XrSessionEvent" = [ "Event" ]; - "XrSpace" = [ "EventTarget" ]; - "XrSystem" = [ "EventTarget" ]; - "XrViewerPose" = [ "XrPose" ]; - "XrWebGlLayer" = [ "EventTarget" "XrLayer" ]; - }; - resolvedDefaultFeatures = [ "AbortController" "AbortSignal" "Blob" "BlobPropertyBag" "Event" "EventTarget" "File" "FormData" "Headers" "MessageEvent" "QueuingStrategy" "ReadableByteStreamController" "ReadableStream" "ReadableStreamByobReader" "ReadableStreamByobRequest" "ReadableStreamDefaultController" "ReadableStreamDefaultReader" "ReadableStreamGetReaderOptions" "ReadableStreamReadResult" "ReadableStreamReaderMode" "ReadableStreamType" "ReadableWritablePair" "Request" "RequestCredentials" "RequestInit" "RequestMode" "Response" "ServiceWorkerGlobalScope" "StreamPipeOptions" "TransformStream" "TransformStreamDefaultController" "Transformer" "UnderlyingSink" "UnderlyingSource" "Window" "Worker" "WorkerGlobalScope" "WritableStream" "WritableStreamDefaultController" "WritableStreamDefaultWriter" ]; - }; - "web-time" = rec { - crateName = "web-time"; - version = "0.2.4"; - edition = "2021"; - sha256 = "1q6gk0nkwbfz30g1pz8g52mq00zjx7m5im36k3474aw73jdh8c5a"; - dependencies = [ - { - name = "js-sys"; - packageId = "js-sys"; - target = { target, features }: ((builtins.elem "wasm" target."family") && (!(("emscripten" == target."os" or null) || ("wasi" == target."os" or null)))); - } - { - name = "wasm-bindgen"; - packageId = "wasm-bindgen"; - usesDefaultFeatures = false; - target = { target, features }: ((builtins.elem "wasm" target."family") && (!(("emscripten" == target."os" or null) || ("wasi" == target."os" or null)))); - } - ]; - - }; - "which" = rec { - crateName = "which"; - version = "4.4.2"; - edition = "2021"; - sha256 = "1ixzmx3svsv5hbdvd8vdhd3qwvf6ns8jdpif1wmwsy10k90j9fl7"; - authors = [ - "Harry Fei <tiziyuanfang@gmail.com>" - ]; - dependencies = [ - { - name = "either"; - packageId = "either"; - } - { - name = "home"; - packageId = "home"; - target = { target, features }: ((target."windows" or false) || (target."unix" or false) || ("redox" == target."os" or null)); - } - { - name = "once_cell"; - packageId = "once_cell"; - target = { target, features }: (target."windows" or false); - } - { - name = "rustix"; - packageId = "rustix"; - usesDefaultFeatures = false; - features = [ "fs" "std" ]; - } - ]; - features = { - "regex" = [ "dep:regex" ]; - }; - }; "winapi" = rec { crateName = "winapi"; version = "0.3.9"; @@ -10617,7 +4253,7 @@ rec { features = { "debug" = [ "impl-debug" ]; }; - resolvedDefaultFeatures = [ "consoleapi" "errhandlingapi" "fileapi" "handleapi" "minwindef" "ntstatus" "processenv" "processthreadsapi" "std" "sysinfoapi" "winbase" "wincon" "winerror" "winnt" ]; + resolvedDefaultFeatures = [ "consoleapi" "errhandlingapi" "fileapi" "handleapi" "processenv" ]; }; "winapi-i686-pc-windows-gnu" = rec { crateName = "winapi-i686-pc-windows-gnu"; @@ -10629,24 +4265,6 @@ rec { ]; }; - "winapi-util" = rec { - crateName = "winapi-util"; - version = "0.1.6"; - edition = "2021"; - sha256 = "15i5lm39wd44004i9d5qspry2cynkrpvwzghr6s2c3dsk28nz7pj"; - authors = [ - "Andrew Gallant <jamslam@gmail.com>" - ]; - dependencies = [ - { - name = "winapi"; - packageId = "winapi"; - target = { target, features }: (target."windows" or false); - features = [ "std" "consoleapi" "errhandlingapi" "fileapi" "minwindef" "processenv" "sysinfoapi" "winbase" "wincon" "winerror" "winnt" ]; - } - ]; - - }; "winapi-x86_64-pc-windows-gnu" = rec { crateName = "winapi-x86_64-pc-windows-gnu"; version = "0.4.0"; @@ -10657,23 +4275,6 @@ rec { ]; }; - "windows-core" = rec { - crateName = "windows-core"; - version = "0.52.0"; - edition = "2021"; - sha256 = "1nc3qv7sy24x0nlnb32f7alzpd6f72l4p24vl65vydbyil669ark"; - authors = [ - "Microsoft" - ]; - dependencies = [ - { - name = "windows-targets"; - packageId = "windows-targets 0.52.4"; - } - ]; - features = { }; - resolvedDefaultFeatures = [ "default" ]; - }; "windows-sys 0.48.0" = rec { crateName = "windows-sys"; version = "0.48.0"; @@ -10965,7 +4566,7 @@ rec { "Win32_Web" = [ "Win32" ]; "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; }; - resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_Pipes" "Win32_System_Registry" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_Time" "Win32_System_WindowsProgramming" "default" ]; + resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_IO" "Win32_System_Pipes" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_WindowsProgramming" "default" ]; }; "windows-sys 0.52.0" = rec { crateName = "windows-sys"; @@ -11212,7 +4813,7 @@ rec { "Win32_Web" = [ "Win32" ]; "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; }; - resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_NetworkManagement" "Win32_NetworkManagement_IpHelper" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Com" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_Memory" "Win32_System_Threading" "Win32_System_WindowsProgramming" "Win32_UI" "Win32_UI_Shell" "default" ]; + resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_System" "Win32_System_Console" "Win32_System_IO" "Win32_System_Threading" "Win32_System_WindowsProgramming" "default" ]; }; "windows-targets 0.48.5" = rec { crateName = "windows-targets"; @@ -11448,52 +5049,6 @@ rec { ]; }; - "winreg" = rec { - crateName = "winreg"; - version = "0.50.0"; - edition = "2018"; - sha256 = "1cddmp929k882mdh6i9f2as848f13qqna6czwsqzkh1pqnr5fkjj"; - authors = [ - "Igor Shaula <gentoo90@gmail.com>" - ]; - dependencies = [ - { - name = "cfg-if"; - packageId = "cfg-if"; - } - { - name = "windows-sys"; - packageId = "windows-sys 0.48.0"; - features = [ "Win32_Foundation" "Win32_System_Time" "Win32_System_Registry" "Win32_Security" "Win32_Storage_FileSystem" "Win32_System_Diagnostics_Debug" ]; - } - ]; - features = { - "chrono" = [ "dep:chrono" ]; - "serde" = [ "dep:serde" ]; - "serialization-serde" = [ "transactions" "serde" ]; - }; - }; - "xz2" = rec { - crateName = "xz2"; - version = "0.1.7"; - edition = "2018"; - sha256 = "1qk7nzpblizvayyq4xzi4b0zacmmbqr6vb9fc0v1avyp17f4931q"; - authors = [ - "Alex Crichton <alex@alexcrichton.com>" - ]; - dependencies = [ - { - name = "lzma-sys"; - packageId = "lzma-sys"; - } - ]; - features = { - "futures" = [ "dep:futures" ]; - "static" = [ "lzma-sys/static" ]; - "tokio" = [ "tokio-io" "futures" ]; - "tokio-io" = [ "dep:tokio-io" ]; - }; - }; "zeroize" = rec { crateName = "zeroize"; version = "1.7.0"; @@ -11509,98 +5064,7 @@ rec { "std" = [ "alloc" ]; "zeroize_derive" = [ "dep:zeroize_derive" ]; }; - resolvedDefaultFeatures = [ "alloc" "default" ]; - }; - "zstd" = rec { - crateName = "zstd"; - version = "0.13.0"; - edition = "2018"; - sha256 = "0401q54s9r35x2i7m1kwppgkj79g0pb6xz3xpby7qlkdb44k7yxz"; - authors = [ - "Alexandre Bury <alexandre.bury@gmail.com>" - ]; - dependencies = [ - { - name = "zstd-safe"; - packageId = "zstd-safe"; - usesDefaultFeatures = false; - features = [ "std" ]; - } - ]; - features = { - "arrays" = [ "zstd-safe/arrays" ]; - "bindgen" = [ "zstd-safe/bindgen" ]; - "debug" = [ "zstd-safe/debug" ]; - "default" = [ "legacy" "arrays" "zdict_builder" ]; - "experimental" = [ "zstd-safe/experimental" ]; - "fat-lto" = [ "zstd-safe/fat-lto" ]; - "legacy" = [ "zstd-safe/legacy" ]; - "no_asm" = [ "zstd-safe/no_asm" ]; - "pkg-config" = [ "zstd-safe/pkg-config" ]; - "thin" = [ "zstd-safe/thin" ]; - "thin-lto" = [ "zstd-safe/thin-lto" ]; - "zdict_builder" = [ "zstd-safe/zdict_builder" ]; - "zstdmt" = [ "zstd-safe/zstdmt" ]; - }; - resolvedDefaultFeatures = [ "arrays" "default" "legacy" "zdict_builder" ]; - }; - "zstd-safe" = rec { - crateName = "zstd-safe"; - version = "7.0.0"; - edition = "2018"; - sha256 = "0gpav2lcibrpmyslmjkcn3w0w64qif3jjljd2h8lr4p249s7qx23"; - authors = [ - "Alexandre Bury <alexandre.bury@gmail.com>" - ]; - dependencies = [ - { - name = "zstd-sys"; - packageId = "zstd-sys"; - usesDefaultFeatures = false; - } - ]; - features = { - "bindgen" = [ "zstd-sys/bindgen" ]; - "debug" = [ "zstd-sys/debug" ]; - "default" = [ "legacy" "arrays" "zdict_builder" ]; - "experimental" = [ "zstd-sys/experimental" ]; - "fat-lto" = [ "zstd-sys/fat-lto" ]; - "legacy" = [ "zstd-sys/legacy" ]; - "no_asm" = [ "zstd-sys/no_asm" ]; - "pkg-config" = [ "zstd-sys/pkg-config" ]; - "std" = [ "zstd-sys/std" ]; - "thin" = [ "zstd-sys/thin" ]; - "thin-lto" = [ "zstd-sys/thin-lto" ]; - "zdict_builder" = [ "zstd-sys/zdict_builder" ]; - "zstdmt" = [ "zstd-sys/zstdmt" ]; - }; - resolvedDefaultFeatures = [ "arrays" "legacy" "std" "zdict_builder" ]; - }; - "zstd-sys" = rec { - crateName = "zstd-sys"; - version = "2.0.9+zstd.1.5.5"; - edition = "2018"; - links = "zstd"; - sha256 = "0mk6a2367swdi22zg03lcackpnvgq96d7120awd4i83lm2lfy5ly"; - authors = [ - "Alexandre Bury <alexandre.bury@gmail.com>" - ]; - buildDependencies = [ - { - name = "cc"; - packageId = "cc"; - features = [ "parallel" ]; - } - { - name = "pkg-config"; - packageId = "pkg-config"; - } - ]; - features = { - "bindgen" = [ "dep:bindgen" ]; - "default" = [ "legacy" "zdict_builder" ]; - }; - resolvedDefaultFeatures = [ "legacy" "std" "zdict_builder" ]; + resolvedDefaultFeatures = [ "alloc" ]; }; }; diff --git a/users/picnoir/tvix-daemon/Cargo.toml b/users/picnoir/tvix-daemon/Cargo.toml index f0642dbd53..2aca99f201 100644 --- a/users/picnoir/tvix-daemon/Cargo.toml +++ b/users/picnoir/tvix-daemon/Cargo.toml @@ -4,14 +4,12 @@ version = "0.1.0" edition = "2021" [dependencies] -tvix-store = { path = "../../../tvix/store/" } -nix-compat = { path = "../../../tvix/nix-compat/" } +nix-compat = { path = "../../../tvix/nix-compat", features = ["wire"] } tokio-listener = "0.3.1" tokio = { version = "1.36.0", features = ["full"] } tracing-subscriber = "0.3.18" tracing = "0.1.40" -anyhow = "1.0.81" -clap = { version = "4.5.3", features = ["derive"] } +clap = { version = "4.5.3", features = ["derive", "env"] } [dev-dependencies] tokio-test = "0.4.4" diff --git a/users/picnoir/tvix-daemon/default.nix b/users/picnoir/tvix-daemon/default.nix index 68aa12d0e2..78b9aa9a1d 100644 --- a/users/picnoir/tvix-daemon/default.nix +++ b/users/picnoir/tvix-daemon/default.nix @@ -1,4 +1,4 @@ -{ depot, lib, pkgs, ... }: +{ depot, pkgs, ... }: let crate2nix = pkgs.callPackage ./Cargo.nix { @@ -19,6 +19,23 @@ in { shell = (import ./shell.nix { inherit pkgs; }); tvix-daemon = crate2nix.rootCrate.build; + clippy = pkgs.stdenv.mkDerivation { + name = "tvix-daemon-clippy"; + # The cleaned sources. + src = depot.third_party.gitignoreSource ./.; + cargoDeps = crate2nix.allWorkspaceMembers; + + nativeBuildInputs = with pkgs; [ + cargo + clippy + pkg-config + protobuf + rustc + rustPlatform.cargoSetupHook + ]; + + buildPhase = "cargo clippy --tests --all-features --benches --examples | tee $out"; + }; meta.ci.targets = [ "tvix-daemon" "shell" diff --git a/users/picnoir/tvix-daemon/src/main.rs b/users/picnoir/tvix-daemon/src/main.rs index 87947cee31..dc49b209e0 100644 --- a/users/picnoir/tvix-daemon/src/main.rs +++ b/users/picnoir/tvix-daemon/src/main.rs @@ -1,24 +1,33 @@ -use anyhow::anyhow; use clap::Parser; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio_listener::{self, SystemOptions, UserOptions}; -use tracing::{debug, error, info, instrument}; +use tracing::{debug, error, info, instrument, Level}; -use nix_compat::wire::primitive; +use nix_compat::worker_protocol::{self, server_handshake_client, ClientSettings, Trust}; +use nix_compat::ProtocolVersion; #[derive(Parser, Debug)] struct Cli { /// Listening unix socket path #[arg(short, long)] socket: Option<String>, + /// Log verbosity level. Can be "error", "warn", "info", "debug", "trace", or a number 1-5 + #[arg(short, long, env)] + verbosity: Option<Level>, } #[tokio::main] #[instrument()] async fn main() { - tracing_subscriber::fmt().compact().try_init().unwrap(); let args = Cli::parse(); - + tracing_subscriber::fmt() + .compact() + .with_max_level( + args.verbosity + .unwrap_or_else(|| panic!("Can't parse log verbosity")), + ) + .try_init() + .unwrap(); info!("Started Tvix daemon"); let addr = args .socket @@ -26,17 +35,30 @@ async fn main() { .parse() .expect("Invalid listening socket address"); let system_options: SystemOptions = Default::default(); - let user_options: UserOptions = Default::default(); + let mut user_options: UserOptions = Default::default(); + user_options.recv_buffer_size = Some(1024); + user_options.send_buffer_size = Some(1024); + info!(user_options.send_buffer_size); + info!(user_options.recv_buffer_size); let mut listener = tokio_listener::Listener::bind(&addr, &system_options, &user_options) .await .unwrap(); - info!("Listening for incoming connections on {:?}", listener); + info!(listener_address = ?listener, "Listening for incoming connections"); while let Ok((conn, addr)) = listener.accept().await { - info!("Incoming connection from {addr}"); + info!(addr = %addr, "Incoming connection"); tokio::spawn(async move { worker(conn).await }); } } +/// Structure used to hold the client socket connection and some +/// metadata about the connection. +#[derive(Debug)] +struct ClientConnection<R: AsyncReadExt + AsyncWriteExt + Unpin> { + pub conn: R, + pub version: ProtocolVersion, + pub client_settings: Option<ClientSettings>, +} + /// Worker in charge to respond a Nix client using the Nix wire /// protocol. #[instrument()] @@ -44,102 +66,51 @@ async fn worker<R>(mut conn: R) where R: AsyncReadExt + AsyncWriteExt + Unpin + std::fmt::Debug, { - match perform_init_handshake(&mut conn).await { - Ok(_) => { - // TODO: process request here, dispatch to operation - // handler. - info!("Handshake done, bye now"); - } - Err(e) => error!("Client handshake failed: {}", e), - } -} - -/// Performs the initial handshake. During the handshake, the client -/// will first send a magic u64, to which the daemon needs to respond -/// with another magic u64. -/// -/// We then retrieve the client version, and discard a bunch of now -/// obsolete data. -#[instrument()] -async fn perform_init_handshake<'a, R: 'a>(mut conn: &'a mut R) -> anyhow::Result<()> -where - &'a mut R: AsyncReadExt + AsyncWriteExt + Unpin + std::fmt::Debug, -{ - let mut magic_hello = vec![0; 8]; - conn.read(&mut magic_hello).await?; - debug!("Hello read"); - if magic_hello != primitive::MAGIC_HELLO { - Err(anyhow!( - "Invalid client hello received: {:?}, expected {:?}", - magic_hello, - primitive::MAGIC_HELLO - )) - } else { - conn.write(&primitive::MAGIC_HELLO_RESPONSE).await?; - conn.write(&primitive::PROTOCOL_VERSION).await?; - conn.flush().await?; - debug!("Hello responded"); - let client_version = primitive::read_u32(&mut conn).await?; - debug!("Version read"); - if client_version < 0x10a { - return Err(anyhow!("The nix client version is too old")); - } - let protocol_minor = client_version & 0x00ff; - let protocol_major = client_version & 0xff00; - debug!(client.version = %client_version, client.minor = %protocol_minor, client.major = %protocol_major); - if protocol_minor >= 14 { - debug!("read cpu affinity"); - // Obsolete CPU affinity. - let read_affinity = primitive::read_u32(&mut conn).await?; - if read_affinity != 0 { - skip_8_bytes(&mut conn).await?; + match server_handshake_client(&mut conn, "2.18.2", Trust::Trusted).await { + Ok(client_protocol_version) => { + let mut client_connection = ClientConnection { + conn, + version: client_protocol_version, + client_settings: None, }; + debug!("Client hanshake succeeded"); + debug!(client_protocol_version = ?client_protocol_version); + // TODO: implement logging. For now, we'll just send + // STDERR_LAST, which is good enough to get Nix respond to + // us. + client_connection + .conn + .write_u64_le(worker_protocol::STDERR_LAST) + .await + .unwrap(); + loop { + let op = worker_protocol::read_op(&mut client_connection.conn) + .await + .unwrap(); + match op { + worker_protocol::Operation::SetOptions => { + let settings = op_set_options(&mut client_connection).await.unwrap(); + client_connection.client_settings = Some(settings); + debug!(settings = ?client_connection.client_settings, "Received client settings"); + } + _ => { + error!(op = ?op, "Unimplemented operation"); + break; + } + } + } } - if protocol_minor >= 11 { - // Obsolete reserveSpace - debug!("read reservespace"); - skip_8_bytes(&mut conn).await?; - } - if protocol_minor >= 33 { - // Nix version. We're plain lying, we're not Nix, but ehโฆ - // Setting it to the 2.3 lineage. Not 100% sure this is a - // good idea. - debug!("write version"); - // Plain str padded to 64 bits. - conn.write(&"2.3.17\0\0".as_bytes()).await?; - } - Ok(()) + Err(e) => error!("Client handshake failed: {}", e), } } -async fn skip_8_bytes<R>(conn: &mut R) -> anyhow::Result<()> +async fn op_set_options<R>(conn: &mut ClientConnection<R>) -> std::io::Result<ClientSettings> where - R: AsyncReadExt + Unpin + std::fmt::Debug, + R: AsyncReadExt + AsyncWriteExt + Unpin + std::fmt::Debug, { - let mut _discard_buffer = [0; 8]; - conn.read_exact(&mut _discard_buffer).await?; - Ok(()) -} - -#[cfg(test)] -mod integration_tests { - use nix_compat::wire::primitive; - #[tokio::test] - async fn test_init_handshake() { - let mut test_conn = tokio_test::io::Builder::new() - .read(&primitive::MAGIC_HELLO) - .write(&primitive::MAGIC_HELLO_RESPONSE) - .write(&primitive::PROTOCOL_VERSION) - // Let's say the client is in sync with the daemon - // protocol-wise - .read(&primitive::PROTOCOL_VERSION) - // cpu affinity - .read(&vec![0; 8]) - // reservespace - .read(&vec![0; 8]) - // version - .write(&"2.3.17\0\0".as_bytes()) - .build(); - crate::worker(&mut test_conn).await; - } + let settings = worker_protocol::read_client_settings(&mut conn.conn, conn.version).await?; + // The client expects us to send some logs when we're processing + // the settings. Sending STDERR_LAST signal we're done processing. + conn.conn.write_u64_le(worker_protocol::STDERR_LAST).await?; + Ok(settings) } diff --git a/users/sterni/machines/ingeborg/default.nix b/users/sterni/machines/ingeborg/default.nix index 0e5a30a7c8..2d026ae05b 100644 --- a/users/sterni/machines/ingeborg/default.nix +++ b/users/sterni/machines/ingeborg/default.nix @@ -17,6 +17,7 @@ ./http/code.sterni.lv.nix ./http/flipdot.openlab-augsburg.de.nix ./tv.nix + ./quassel.nix # Inactive: # ./http/likely-music.sterni.lv.nix diff --git a/users/sterni/machines/ingeborg/quassel.nix b/users/sterni/machines/ingeborg/quassel.nix new file mode 100644 index 0000000000..cd8dacc917 --- /dev/null +++ b/users/sterni/machines/ingeborg/quassel.nix @@ -0,0 +1,18 @@ +{ depot, ... }: + +{ + imports = [ + (depot.path.origSrc + "/ops/modules/quassel.nix") + ]; + + config = { + services.depot.quassel = { + enable = true; + acmeHost = "sterni.lv"; + bindAddresses = [ + "0.0.0.0" + "::" + ]; + }; + }; +} diff --git a/users/sterni/modules/common.nix b/users/sterni/modules/common.nix index ef039fe4de..2c513acad3 100644 --- a/users/sterni/modules/common.nix +++ b/users/sterni/modules/common.nix @@ -58,7 +58,6 @@ in }; environment.systemPackages = [ - pkgs.weechat pkgs.wget pkgs.git pkgs.stow diff --git a/users/tazjin/home/shared.nix b/users/tazjin/home/shared.nix index 65e18fa8d8..38d8add4ac 100644 --- a/users/tazjin/home/shared.nix +++ b/users/tazjin/home/shared.nix @@ -8,7 +8,7 @@ let # URL handler to open `tg://` URLs in telega.el telega-launcher = pkgs.writeShellScriptBin "telega-launcher" '' echo "Opening ''${1} in telega.el ..." - ${pkgs.emacs-unstable}/bin/emacsclient -e "(telega-browse-url \"''${1}\")" + ${depot.users.tazjin.emacs.emacs}/bin/emacsclient -e "(telega-browse-url \"''${1}\")" ''; in { diff --git a/users/tazjin/homepage/entries.nix b/users/tazjin/homepage/entries.nix index 5352f8cdcf..0e98c073ef 100644 --- a/users/tazjin/homepage/entries.nix +++ b/users/tazjin/homepage/entries.nix @@ -6,6 +6,16 @@ let in [ { + class = "project"; + title = "VolgaSprint - Nix hacking in Kazan"; + url = "https://volgasprint.org/"; + date = 1712307024; + description = '' + Hacking on Nix projects for a week in Kazan, Russia, in August + 2024. Come join us! + ''; + } + { class = "misc"; title = "@tazlog on Telegram"; url = "https://t.me/tazlog"; diff --git a/users/tazjin/keys/default.nix b/users/tazjin/keys/default.nix index 3022c46f91..16b232b094 100644 --- a/users/tazjin/keys/default.nix +++ b/users/tazjin/keys/default.nix @@ -7,6 +7,6 @@ in withAll { tverskoy_ed25519 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM1fGWz/gsq+ZeZXjvUrV+pBlanw1c3zJ9kLTax9FWQy tazjin@tverskoy"; zamalek_sk = "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIOAw3OaPAjnC6hArGYEmBoXhPf7aZdRGlDZcSqm6gbB8AAAABHNzaDo= tazjin@zamalek"; zamalek_ed25519 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDBRXeb8EuecLHP0bW4zuebXp4KRnXgJTZfeVWXQ1n1R tazjin@zamalek"; - khamovnik_yk = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7G+RWknTydjXe971ZkwUCAvKCSe1H+j9Zzz/nAHGsKU5Rs9VNnbB0Vzq5HzXcvvy68B8XG8Iecrgn3AeUmiKHIgg0MCI+B7cSc+OXZHfh+J+9YFZiTaVhO5NDfqFgkmtd9jxBV/1FvcweBLFocm0z+gZ7dNDuEECvmlai9stkrCuRMUEMlow5GaJtbdRGUljMse294XqN7kUV1DHOA1vNOp2qr2sDCxd9r2L2optCmIB1PxOM/XNizRn2w9yA5nDR673UCHvr2pDXfItYswfdo10Zbe+BcNkjT8TiHAsVBCBaKytUdqXk53H8SHMPHHhQH+HRrnZMOfWbxag7n+e5 tazjin@khamovnik"; + khamovnik_yk = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPgOyR4rRM8IaVGgN2ZxGlKtd7GLYbxdRTRa3u9EhRNSkHAvRTN9sgw7mm0iPLnHChPy10anKV43vTaIm906Gm8="; khamovnik_agenix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIG4YSl5+DHQR3rOoBJLQfQ840U0CrYkByMKdzu/LDxoT tazjin@khamovnik"; } diff --git a/users/tazjin/nixos/khamovnik/default.nix b/users/tazjin/nixos/khamovnik/default.nix index 414301159b..8ea925c90d 100644 --- a/users/tazjin/nixos/khamovnik/default.nix +++ b/users/tazjin/nixos/khamovnik/default.nix @@ -72,6 +72,7 @@ in hardware.opengl.extraPackages = with pkgs; [ intel-compute-runtime intel-media-driver + intel-vaapi-driver ]; # from generated configuration.nix @@ -116,6 +117,8 @@ in # Try to work around Intel CPU throttling bugs services.throttled.enable = true; + virtualisation.docker.enable = true; + hardware.bluetooth.enable = true; users.users.tazjin.extraGroups = [ "tss" ]; diff --git a/users/tazjin/nixos/koptevo/default.nix b/users/tazjin/nixos/koptevo/default.nix index 39a4887c72..ea8dfd4bd8 100644 --- a/users/tazjin/nixos/koptevo/default.nix +++ b/users/tazjin/nixos/koptevo/default.nix @@ -129,10 +129,11 @@ in ''; }; - # I don't use the podcast feature, but I *have to* supply podcasts - # to gonic ... + # I don't use the podcast nor playlist feature, + # but I *have to* supply podcasts to gonic ... systemd.tmpfiles.rules = [ "d /tmp/fake-podcasts 0555 nobody nobody -" + "d /tmp/fake-playlists 0555 nobody nobody -" ]; services.gonic = { @@ -142,6 +143,7 @@ in scan-interval = 5; scan-at-start-enabled = true; podcast-path = [ "/tmp/fake-podcasts" ]; + playlists-path = [ "/tmp/fake-playlists" ]; music-path = [ "/var/lib/geesefs/tazjins-files/music" ]; }; }; diff --git a/users/tazjin/nixos/modules/physical.nix b/users/tazjin/nixos/modules/physical.nix index 6d48a076bf..d469da7e5a 100644 --- a/users/tazjin/nixos/modules/physical.nix +++ b/users/tazjin/nixos/modules/physical.nix @@ -24,7 +24,7 @@ in users.tazjin.chase-geese config.tazjin.emacs third_party.agenix.cli - third_party.josh + tools.when ]) ++ # programs from nixpkgs @@ -50,6 +50,7 @@ in hyperfine iftop imagemagick + josh jq lieer maim diff --git a/users/tazjin/tgsa/Cargo.lock b/users/tazjin/tgsa/Cargo.lock index f22d6c07e6..6be9c490d4 100644 --- a/users/tazjin/tgsa/Cargo.lock +++ b/users/tazjin/tgsa/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" [[package]] name = "ascii" @@ -31,9 +31,9 @@ checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" [[package]] name = "autocfg" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" [[package]] name = "base64" @@ -43,9 +43,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.5" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "bitflags" @@ -55,9 +55,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" [[package]] name = "buf_redux" @@ -71,9 +71,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" [[package]] name = "byteorder" @@ -83,12 +83,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "cc" -version = "1.0.84" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f8e7c90afad890484a21653d08b6e209ae34770fb5ee298f9c699fcc1e5c856" -dependencies = [ - "libc", -] +checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" [[package]] name = "cfg-if" @@ -98,21 +95,21 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e" dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", - "windows-targets", + "windows-targets 0.52.4", ] [[package]] name = "chunked_transfer" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cca491388666e04d7248af3f60f0c40cfb0991c72205595d7c396e3510207d1a" +checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" [[package]] name = "convert_case" @@ -122,9 +119,9 @@ checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "crimp" @@ -161,14 +158,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ "quote", - "syn 2.0.39", + "syn 2.0.57", ] [[package]] name = "curl" -version = "0.4.44" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "509bd11746c7ac09ebd19f0b17782eae80aadee26237658a6b4808afb5c11a22" +checksum = "1e2161dd6eba090ff1594084e95fd67aeccf04382ffea77999ea94ed42ec67b6" dependencies = [ "curl-sys", "libc", @@ -176,14 +173,14 @@ dependencies = [ "openssl-sys", "schannel", "socket2", - "winapi", + "windows-sys", ] [[package]] name = "curl-sys" -version = "0.4.68+curl-8.4.0" +version = "0.4.72+curl-8.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4a0d18d88360e374b16b2273c832b5e57258ffc1d4aa4f96b108e0738d5752f" +checksum = "29cbdc8314c447d11e8fd156dcdd031d9e02a7a976163e396b548c03153bc9ea" dependencies = [ "cc", "libc", @@ -196,9 +193,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", ] @@ -239,9 +236,9 @@ checksum = "3a68a4904193147e0a8dec3314640e6db742afd5f6e634f428a6af230d9b3591" [[package]] name = "errno" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c18ee0ed65a5f1f81cac6b1d213b69c35fa47d4252ad41f1486dbd8226fe36e" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ "libc", "windows-sys", @@ -249,19 +246,19 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" [[package]] name = "filetime" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall", "windows-sys", ] @@ -282,9 +279,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -330,9 +327,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "libc", @@ -341,9 +338,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "html5ever" @@ -373,9 +370,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -396,9 +393,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -412,15 +409,15 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" -version = "0.3.65" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -433,15 +430,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.150" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libz-sys" -version = "1.1.12" +version = "1.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +checksum = "5e143b5e666b2695d28f6bca6497720813f699c9602dd7f5cac91008b8ada7f9" dependencies = [ "cc", "libc", @@ -451,9 +448,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.11" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" @@ -467,9 +464,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "mac" @@ -499,9 +496,9 @@ checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "mime" @@ -539,9 +536,9 @@ dependencies = [ [[package]] name = "new_debug_unreachable" -version = "1.0.4" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" [[package]] name = "nodrop" @@ -550,10 +547,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" [[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", ] @@ -570,26 +573,26 @@ dependencies = [ [[package]] name = "num_threads" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" dependencies = [ "libc", ] [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "openssl" -version = "0.10.59" +version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a257ad03cd8fb16ad4172fedf8094451e1af1c4b70097636ef2eac9a5f0cc33" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "cfg-if", "foreign-types", "libc", @@ -606,7 +609,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.57", ] [[package]] @@ -617,9 +620,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.95" +version = "0.9.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" dependencies = [ "cc", "libc", @@ -645,16 +648,16 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.4.1", + "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "phf" @@ -750,9 +753,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "powerfmt" @@ -780,9 +783,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] @@ -795,9 +798,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -862,7 +865,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.11", + "getrandom 0.2.12", ] [[package]] @@ -885,15 +888,6 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_syscall" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" @@ -949,11 +943,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.21" +version = "0.38.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" +checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "errno", "libc", "linux-raw-sys", @@ -962,9 +956,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "safemem" @@ -974,9 +968,9 @@ checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ "windows-sys", ] @@ -1025,37 +1019,37 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.20" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" [[package]] name = "serde" -version = "1.0.192" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.192" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.57", ] [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" dependencies = [ - "itoa 1.0.9", + "itoa 1.0.11", "ryu", "serde", ] @@ -1084,18 +1078,18 @@ checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" [[package]] name = "smallvec" -version = "1.11.2" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.4.10" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" dependencies = [ "libc", - "winapi", + "windows-sys", ] [[package]] @@ -1149,9 +1143,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.39" +version = "2.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +checksum = "11a6ae1e52eb25aab8f3fb9fca13be982a373b8f1157ca14b897a825ba4a2d35" dependencies = [ "proc-macro2", "quote", @@ -1160,13 +1154,12 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.1" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", "fastrand", - "redox_syscall 0.4.1", "rustix", "windows-sys", ] @@ -1187,7 +1180,7 @@ name = "tgsa" version = "0.1.0" dependencies = [ "anyhow", - "base64 0.21.5", + "base64 0.21.7", "crimp", "ego-tree", "lazy_static", @@ -1217,12 +1210,13 @@ dependencies = [ [[package]] name = "time" -version = "0.3.30" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" dependencies = [ "deranged", "libc", + "num-conv", "num_threads", "powerfmt", "serde", @@ -1282,9 +1276,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -1294,9 +1288,9 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] @@ -1315,9 +1309,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -1356,9 +1350,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.88" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1366,24 +1360,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.88" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.57", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.88" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1391,28 +1385,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.88" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.57", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.88" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "web-sys" -version = "0.3.65" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", @@ -1442,20 +1436,20 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.51.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets", + "windows-targets 0.52.4", ] [[package]] name = "windows-sys" -version = "0.48.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.4", ] [[package]] @@ -1464,13 +1458,28 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", ] [[package]] @@ -1480,37 +1489,79 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + +[[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + +[[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + +[[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + +[[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + +[[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + +[[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" diff --git a/views/kit/default.nix b/views/kit/default.nix index b6d21f6196..bb4b37e36c 100644 --- a/views/kit/default.nix +++ b/views/kit/default.nix @@ -23,14 +23,22 @@ pkgs.lib.fix (self: { buildkite = import ./buildkite { inherit pkgs; - depot.nix.readTree = self.readTree; + depot.nix = { + inherit (self) readTree dependency-analyzer; + }; }; checks = import ./checks { inherit pkgs; }; + dependency-analyzer = import ./dependency-analyzer { + inherit pkgs; + inherit (pkgs) lib; + depot.nix.stateMonad = self.stateMonad; + }; lazy-deps = import ./lazy-deps { inherit pkgs; lib = pkgs.lib; }; magrathea = import ./magrathea { inherit pkgs; }; readTree = import ./readTree { }; + stateMonad = import ./stateMonad { }; }) diff --git a/views/kit/workspace.josh b/views/kit/workspace.josh index 202dee350d..63b3cd49b4 100644 --- a/views/kit/workspace.josh +++ b/views/kit/workspace.josh @@ -3,8 +3,10 @@ besadii = :/ops/besadii :/nix:[ ::buildGo/ ::buildkite/ + ::dependency-analyzer/ ::lazy-deps/ ::readTree/ + ::stateMonad/ ] :/third_party:[ ::nixpkgs/ diff --git a/views/tvix/workspace.josh b/views/tvix/workspace.josh index 58c83f861b..57d5f8a8c4 100644 --- a/views/tvix/workspace.josh +++ b/views/tvix/workspace.josh @@ -1,8 +1,9 @@ :/third_party:[ - ::nixpkgs/ + ::nixpkgs/, ::sources/ ] ::tvix:exclude[ ::tvix/default.nix ::tvix/**/default.nix:exclude[::tvix/eval/src/tests/tvix_tests/directory/default.nix] ]:/tvix +::third_party/overlays/patches/cbtemulator-uds.patch:/third_party/overlays/patches:prefix=nixpkgs diff --git a/web/panettone/default.nix b/web/panettone/default.nix index 283f834994..60fca99e75 100644 --- a/web/panettone/default.nix +++ b/web/panettone/default.nix @@ -1,4 +1,4 @@ -{ depot, ... }: +{ depot, pkgs, ... }: depot.nix.buildLisp.program { name = "panettone"; @@ -9,6 +9,7 @@ depot.nix.buildLisp.program { cl-ppcre cl-smtp cl-who + str defclass-std drakma easy-routes @@ -23,6 +24,15 @@ depot.nix.buildLisp.program { srcs = [ ./panettone.asd ./src/packages.lisp + (pkgs.writeText "build.lisp" '' + (defpackage build + (:use :cl :alexandria) + (:export :*migrations-dir* :*static-dir*)) + (in-package :build) + (declaim (optimize (safety 3))) + (defvar *migrations-dir* "${./src/migrations}") + (defvar *static-dir* "${./src/static}") + '') ./src/util.lisp ./src/css.lisp ./src/email.lisp diff --git a/web/panettone/docker-compose.yml b/web/panettone/docker-compose.yml index 84723667e6..18e3498306 100644 --- a/web/panettone/docker-compose.yml +++ b/web/panettone/docker-compose.yml @@ -1,7 +1,7 @@ -version: '3.4' +version: "3.4" services: postgres: - image: postgres:11 + image: postgres:16 restart: always environment: POSTGRES_USER: panettone diff --git a/web/panettone/src/authentication.lisp b/web/panettone/src/authentication.lisp index 3ce07aa8d7..496a0e0bd7 100644 --- a/web/panettone/src/authentication.lisp +++ b/web/panettone/src/authentication.lisp @@ -42,23 +42,31 @@ the user, however email addresses are temporarily not available." (or (uiop:getenv "OAUTH2_REDIRECT_URI") "https://b.tvl.fyi/auth")) +(comment + (setq *oauth2-redirect-uri* "http://localhost:6161/auth") + ) + (defun initialise-oauth2 () "Initialise all settings needed for OAuth2" (setq *oauth2-auth-endpoint* - (or (uiop:getenv "OAUTH2_AUTH_ENDPOINT") + (or *oauth2-auth-endpoint* + (uiop:getenv "OAUTH2_AUTH_ENDPOINT") "https://auth.tvl.fyi/auth/realms/TVL/protocol/openid-connect/auth")) (setq *oauth2-token-endpoint* - (or (uiop:getenv "OAUTH2_TOKEN_ENDPOINT") + (or *oauth2-token-endpoint* + (uiop:getenv "OAUTH2_TOKEN_ENDPOINT") "https://auth.tvl.fyi/auth/realms/TVL/protocol/openid-connect/token")) (setq *oauth2-client-id* - (or (uiop:getenv "OAUTH2_CLIENT_ID") + (or *oauth2-client-id* + (uiop:getenv "OAUTH2_CLIENT_ID") "panettone")) (setq *oauth2-client-secret* - (or (uiop:getenv "OAUTH2_CLIENT_SECRET") + (or *oauth2-client-secret* + (uiop:getenv "OAUTH2_CLIENT_SECRET") (error "OAUTH2_CLIENT_SECRET must be set!")))) (defun auth-url () diff --git a/web/panettone/src/css.lisp b/web/panettone/src/css.lisp index aa753cb50f..3bba2bb591 100644 --- a/web/panettone/src/css.lisp +++ b/web/panettone/src/css.lisp @@ -49,7 +49,24 @@ :color "var(--primary)"))) (.comment-count - :color "var(--gray)"))) + :color "var(--gray)") + + (.issue-links + :display "flex" + :flex-direction "row" + :align-items "center" + :justify-content "space-between" + :flex-wrap "wrap") + + (.issue-search + ((:and input (:= type "search")) + :padding "0.5rem" + :background-image "url('static/search.png')" + :background-position "10px 10px" + :background-repeat "no-repeat" + :background-size "1rem" + :padding-left "2rem" + :border "1px" "solid" "var(--gray)")))) (defparameter issue-history-styles `((.issue-history @@ -220,4 +237,15 @@ :margin "0 auto") (.created-by-at - :color "var(--gray)"))) + :color "var(--gray)") + + ;; screen-reader-only content + (.sr-only + :border 0 + :clip "rect(0 0 0 0)" + :height "1px" + :margin "-1px" + :overflow "hidden" + :padding 0 + :position "absolute" + :width "1px"))) diff --git a/web/panettone/src/migrations/1-init-schema.lisp b/web/panettone/src/migrations/1-init-schema.lisp new file mode 100644 index 0000000000..3be6c4fcc0 --- /dev/null +++ b/web/panettone/src/migrations/1-init-schema.lisp @@ -0,0 +1,23 @@ +"Initialize the database schema from before migrations were added" + +(defun ddl/create-issue-status () + "Issue DDL to create the `issue-status' type, if it doesn't exist" + (unless (query (:select (:exists (:select 1 + :from 'pg_type + :where (:= 'typname "issue_status")))) + :single) + (query (sql-compile + `(:create-enum issue-status ,panettone.model:+issue-statuses+))))) + +(defun ddl/create-tables () + "Issue DDL to create all tables, if they don't already exist." + (dolist (table '(panettone.model:issue + panettone.model:issue-comment + panettone.model:issue-event + panettone.model:user-settings)) + (unless (table-exists-p (dao-table-name table)) + (create-table table)))) + +(defun up () + (ddl/create-issue-status) + (ddl/create-tables)) diff --git a/web/panettone/src/migrations/3920286378-add-issue-tsv.lisp b/web/panettone/src/migrations/3920286378-add-issue-tsv.lisp new file mode 100644 index 0000000000..2a965a7bba --- /dev/null +++ b/web/panettone/src/migrations/3920286378-add-issue-tsv.lisp @@ -0,0 +1,5 @@ +"Add tsvector for full-text search of issues" + +(defun up () + (query "ALTER TABLE issues ADD COLUMN tsv tsvector GENERATED ALWAYS AS (to_tsvector('english', subject || ' ' || body)) STORED") + (query "CREATE INDEX issues_tsv_index ON issues USING GIN (tsv);")) diff --git a/web/panettone/src/migrations/3921488651-create-users-table.lisp b/web/panettone/src/migrations/3921488651-create-users-table.lisp new file mode 100644 index 0000000000..2598ab101e --- /dev/null +++ b/web/panettone/src/migrations/3921488651-create-users-table.lisp @@ -0,0 +1,6 @@ +"Add a table to store information about users, load the initial set of users +from the authentication provider, and change fks for other tables" + +(defun up () + (panettone.model:create-table-if-not-exists + 'panettone.model:user)) diff --git a/web/panettone/src/model.lisp b/web/panettone/src/model.lisp index c54a0ae474..a106e9479b 100644 --- a/web/panettone/src/model.lisp +++ b/web/panettone/src/model.lisp @@ -1,6 +1,8 @@ (in-package :panettone.model) (declaim (optimize (safety 3))) +(setq pomo:*ignore-unknown-columns* t) + (defvar *pg-spec* nil "Connection spec for use with the with-connection macro. Needs to be initialised at launch time.") @@ -20,10 +22,39 @@ initialised at launch time.") "Initialises the connection spec used for all Postgres connections." (setq *pg-spec* (make-pg-spec))) +(defun connect-to-db () + "Connect using *PG-SPEC* at the top-level, for use during development" + (apply #'connect-toplevel + (loop for v in *pg-spec* + until (eq v :pooled-p) + collect v))) + +(defun pg-spec->url (&optional (spec *pg-spec*)) + (destructuring-bind (db user password host &key port &allow-other-keys) spec + (format nil + "postgres://~A:~A@~A:~A/~A" + user password host port db))) + ;;; ;;; Schema ;;; +(defclass user () + ((sub :col-type uuid :initarg :sub :accessor sub + :documentation + "ID for the user in the authentication provider. Taken from the `:SUB' + field in the JWT when the user first logged in") + (username :col-type string :initarg :username :accessor username) + (email :col-type string :initarg :email :accessor email)) + (:metaclass dao-class) + (:keys sub) + (:table-name users) + (:documentation + "Panettone users. Uses an external authentication provider.")) + +(deftable (user "users") + (!dao-def)) + (defclass user-settings () ((user-dn :col-type string :initarg :user-dn :accessor user-dn) (enable-email-notifications @@ -77,15 +108,6 @@ initialised at launch time.") "Type specifier for the status of an `issue'" (cons 'member +issue-statuses+)) -(defun ddl/create-issue-status () - "Issue DDL to create the `issue-status' type, if it doesn't exist" - (unless (query (:select (:exists (:select 1 - :from 'pg_type - :where (:= 'typname "issue_status")))) - :single) - (query (sql-compile - `(:create-enum issue-status ,+issue-statuses+))))) - (defclass has-created-at () ((created-at :col-type timestamp :col-default (local-time:now) @@ -192,23 +214,171 @@ its new value will be formatted using ~A into NEW-VALUE")) (!dao-def) (!foreign 'issues 'issue-id 'id :on-delete :cascade :on-update :cascade)) -(define-constant +all-tables+ - '(issue - issue-comment - issue-event - user-settings) - :test #'equal) +(defclass migration () + ((version + :col-type bigint + :primary-key t + :initarg :version + :accessor version) + (name :col-type string :initarg :name :accessor name) + (docstring :col-type string :initarg :docstring :accessor docstring) + (path :col-type string + :type pathname + :initarg :path + :accessor path + :col-export namestring + :col-import parse-namestring) + (package :type keyword :initarg :package :accessor migration-package)) + (:metaclass dao-class) + (:keys version) + (:table-name migrations) + (:documentation "Migration scripts that have been run on the database")) +(deftable migration (!dao-def)) -(defun ddl/create-tables () - "Issue DDL to create all tables, if they don't already exist." - (dolist (table +all-tables+) - (unless (table-exists-p (dao-table-name table)) - (create-table table)))) +;;; +;;; Utils +;;; + +(defun create-table-if-not-exists (name) + " Takes the name of a dao-class and creates the table identified by symbol by +executing all forms in its definition as found in the *tables* list, if it does +not already exist." + (unless (table-exists-p (dao-table-name name)) + (create-table name))) + +;;; +;;; Migrations +;;; -(defun ddl/init () - "Idempotently initialize the full database schema for Panettone" - (ddl/create-issue-status) - (ddl/create-tables)) +(defun ensure-migrations-table () + "Ensure the migrations table exists" + (unless (table-exists-p (dao-table-name 'migration)) + (create-table 'migration))) + +(define-build-time-var *migrations-dir* "migrations/" + "The directory where migrations are stored") + +(defun load-migration-docstring (migration-path) + "If the first form in the file pointed to by `migration-pathname` is + a string, return it, otherwise return NIL." + + (handler-case + (with-open-file (s migration-path) + (when-let ((form (read s))) + (when (stringp form) form))) + (t () nil))) + +(defun load-migration (path) + (let* ((parts (str:split #\- (pathname-name path) :limit 2)) + (version (parse-integer (car parts))) + (name (cadr parts)) + (docstring (load-migration-docstring path)) + (package (intern (format nil "MIGRATION-~A" version) + :keyword)) + (migration (make-instance 'migration + :version version + :name name + :docstring docstring + :path path + :package package))) + (uiop/package:ensure-package package + :use '(#:common-lisp + #:postmodern + #:panettone.model)) + (let ((*package* (find-package package))) + (load path)) + + migration)) + +(defun run-migration (migration) + (declare (type migration migration)) + (with-transaction () + (format t "Running migration ~A (version ~A)" + (name migration) + (version migration)) + (query + (sql-compile + `(:delete-from migrations + :where (= version ,(version migration))))) + (uiop:symbol-call (migration-package migration) :up) + (insert-dao migration))) + +(defun list-migration-files () + (remove-if-not + (lambda (pn) (string= "lisp" (pathname-type pn))) + (uiop:directory-files (util:->dir *migrations-dir*)))) + +(defun load-migrations () + (mapcar #'load-migration (list-migration-files))) + +(defun generate-migration (name &key documentation) + "Generate a new database migration with the given NAME, optionally +prepopulated with the given DOCUMENTATION. + +Returns the file that the migration is located at, as a `pathname'. Write Lisp +code in this migration file to define a function called `up', which will be run +in the context of a database transaction and should perform the migration." + (let* ((version (get-universal-time)) + (filename (format nil "~A-~A.lisp" + version + name)) + (pathname + (merge-pathnames filename *migrations-dir*))) + (with-open-file (stream pathname + :direction :output + :if-does-not-exist :create) + (when documentation + (format stream "~S~%~%" documentation)) + + (format stream "(defun up ()~%)")) + pathname)) + +(defun migrations-already-run () + "Query the database for a list of migrations that have already been run" + (query-dao 'migration (sql-compile '(:select * :from migrations)))) + +(define-condition migration-name-mismatch () + ((version :type integer :initarg :version) + (name-in-database :type string :initarg :name-in-database) + (name-in-code :type string :initarg :name-in-code)) + (:report + (lambda (cond stream) + (format stream "Migration mismatch: Migration version ~A has name ~S in the database, but we have name ~S" + (slot-value cond 'version) + (slot-value cond 'name-in-database) + (slot-value cond 'name-in-code))))) + +(defun migrate () + "Migrate the database, running all migrations that have not yet been run" + (ensure-migrations-table) + (format t "Running migrations from ~A...~%" *migrations-dir*) + (let* ((all-migrations (load-migrations)) + (already-run (migrations-already-run)) + (num-migrations-run 0)) + (iter (for migration in all-migrations) + (if-let ((existing (find-if (lambda (existing) + (= (version existing) + (version migration))) + already-run))) + (progn + (unless (string= (name migration) + (name existing)) + (restart-case + (error 'migration-name-mismatch + :version (version existing) + :name-in-database (name existing) + :name-in-code (name migration)) + (skip () + :report "Skip this migration" + (next-iteration)) + (run-and-overwrite () + :report "Run this migration anyway, overwriting the previous migration" + (run-migration migration)))) + (next-iteration)) + ;; otherwise, run the migration + (run-migration migration)) + (incf num-migrations-run)) + (format t "Ran ~A migration~:P~%" num-migrations-run))) ;;; ;;; Querying @@ -243,24 +413,31 @@ type `ISSUE-NOT-FOUND'." :where (:= 'id id)))) :single)) -(defun list-issues (&key status (with '(:num-comments))) +(defun list-issues (&key status search (with '(:num-comments))) "Return a list of all issues with the given STATUS (or all if nil), ordered by ID descending. If WITH contains `:NUM-COMMENTS' (the default) each issue will have the `num-comments' slot filled with the number of comments on that issue (to avoid N+1 queries)." - (let* ((condition (unless (null status) - `(:where (:= status $1)))) + (let* ((conditions + (and-where* + (unless (null status) + `(:= status $1)) + (when (str:non-blank-string-p search) + `(:@@ tsv (:websearch-to-tsquery ,search))))) (select (if (find :num-comments with) `(:select issues.* (:as (:count issue-comments.id) num-comments) - :from issues - :left-join issue-comments - :on (:= issues.id issue-comments.issue-id) - ,@condition - :group-by issues.id) - `(:select * :from issues ,@condition))) + :from issues + :left-join issue-comments + :on (:= issues.id issue-comments.issue-id) + :where ,conditions + :group-by issues.id) + `(:select * :from issues :where ,conditions))) + (order (if (str:non-blank-string-p search) + `(:desc (:ts-rank-cd tsv (:websearch-to-tsquery ,search))) + `(:desc id))) (query (sql-compile - `(:order-by ,select (:desc id))))) + `(:order-by ,select ,order)))) (with-column-writers ('num_comments 'num-comments) (query-dao 'issue query status)))) @@ -409,12 +586,23 @@ explicitly subscribing to / unsubscribing from individual issues." (comment - (ddl/init) + (make-instance 'issue :subject "test") - (create-issue :subject "test" - :author-dn "cn=grfn,ou=users,dc=tvl,dc=fyi") + + (with-connection *pg-spec* + (create-issue :subject "test" + :author-dn "cn=aspen,ou=users,dc=tvl,dc=fyi")) (issue-commenter-dns 1) (issue-subscribers 1) + ;; Creating new migrations + (setq *migrations-dir* (merge-pathnames "migrations/")) + (generate-migration "create-users-table" + :documentation "Add a table to store information about users") + (load-migrations) + + ;; Running migrations + (with-connection *pg-spec* + (migrate)) ) diff --git a/web/panettone/src/packages.lisp b/web/panettone/src/packages.lisp index 4ff4c070f0..8e77c0ff2b 100644 --- a/web/panettone/src/packages.lisp +++ b/web/panettone/src/packages.lisp @@ -1,7 +1,10 @@ (defpackage panettone.util + (:nicknames :util) (:use :cl :klatre) (:import-from :alexandria :when-let) - (:export :integer-env :add-missing-base64-padding)) + (:export + :integer-env :add-missing-base64-padding :and-where :and-where* + :define-build-time-var :->dir)) (defpackage panettone.css (:use :cl :lass) @@ -36,16 +39,21 @@ (:import-from :alexandria :if-let :when-let :define-constant) (:export :prepare-db-connections - :ddl/init + :migrate :*pg-spec* + :create-table-if-not-exists + + :user + :sub :username :email + :user-settings :user-dn :enable-email-notifications-p :settings-for-user :update-user-settings :enable-email-notifications - :issue :issue-comment :issue-event + :issue :issue-comment :issue-event :migration :id :subject :body :author-dn :issue-id :status :created-at :acting-user-dn - :field :previous-value :new-value + :field :previous-value :new-value :+issue-statuses+ :get-issue :issue-exists-p :list-issues :create-issue :set-issue-status :update-issue :delete-issue :issue-not-found :not-found-id diff --git a/web/panettone/src/panettone.lisp b/web/panettone/src/panettone.lisp index d87ac5ed46..37d194d0f9 100644 --- a/web/panettone/src/panettone.lisp +++ b/web/panettone/src/panettone.lisp @@ -193,7 +193,21 @@ (who:esc (format nil "~A comment~:p" num-comments)))))))))))))) -(defun render/index (&key issues) +(defun render/issue-search (&key search) + (who:with-html-output (*standard-output*) + (:form + :method "get" + :class "issue-search" + (:input :type "search" + :name "search" + :title "Issue search query" + :value search) + (:input + :type "submit" + :value "Search Issues" + :class "sr-only")))) + +(defun render/index (&key issues search) (render () (:header (:h1 "Issues") @@ -205,17 +219,19 @@ (:main (:div :class "issue-links" - (:a :href "/issues/closed" "View closed issues")) + (:a :href "/issues/closed" "View closed issues") + (render/issue-search :search search)) (render/issue-list :issues issues)))) -(defun render/closed-issues (&key issues) +(defun render/closed-issues (&key issues search) (render () (:header (:h1 "Closed issues")) (:main (:div :class "issue-links" - (:a :href "/" "View open isues")) + (:a :href "/" "View open isues") + (render/issue-search :search search)) (render/issue-list :issues issues)))) (defun render/issue-form (&optional issue message) @@ -442,9 +458,11 @@ given subject an body (in a thread, to avoid blocking)" (hunchentoot:delete-session-value 'user) (hunchentoot:redirect "/")) -(defroute index ("/" :decorators (@auth-optional @db)) () - (let ((issues (model:list-issues :status :open))) - (render/index :issues issues))) +(defroute index ("/" :decorators (@auth-optional @db)) (&get search) + (let ((issues (model:list-issues :status :open + :search search))) + (render/index :issues issues + :search search))) (defroute settings ("/settings" :method :get :decorators (@auth @db)) () (render/settings)) @@ -458,9 +476,12 @@ given subject an body (in a thread, to avoid blocking)" (render/settings))) (defroute handle-closed-issues - ("/issues/closed" :decorators (@auth-optional @db)) () - (let ((issues (model:list-issues :status :closed))) - (render/closed-issues :issues issues))) + ("/issues/closed" :decorators (@auth-optional @db)) + (&get search) + (let ((issues (model:list-issues :status :closed + :search search))) + (render/closed-issues :issues issues + :search search))) (defroute new-issue ("/issues/new" :decorators (@auth)) () (render/issue-form)) @@ -606,7 +627,10 @@ given subject an body (in a thread, to avoid blocking)" (defun migrate-db () "Migrate the database to the latest version of the schema" (pomo:with-connection *pg-spec* - (model:ddl/init))) + (model:migrate))) + +(define-build-time-var *static-dir* "static/" + "Directory to serve static files from") (defun start-panettone (&key port session-secret) (authn:initialise-oauth2) @@ -619,7 +643,14 @@ given subject an body (in a thread, to avoid blocking)" (setq hunchentoot:*session-max-time* (* 60 60 24 90)) (setq *acceptor* - (make-instance 'easy-routes:routes-acceptor :port port)) + (make-instance 'easy-routes:easy-routes-acceptor :port port)) + + (push + (hunchentoot:create-folder-dispatcher-and-handler + "/static/" + (util:->dir *static-dir*)) + hunchentoot:*dispatch-table*) + (hunchentoot:start *acceptor*)) (defun main () diff --git a/web/panettone/src/static/search.png b/web/panettone/src/static/search.png new file mode 100644 index 0000000000..0fd78c6651 --- /dev/null +++ b/web/panettone/src/static/search.png Binary files differdiff --git a/web/panettone/src/util.lisp b/web/panettone/src/util.lisp index 2abedf7b8f..4c3c4f1aa6 100644 --- a/web/panettone/src/util.lisp +++ b/web/panettone/src/util.lisp @@ -13,3 +13,27 @@ that it can be successfully decoded by the `BASE64' package" (let* ((needed-padding (mod (length s) 4)) (pad-chars (if (zerop needed-padding) 0 (- 4 needed-padding)))) (format nil "~A~v@{~A~:*~}" s pad-chars "="))) + +(defun and-where (clauses) + "Combine all non-nil clauses in CLAUSES into a single S-SQL WHERE form" + (let ((clauses (remove nil clauses))) + (if (null clauses) t + (reduce (lambda (x y) `(:and ,x ,y)) clauses)))) + +(defun and-where* (&rest clauses) + "Combine all non-nil clauses in CLAUSES into a single S-SQL WHERE form" + (and-where clauses)) + +(defmacro define-build-time-var + (name value-if-not-in-build &optional (doc nil)) + `(defvar ,name + (or (when-let ((package (find-package :build))) + (let ((sym (find-symbol ,(symbol-name name) package))) + (when (boundp sym) (symbol-value sym)))) + ,value-if-not-in-build) + ,doc)) + +(defun ->dir (dir) + (if (char-equal (uiop:last-char dir) #\/) + dir + (concatenate 'string dir "/"))) |
