From 85132ec6068a16e1df3f7d6e0e8daa200fdf96be Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Thu, 24 Oct 2024 13:37:10 +0200 Subject: feat(users/flokli/nixon-2024): init Change-Id: I547a5504f6782169dd7280c0b5c21cbe66068a2e Reviewed-on: https://cl.tvl.fyi/c/depot/+/12682 Tested-by: BuildkiteCI Reviewed-by: flokli Autosubmit: flokli --- .../2024-10-25-nixcon-tvix/.gitignore | 1 + .../2024-10-25-nixcon-tvix/default.nix | 35 +++ .../2024-10-25-nixcon-tvix/nar-bridge.png | Bin 0 -> 277384 bytes .../2024-10-25-nixcon-tvix/nix-compat.png | Bin 0 -> 380478 bytes .../2024-10-25-nixcon-tvix/presentation.md | 261 +++++++++++++++++++++ .../2024-10-25-nixcon-tvix/tvix-build.png | Bin 0 -> 216565 bytes .../2024-10-25-nixcon-tvix/tvix-castore.png | Bin 0 -> 409980 bytes .../2024-10-25-nixcon-tvix/tvix-eval.png | Bin 0 -> 248106 bytes .../2024-10-25-nixcon-tvix/tvix-glue.png | Bin 0 -> 201091 bytes .../2024-10-25-nixcon-tvix/tvix-store.png | Bin 0 -> 229777 bytes .../2024-10-25-nixcon-tvix/tvix_solves_this.jpg | Bin 0 -> 97147 bytes 11 files changed, 297 insertions(+) create mode 100644 users/flokli/presentations/2024-10-25-nixcon-tvix/.gitignore create mode 100644 users/flokli/presentations/2024-10-25-nixcon-tvix/default.nix create mode 100644 users/flokli/presentations/2024-10-25-nixcon-tvix/nar-bridge.png create mode 100644 users/flokli/presentations/2024-10-25-nixcon-tvix/nix-compat.png create mode 100644 users/flokli/presentations/2024-10-25-nixcon-tvix/presentation.md create mode 100644 users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-build.png create mode 100644 users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-castore.png create mode 100644 users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-eval.png create mode 100644 users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-glue.png create mode 100644 users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-store.png create mode 100644 users/flokli/presentations/2024-10-25-nixcon-tvix/tvix_solves_this.jpg (limited to 'users/flokli/presentations') diff --git a/users/flokli/presentations/2024-10-25-nixcon-tvix/.gitignore b/users/flokli/presentations/2024-10-25-nixcon-tvix/.gitignore new file mode 100644 index 000000000000..397b4a7624e3 --- /dev/null +++ b/users/flokli/presentations/2024-10-25-nixcon-tvix/.gitignore @@ -0,0 +1 @@ +*.log diff --git a/users/flokli/presentations/2024-10-25-nixcon-tvix/default.nix b/users/flokli/presentations/2024-10-25-nixcon-tvix/default.nix new file mode 100644 index 000000000000..88417a4a896a --- /dev/null +++ b/users/flokli/presentations/2024-10-25-nixcon-tvix/default.nix @@ -0,0 +1,35 @@ +{ depot, pkgs, ... }: + +let + inherit (pkgs) + fontconfig qrencode runCommand stdenv; + mkQr = url: runCommand "qrcode.png" { } '' + ${qrencode}/bin/qrencode -o $out -t SVG -s 5 \ + --background=fafafa \ + --foreground=000000 \ + ${url} + ''; +in +stdenv.mkDerivation { + name = "2024-nixcon-tvix"; + src = ./.; + + FONTCONFIG_FILE = pkgs.makeFontsConf { + fontDirectories = with pkgs; [ jetbrains-mono fira fira-code fira-mono lato ]; + }; + + PUPPETEER_EXECUTABLE_PATH = "${pkgs.chromium}/bin/chromium"; + PUPPETEER_SKIP_CHROMIUM_DOWNLOAD = "1"; + + nativeBuildInputs = [ pkgs.reveal-md pkgs.graphviz ]; + + buildPhase = '' + cp ${depot.tvix.logo}/logo.png tvix-logo.png + cp ${mkQr "https://flokli.de"} qrcode-flokli.svg + cp ${mkQr "https://tvix.dev"} qrcode-tvix.svg + + mkdir -p $out + reveal-md --static $out presentation.md + reveal-md --print $out/slides.pdf presentation.md + ''; +} diff --git a/users/flokli/presentations/2024-10-25-nixcon-tvix/nar-bridge.png b/users/flokli/presentations/2024-10-25-nixcon-tvix/nar-bridge.png new file mode 100644 index 000000000000..1cf449a999cc Binary files /dev/null and b/users/flokli/presentations/2024-10-25-nixcon-tvix/nar-bridge.png differ diff --git a/users/flokli/presentations/2024-10-25-nixcon-tvix/nix-compat.png b/users/flokli/presentations/2024-10-25-nixcon-tvix/nix-compat.png new file mode 100644 index 000000000000..958ce1bf8dcf Binary files /dev/null and b/users/flokli/presentations/2024-10-25-nixcon-tvix/nix-compat.png differ diff --git a/users/flokli/presentations/2024-10-25-nixcon-tvix/presentation.md b/users/flokli/presentations/2024-10-25-nixcon-tvix/presentation.md new file mode 100644 index 000000000000..ca636e0a86b2 --- /dev/null +++ b/users/flokli/presentations/2024-10-25-nixcon-tvix/presentation.md @@ -0,0 +1,261 @@ +--- +author: +- Florian Klink +date: 2024-10-25 +title: "Tvix: Status update" +theme: solarized +revealOptions: + transition: 'fade' +--- + +# Tvix: Status update + +![Tvix Logo](tvix-logo.png) + +2023-10-25 + +Florian Klink (flokli) + +--- + +## Whoami + +- flokli +- nixpkgs contributor since 2018, maintaining systemd, nsncd and more +- Freelance Nix/DevOps consultant + +--- + +## What is Tvix? + +--- + +![tvix solves this](./tvix_solves_this.jpg) + + +--- + +## What is Tvix? + +- A Rust re-implementation of the components of the Nix package manager +- Uses different underlying approaches while retaining Nix compatibility "on the surface" +- modular architecture, allowing to recombine aspects to solve your usecase +- No "end-user CLI" for now, focus on getting the foundational architecture right and 100% correctness with Nix + +--- + +## Topics +- Component overview +- Recent developments +- Next steps + +Note: Component overview, to understand a bit better how it's different / updates since the last talk / outlook on roadmap + +--- + +## Structure + + - + **tvix-castore**, the granular data storage/syncing engine. + - + **tvix-store**, the "Nix store implementation" on top of tvix-castore + - + **nix-compat**, a library providing access to data formats, protocols and concepts + - + **tvix-eval**, a bytecode interpreter evaluator and core Nix *language* concepts and builtins + - + **tvix-build**, a generic builder interface + - + **tvix-glue**, combines tvix-eval with tvix-[ca]store and tvix-build + + +Note: one big cargo workspace / go into detail later! / nix-compat: concepts like output path calculation, doesn't depend on tvix crates +tvix-eval: language concepts being Scopes, Thunks, Nix values, "core builtins" + +---- + +![tvix-castore](./tvix-castore.png) + + +Note: Nix does do content addressing on a store path level, we're on a per-file/chunk in file. That model allows granular syncing and reuse of parts, which will speedup substitution/copying. Because everything is ca, it'll also allow decentralization and local p2p substitution. Think about everyone in the same room serving chunks. verified streaming. + +---- + +![tvix-store](./tvix-store.png) + + +Note: track all metadata about store paths (think about the sqlite db), and link to castore instead of storing NAR. NarCalculation: this computation, is super nice to cache, as this info +never needs to expire, and is reconstructable in a pure CA manner. + +---- + +![tvix-store](./nix-compat.png) + + +Note: kept as a somewhat separate library, Tvix "first consumer". But use from your code, it doesn't depend on Tvix bits. Regularly factoring out Nix concepts into this library + +---- + +![tvix-eval](./tvix-eval.png) + + +Note: only includes basic builtins, like string manipulation, math, …. Other crates can bring their own builtins + +---- + +![tvix-build](./tvix-build.png) + + +Note: Not aware of Nix, store paths etc. just flexible enough to express everything in there. Use it for your own build system! + +---- + +![tvix-glue](./tvix-glue.png) + + +Note: connects the evaluator to store and builders + +---- + +![nar-bridge](./nar-bridge.png) + + +Note: Example on how to use this: Nix HTTP binary cache lens into tvix-[ca]store, allows Nix to download from and push into. It renders NARs on-the-fly + +--- + +## Updates +Rough overview. Check blog posts and `git log` for details! + +---- + +## Updates (#1) + + - + Fixes on error catchability and context behaviour + - + More compact Nix Value types (memory-wise) + - + `tvix-cli` REPL global scope manipulation (assign variables and use them in next command) + - + `firefox.outPath` and `pkgsCross.aarch64-multiplatform.firefox.outPath` correct and added to CI 🎉 + +Notes: catchability/context to align behaviour with nix / … / assign variables and use them in the next REPL command + +---- + +## Updates (#2) + + - + OpenTelemetry integration, trace propagation throughout the entire stack + - + more backends in tvix-[ca]store (`object_store`/`local fs`/`redb`/`bigtable`/…) + - + nar-bridge RIIR, was deployed as a fetch-through cache for cache.nixos.org at Bornhack + - + store composition/combinators + - + wiring up of builds (without reference propagation yet, but reference scanning) + +Notes: o11y already proven super helpful for debugging where time is spent / first version of composition / builds waiting on reference propagation + +---- + +## Other Updates + + - + A lot of the tooling for `cache.nixos.org` usage / closure analysis making use of and contributing to `nix-compat` (`@edef`) 🎉 + - + "Replit using `tvix-[ca]store` and reporting 10x storage reduction" (`@cbrewster`) 🎉 + - + "Devenv is switching to Tvix" (Talk on Saturday 12:55, `@domenkozar`) 🎉 + +--- + +## Next steps (in no specific order): + + - + Test suite classification system, to decouple test cases from test runner and share with other Nix impls + - + (Continuous) Docs deployment, website restructuring + - + Interactions with the evaluator (LSP, DAP) + - + Persistent deployment of nar-bridge, as fetch-through cache for `cache.nixos.org` + - + Blob / Chunking protocol improvements (use local chunks where possible, allow readahead) + +Note: Allows filtering, reusing test cases in Nixcpp and Lix. + Mention some behavorial changes found. + Mention fetchTree + Make it easier for new contributors to get started + LSP / Debug adapter protocol / tvix-eval jobs + / … / is gonna improve performance for IO into store paths + +---- + +## Next steps (in no specific order) (cont.): + + - + "Build/Fetch realization goal engine" (tradeoff network bandwidth and CPU time) + - + More per-store metrics and instance names + - + More backends (p2p discovery, ipfs, …) + +Note: needed for builds / to get better insights into cache hit ratios etc. + +--- + +## Contributing + +- + Join the IRC channel (`#tvix-dev` on `hackint`), bridged to Matrix and XMPP +- + Check our issue tracker (b.tvl.fyi), as well as `tvix/docs/src/TODO.md` (but ask!) +- + Try to use it and tell us how you broke it! +- + Sponsoring + +Note: make sure to ask in the channel to ensure noone is already working on this + +--- + +# Thanks to... + +- + all Tvix contributors +- + Nix community members for their input on the architecture +- + Sponsors + +Note: some drive-by, some sticking around longer / NLNET / Clan + +---- + +# Questions? + + + +
+ +
+Florian Klink / flokli.de
+ +
+ +
+Tvix / tvix.dev
+ +
+ +
diff --git a/users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-build.png b/users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-build.png new file mode 100644 index 000000000000..5d11e73f7f12 Binary files /dev/null and b/users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-build.png differ diff --git a/users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-castore.png b/users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-castore.png new file mode 100644 index 000000000000..c1f42e1112b6 Binary files /dev/null and b/users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-castore.png differ diff --git a/users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-eval.png b/users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-eval.png new file mode 100644 index 000000000000..e7c034e6795d Binary files /dev/null and b/users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-eval.png differ diff --git a/users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-glue.png b/users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-glue.png new file mode 100644 index 000000000000..00427aa143c7 Binary files /dev/null and b/users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-glue.png differ diff --git a/users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-store.png b/users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-store.png new file mode 100644 index 000000000000..c35723afc9ab Binary files /dev/null and b/users/flokli/presentations/2024-10-25-nixcon-tvix/tvix-store.png differ diff --git a/users/flokli/presentations/2024-10-25-nixcon-tvix/tvix_solves_this.jpg b/users/flokli/presentations/2024-10-25-nixcon-tvix/tvix_solves_this.jpg new file mode 100644 index 000000000000..d5494b1ec507 Binary files /dev/null and b/users/flokli/presentations/2024-10-25-nixcon-tvix/tvix_solves_this.jpg differ -- cgit 1.4.1