From b8ef3e52637fc783ec03b209ce2f4f4a25152a27 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Fri, 13 Oct 2023 15:53:02 +0200 Subject: feat(users/flokli): add NixCon 2023 presentation This adds the materials for the talk held at NixCon 2023, Darmstadt. It seems it's not really possible to export a PDF out of this, at least I couldn't get reveal-md to do it properly. The video recordings live on https://media.ccc.de/v/nixcon-2023-35254-tvix and https://www.youtube.com/watch?v=j67prAPYScY . Change-Id: Id27dc8c8637ffd455981141383c54d8d1484679e Reviewed-on: https://cl.tvl.fyi/c/depot/+/9750 Reviewed-by: flokli Tested-by: BuildkiteCI --- .../2023-09-09-nixcon-tvix/.gitignore | 1 + .../2023-09-09-nixcon-tvix/architecture.dot | 5 + .../cppnix-example-lexer.cpp | 13 + .../2023-09-09-nixcon-tvix/crate-deps.dot | 19 ++ .../2023-09-09-nixcon-tvix/default.nix | 37 +++ .../2023-09-09-nixcon-tvix/presentation.md | 294 +++++++++++++++++++++ .../2023-09-09-nixcon-tvix/tvix-store-graph.svg | 17 ++ .../2023-09-09-nixcon-tvix/tvixbolt.webm | Bin 0 -> 983213 bytes 8 files changed, 386 insertions(+) create mode 100644 users/flokli/presentations/2023-09-09-nixcon-tvix/.gitignore create mode 100644 users/flokli/presentations/2023-09-09-nixcon-tvix/architecture.dot create mode 100644 users/flokli/presentations/2023-09-09-nixcon-tvix/cppnix-example-lexer.cpp create mode 100644 users/flokli/presentations/2023-09-09-nixcon-tvix/crate-deps.dot create mode 100644 users/flokli/presentations/2023-09-09-nixcon-tvix/default.nix create mode 100644 users/flokli/presentations/2023-09-09-nixcon-tvix/presentation.md create mode 100644 users/flokli/presentations/2023-09-09-nixcon-tvix/tvix-store-graph.svg create mode 100644 users/flokli/presentations/2023-09-09-nixcon-tvix/tvixbolt.webm (limited to 'users/flokli') diff --git a/users/flokli/presentations/2023-09-09-nixcon-tvix/.gitignore b/users/flokli/presentations/2023-09-09-nixcon-tvix/.gitignore new file mode 100644 index 000000000000..397b4a7624e3 --- /dev/null +++ b/users/flokli/presentations/2023-09-09-nixcon-tvix/.gitignore @@ -0,0 +1 @@ +*.log diff --git a/users/flokli/presentations/2023-09-09-nixcon-tvix/architecture.dot b/users/flokli/presentations/2023-09-09-nixcon-tvix/architecture.dot new file mode 100644 index 000000000000..a6ea0460efe6 --- /dev/null +++ b/users/flokli/presentations/2023-09-09-nixcon-tvix/architecture.dot @@ -0,0 +1,5 @@ +digraph { + Builder + Store + Evaluator +} diff --git a/users/flokli/presentations/2023-09-09-nixcon-tvix/cppnix-example-lexer.cpp b/users/flokli/presentations/2023-09-09-nixcon-tvix/cppnix-example-lexer.cpp new file mode 100644 index 000000000000..7c52bce8b6d2 --- /dev/null +++ b/users/flokli/presentations/2023-09-09-nixcon-tvix/cppnix-example-lexer.cpp @@ -0,0 +1,13 @@ +attrpath + : attrpath '.' attr { + $$ = $1; $1->push_back(AttrName(data->symbols.create($3))); + } + | attrpath '.' string_attr + { $$ = $1; + ExprString * str = dynamic_cast($3); + if (str) { + $$->push_back(AttrName(data->symbols.create(str->s))); + delete str; + } else + $$->push_back(AttrName($3)); + } diff --git a/users/flokli/presentations/2023-09-09-nixcon-tvix/crate-deps.dot b/users/flokli/presentations/2023-09-09-nixcon-tvix/crate-deps.dot new file mode 100644 index 000000000000..66ead74b1e53 --- /dev/null +++ b/users/flokli/presentations/2023-09-09-nixcon-tvix/crate-deps.dot @@ -0,0 +1,19 @@ +digraph { + bgcolor="transparent" + node [fillcolor="lightgrey",style="filled"] + + tvix_cli + tvix_eval + nix_compat + tvix_serde + tvix_store + + tvix_cli -> tvix_store + tvix_cli -> nix_compat + tvix_cli -> tvix_eval + + tvix_store -> nix_compat + tvix_eval -> nix_compat + + tvix_serde -> tvix_eval +} diff --git a/users/flokli/presentations/2023-09-09-nixcon-tvix/default.nix b/users/flokli/presentations/2023-09-09-nixcon-tvix/default.nix new file mode 100644 index 000000000000..1ec0a0bd0ed7 --- /dev/null +++ b/users/flokli/presentations/2023-09-09-nixcon-tvix/default.nix @@ -0,0 +1,37 @@ +{ depot, pkgs, ... }: + +let + inherit (pkgs) + fontconfig qrencode runCommand stdenv; + mkQr = url: runCommand "qrcode.png" { } '' + ${qrencode}/bin/qrencode -o $out -t SVG -s 5 \ + --background=fafafa \ + --foreground=000000 \ + ${url} + ''; +in +stdenv.mkDerivation { + name = "2023-nixcon-tvix"; + src = ./.; + + FONTCONFIG_FILE = pkgs.makeFontsConf { + fontDirectories = with pkgs; [ jetbrains-mono fira fira-code fira-mono lato ]; + }; + + PUPPETEER_EXECUTABLE_PATH = "${pkgs.chromium}/bin/chromium"; + PUPPETEER_SKIP_CHROMIUM_DOWNLOAD = "1"; + + nativeBuildInputs = [ pkgs.reveal-md pkgs.graphviz ]; + + buildPhase = '' + cp ${depot.tvix.logo}/logo.png tvix-logo.png + dot -Tsvg crate-deps.dot > crate-deps.svg + cp ${mkQr "https://flokli.de"} qrcode-flokli.svg + cp ${mkQr "https://tvix.dev"} qrcode-tvix.svg + + mkdir -p $out + reveal-md --static $out presentation.md + reveal-md --print $out/slides.pdf presentation.md + cp tvixbolt.webm $out + ''; +} diff --git a/users/flokli/presentations/2023-09-09-nixcon-tvix/presentation.md b/users/flokli/presentations/2023-09-09-nixcon-tvix/presentation.md new file mode 100644 index 000000000000..0006763d6d82 --- /dev/null +++ b/users/flokli/presentations/2023-09-09-nixcon-tvix/presentation.md @@ -0,0 +1,294 @@ +--- +author: +- Florian Klink +date: 2023-09-09 +title: "Tvix: Status update" +theme: moon +revealOptions: + transition: 'fade' +--- + +# Tvix: Status update + +![Tvix Logo](tvix-logo.png) + +2023-09-09 + +Florian Klink + +--- + +## Whoami + +- flokli +- nixpkgs contributor since 2018, maintaining systemd, nsncd and some + more stuff +- Freelance Nix/DevOps consultant +- I spend too much time on computers :-) + +--- + +## What is Tvix? + +- A new implementation of Nix +- modular +- written in Rust +- developed in the [TVL](https://tvl.fyi) monorepo +- subtree exported to [github:tvlfyi/tvix](https://github.com/tvlfyi/tvix) + +--- + +## Structure + +- strong separation between **Evaluator**, **Store** and **Builder** +- Defined interfaces between different components (Protobuf/gRPC) + - Allows adding to/combining with your own components +- A lot of helper code for some of the Nix internals in the `nix-compat` crate + +Note: Derivation types, serializers. NAR writers, nixbase32 enc/dec, Nix Hash function, stringparsing etc. + +---- + +![crate-deps.svg](crate-deps.svg) + +--- + +## Evaluator: Design + +- + Nix code is parsed via [rnix](https://github.com/nix-community/rnix-parser) +- + AST traversal, generate bytecode (with some transformations) +- + Bytecode is executed by an abstract machine with a Nix-specific instruction set + +---- + +## Evaluator: Design + +- + Builtins are separate from the "evaluator core" + - + inject your own builtins + - + this includes `builtins.derivation`! +- + IO is nicely abstracted away + - + We can run a Nixlang subset without IO in wasm (see [tvixbolt](https://tvixbolt.tvl.su/)), + or parse Nix into a config struct with `tvix-serde` + +---- + + +Tvixbolt Demo + +---- + +### Evaluator: Current Work + +- + Current goal: **evaluate nixpkgs the same way as Nix does** +- + Checked by comparing the calculated output paths, which checks correctness of all \"parent\" output paths too. +- + Required implementing a lot of Nix internals in `nix-compat`, and `tvix-store` (A-Term, hash modulo, NAR writer/hasher) + +Note: Getting output hashing correct, and exposing this in a re-usable fashion took quite some iterations to get right. + +---- + +### Evaluator: Current Work (cont.) +- + 🎉 Already correct for (and continously checked by CI on every commit): + - + `stdenv`, `hello` + - + `pkgsCross.aarch64-multiplatform.stdenv`, `pkgsCross.aarch64-multiplatform.hello` +- + Some work left for more complicated expressions + - + infinite recursion [when inheriting from a `builtins.tryEval` multiple times](https://b.tvl.fyi/281) + - + small details around file imports +- + Not too much performance finetuning until we're correct first. + +---- + +### Evaluator: Demo + +[![asciicast](https://asciinema.org/a/MH4tuVPLsKewJSGJUYEyIKUpj.svg)](https://asciinema.org/a/MH4tuVPLsKewJSGJUYEyIKUpj) + +--- + +## Store: Design + +- + Uses a very different underlying data model: + - + Nix stores on a per- `StorePath` granularity + - + tvix-store uses a Merkle DAG of directories, similar to git trees, but with [BLAKE3](https://github.com/BLAKE3-team/BLAKE3) digests as pointers. + - + Compat layer in front to still render/calculate NAR on demand where needed + - + Substitution, caching, ... possible to describe via composition/layering + +---- + +![tvix-store graph](tvix-store-graph.svg) + +---- + +### Store: Advantages + +- + Less downloading of data that didn't change +- + Less duplicate data on disk/storage +- + Inherently content-addressed, so P2P substitution possible +- + Allows doing verified blob streaming ([BAO](https://github.com/oconnor663/bao), [bao-tree](https://github.com/n0-computer/bao-tree)) +- + Protocol has some \"smarter\" methods to avoid roundtrips, but could all be statically pre-rendered +- + Very little data that needs to be fetched from a trusted party (or be signed) + +Note: Our way of addressing blobs by their raw blake3 digest is natively compatible with iroh, the IPFS Re-implementation in Rust + +---- + +### Store: Status + +- + Whole Merkle-based store implementation (and Nix NAR compat layer) is there + - + exercised by the output path CI tests, and a test suite. + - + three backends (Sled, in-memory, gRPC client) + - + more backends and more test suites planned. +- + FUSE filesystem to expose the store (to Tvix Builders, \"appliances\") + +Note: backends: RocksDB, sqlite, s3. fuse: lazy fetching of build input files | think about a minimal initrd to bring up network and mount the store, then run any closure. + +---- + +### Store: Demo + +[![asciicast](https://asciinema.org/a/YFB9yycHdx0OUH9N0WdAkIYua.svg)](https://asciinema.org/a/YFB9yycHdx0OUH9N0WdAkIYua) + +---- + +### Store: Status (cont.) +- + More work on store composition needed (necessary for substition and caching) +- + More work on more granular blob substititon needed. +- + More work on bridges with Nix needed + - + Get Nix to talk to a tvix-store + - + Expose existing binary caches to tvix-store + +--- + +### Builder: Design + +- + Build requests/Build protocol is less Nix-specific + - + allows reusing builders for other usages (non-sandboxed builds, other build systems, playing with other addressing mechanisms) +- + Distinction between a **specific build attempt** and the **general build recipe** + - + allows keeping metadata about failed builds + - + stats (memory/cpu consumption) + - + comparing different produced binary outputs (r11y) + +---- + +### Builder: Design + +- + Invididual builds can be run in your desired container/virt engine/scheduler, as long as it speaks the same Build API +- + Build API composition/proxying, similar to Store composition +- + allows "unattended building" (evaluate nixpkgs locally and send all build requests to a remote builder) +- + allows tailing logs from currently/already running builds + +---- + +### Builder: Status + +- + Dummy Builder implementation in `go-nix` (using OCI) +- + Some scribble notes on the Build Protocol +- + Glue code to trigger builds from inside `builtins.derivation` needs to be written +- + Builder implementation (using `systemd-nspwan` or some container engine needs to be written. +- + Web interface to visualize store contents and build graphs/builds/logs + +--- + +## Contributing + +- + Join the IRC channel (`#tvl` on `hackint`), bridged to Matrix and XMPP +- + Check our issue tracker +- + Try to use it and tell us how you broke it! +- + Add various Nix bits to `nix-compat` + +Note: or if you like what you seeing and want to sponsor parts, that's also cool :-) + +--- + +# Thanks to... + +- + all TVL contributors (some drive-by, some long-term contributors) +- + countless Nix community members for their input on the architecture and rubberducking +- + NLNET and others to sponsor parts of this + +---- + +# Questions? + + + +
+ +
+Florian Klink / flokli.de
+ +
+ +
+Tvix / tvix.dev
+ +
+ +
diff --git a/users/flokli/presentations/2023-09-09-nixcon-tvix/tvix-store-graph.svg b/users/flokli/presentations/2023-09-09-nixcon-tvix/tvix-store-graph.svg new file mode 100644 index 000000000000..56338b587e94 --- /dev/null +++ b/users/flokli/presentations/2023-09-09-nixcon-tvix/tvix-store-graph.svg @@ -0,0 +1,17 @@ + + + + + + + + 0x01 0x02 0x030x04 0x05BlobsDirectoriesdirectories: []files: - name: .keep digest: <empty-blob-digest> size: 0 executable: falsesymlinks: []directories: - name: keep digest: <directory-with-keep-digest> size: 1 executable: falsefiles: - name: .keep digest: <empty-blob-digest> size: 0 executable: falsesymlinks: - name: aa target: /nix/store/somewhereelsedirectories: []files: []symlinks: []directories: - name: a digest: <directory-a-digest> size: 0files: []symlinks: []DIRECTORY_WITH_KEEPDIRECTORY_COMPLICATEDDIRECTORY_ADIRECTORY_BBLOB_ABLOB_BBLOB_EMPTYPathInfonode: symlink: name: 00000000000000000000000000000000-test target: /fooreferences: []narinfo: …node: directory: name: 11111111111111111111111111111111-test digest: <directory-complicated-digest> size: 4references: []narinfo: …node: directory: name: 22222222222222222222222222222222-test digest: <directory-with-keep-digest> size: 1references: []narinfo: …node: file: name: 33333333333333333333333333333333-test digest: <blob-a-digest> size: 3 executable: falsereferences: []narinfo: … \ No newline at end of file diff --git a/users/flokli/presentations/2023-09-09-nixcon-tvix/tvixbolt.webm b/users/flokli/presentations/2023-09-09-nixcon-tvix/tvixbolt.webm new file mode 100644 index 000000000000..69bd20f193cb Binary files /dev/null and b/users/flokli/presentations/2023-09-09-nixcon-tvix/tvixbolt.webm differ -- cgit 1.4.1