From 8b09ae54b1d635e77c394dd965915479283489a2 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Fri, 13 Oct 2023 16:01:06 +0200 Subject: feat(users/flokli): add ASG Lightning talk presentation This adds the materials for the lightning talk held at All Systems Go Conference 2023, Berlin. Talk lives at https://media.ccc.de/v/all-systems-go-2023-245-tvix-store Change-Id: I114b1aec9f1953c148dd29ca88888c16b9fc655d Reviewed-on: https://cl.tvl.fyi/c/depot/+/9751 Reviewed-by: flokli Tested-by: BuildkiteCI --- .../2023-09-13-asg-tvix-store/default.nix | 32 +++++ .../2023-09-13-asg-tvix-store/presentation.md | 138 +++++++++++++++++++++ .../tvix-store-graph-blob-directory.svg | 17 +++ 3 files changed, 187 insertions(+) create mode 100644 users/flokli/presentations/2023-09-13-asg-tvix-store/default.nix create mode 100644 users/flokli/presentations/2023-09-13-asg-tvix-store/presentation.md create mode 100644 users/flokli/presentations/2023-09-13-asg-tvix-store/tvix-store-graph-blob-directory.svg diff --git a/users/flokli/presentations/2023-09-13-asg-tvix-store/default.nix b/users/flokli/presentations/2023-09-13-asg-tvix-store/default.nix new file mode 100644 index 000000000000..840f21de8103 --- /dev/null +++ b/users/flokli/presentations/2023-09-13-asg-tvix-store/default.nix @@ -0,0 +1,32 @@ +{ depot, pkgs, ... }: + +let + inherit (pkgs) + fontconfig qrencode runCommand stdenv; + mkQr = url: runCommand "qrcode.png" { } '' + ${qrencode}/bin/qrencode -o $out -t SVG -s 5 \ + --background=fafafa \ + --foreground=000000 \ + ${url} + ''; +in +stdenv.mkDerivation { + name = "2023-asg-tvix-store"; + src = ./.; + + FONTCONFIG_FILE = pkgs.makeFontsConf { + fontDirectories = with pkgs; [ jetbrains-mono fira fira-code fira-mono lato ]; + }; + + nativeBuildInputs = [ pkgs.reveal-md pkgs.graphviz ]; + + buildPhase = '' + cp ${depot.tvix.logo}/logo.png tvix-logo.png + cp ${mkQr "https://flokli.de"} qrcode-flokli.svg + cp ${mkQr "https://tvix.dev"} qrcode-tvix.svg + + mkdir -p $out + cp tvix-store-graph-blob-directory.svg $out/ + reveal-md --static $out presentation.md + ''; +} diff --git a/users/flokli/presentations/2023-09-13-asg-tvix-store/presentation.md b/users/flokli/presentations/2023-09-13-asg-tvix-store/presentation.md new file mode 100644 index 000000000000..978934f9a48d --- /dev/null +++ b/users/flokli/presentations/2023-09-13-asg-tvix-store/presentation.md @@ -0,0 +1,138 @@ +--- +author: +- Florian Klink +date: 2023-09-09 +title: "tvix-store: A content-addressed file system and sync protocol" +theme: moon +revealOptions: + transition: 'fade' +--- + +## tvix-store +### A content-addressed file system and sync protocol + +2023-09-13 + +Florian Klink / flokli + +--- + +## Whoami + +- + flokli +- + Nix/NixOS contributor + - maintain systemd, nss and more low-level stuff there +- + Freelance Nix/DevOps consultant + +Note: more Kubernetes/DevOps exposure with work + +--- + +## What is tvix-store? +- + A new implementation of a content-addressed "storage system" + - + part of the Tvix Project, a (WIP) reimplementation of Nix and auxillary components in Rust + - + Storage model: think about git trees and its Merkle DAG… + - + … but with nicer wire format (`.proto`) and hash function (blake3) + +--- + +## Storage model +- + Once you know the root: everything else is content-addressed + - + No timestamps, no uid/gid, no xattrs, only one way to represent the same tree +- + Automatic dedup of identical subtrees in different file system trees +- + Automatic dedup of identical blobs (and you can do more chunking underneath too) + +--- + +## Storage model (cont.) +- + Granular seekable access into blobs +- + verified streaming thanks to BLAKE3 and Bao, faulty data is detected early on +- + Everything below can be retrieved from anyone without having to trust (P2P substitution, CDNs, …) + +--- + +## Usecases +- + File system tree delivery +- + Container image delivery +- + Backing store for VCS +- + Granular access into large datasets + +--- + +## Status +- + In-memory backend, a local K/V backend (Sled) +- + FUSE filesystem +- + A gRPC API to transfer things, bindings for golang and rust +- + some object storage backends in development (GCS, NATS) +- + FUTUREWORK: more storage backends / store composition / in-kernel module? + +Notes: of course you can use your own network protocol too, like HTTP CAS or iroh....plug different stores together to represent caches, blobfs + +--- + +## Contributing + +- + Join the IRC channel (`#tvl` on `hackint`), bridged to Matrix and XMPP +- + Check our issue tracker +- + Try to use it and tell us how you broke it! + +Note: if this sounds useful to you, reach out! + +--- + +# Thanks! + + + +
+ +
+Florian Klink / flokli.de
+ +
+ +
+Tvix / tvix.dev
+ +
+ +
+ +--- + +## Structure + +[tvix-store graph](tvix-store-graph-blob-directory.svg) diff --git a/users/flokli/presentations/2023-09-13-asg-tvix-store/tvix-store-graph-blob-directory.svg b/users/flokli/presentations/2023-09-13-asg-tvix-store/tvix-store-graph-blob-directory.svg new file mode 100644 index 000000000000..2c87350d5b79 --- /dev/null +++ b/users/flokli/presentations/2023-09-13-asg-tvix-store/tvix-store-graph-blob-directory.svg @@ -0,0 +1,17 @@ + + + + + + + + 0x01 0x02 0x030x04 0x05BlobsDirectoriesdirectories: []files: - name: .keep digest: <empty-blob-digest> size: 0 executable: falsesymlinks: []directories: - name: keep digest: <directory-with-keep-digest> size: 1 executable: falsefiles: - name: .keep digest: <empty-blob-digest> size: 0 executable: falsesymlinks: - name: aa target: /nix/store/somewhereelsedirectories: []files: []symlinks: []directories: - name: a digest: <directory-a-digest> size: 0files: []symlinks: []DIRECTORY_WITH_KEEPDIRECTORY_COMPLICATEDDIRECTORY_ADIRECTORY_BBLOB_ABLOB_BBLOB_EMPTY \ No newline at end of file -- cgit 1.4.1