feat(tvix/tests): add basic plumbing for integration tests r/6645

This adds a basic skeleton to test various tvix components in
combination.

As a first start, we test the virtiofs integration, by seeding the tvix-
store with some data, booting a VM, and listing the contents from there.

Change-Id: Ice6516d0eb508d19ea8fabefee860a0950046171
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9449
Autosubmit: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
Reviewed-by: Connor Brewster <cbrewster@hey.com>

3 files changed, 337 insertions, 0 deletions

diff --git a/tvix/tests/README.md b/tvix/tests/README.md
new file mode 100644
index 000000000000..b079a85dbddd
--- /dev/null
+++ b/tvix/tests/README.md
@@ -0,0 +1,115 @@
+# tvix/tests
+
+This directory provides a bunch of integration tests using tvix.
+
+The currently most interesting (and only) ones ;-) are using a cloud-hypervisor
+VM.
+
+## //tvix/tests:test-docs
+This is a test encapsulated in a nix build.
+It seeds a tvix-store with the tvix docs, then starts a VM, asks it to list all
+files in /nix/store, and ensures the store path is present, which acts as a
+nice smoketest.
+
+## //tvix/tests:runVM
+A script spinning up a `tvix-store virtiofs` daemon, then starting a cloud-
+hypervisor VM.
+
+The cloud-hypervisor VM is using a (semi-)minimal kernel image with virtiofs
+support, and a custom initrd (using u-root). It supports various command line
+options, to be able to do VM tests, act as an interactive shell or exec a binary
+from a closure.
+
+It supports the following env vars:
+ - `CH_NUM_CPUS=1` controls the number of CPUs available to the VM
+ - `CH_MEM_SIZE=512M` controls the memory availabe to the VM
+ - `CH_CMDLINE=` controls the kernel cmdline (which can be used to control the
+   boot)
+
+### Usage
+First, ensure you have `tvix-store` in `$PATH`, as that's what `run-tvix-vm`
+expects:
+
+Assuming you ran `cargo build --profile=release-with-debug` before, and are in
+the `tvix` directory:
+
+```
+export PATH=$PATH:$PWD/target/release-with-debug
+```
+
+Secondly, configure tvix to use the local backend:
+
+```
+export BLOB_SERVICE_ADDR=sled://$PWD/blobs.sled
+export DIRECTORY_SERVICE_ADDR=sled://$PWD/directories.sled
+export PATH_INFO_SERVICE_ADDR=sled://$PWD/pathinfo.sled
+```
+
+Potentially copy some data into tvix-store (via nar-bridge):
+
+```
+mg run //tvix:store -- daemon &
+mg run //tvix:nar-bridge -- &
+rm -Rf ~/.cache/nix; nix copy --to http://localhost:9000\?compression\=none $(mg build //third_party/nixpkgs:hello)
+pkill nar-bridge; pkill tvix-store
+```
+
+#### Interactive shell
+Run the VM like this:
+
+```
+CH_CMDLINE=tvix.shell mg run //tvix/tests:runVM --
+```
+
+You'll get dropped into a shell, from which you can list the store contents:
+
+```
+[    0.282381] Run /init as init process
+2023/09/24 13:03:38 Welcome to u-root!
+                              _
+   _   _      _ __ ___   ___ | |_
+  | | | |____| '__/ _ \ / _ \| __|
+  | |_| |____| | | (_) | (_) | |_
+   \__,_|    |_|  \___/ \___/ \__|
+
+2023/09/24 13:03:38 Running tvix-init…
+2023/09/24 13:03:38 Creating /nix/store
+2023/09/24 13:03:38 Mounting…
+2023/09/24 13:03:38 Invoking shell
+…
+/# ls -la /nix/store/
+dr-xr-xr-x root 0 0   Jan  1 00:00 .
+dr-xr-xr-x root 0 989 Jan  1 00:00 aw2fw9ag10wr9pf0qk4nk5sxi0q0bn56-glibc-2.37-8
+dr-xr-xr-x root 0 3   Jan  1 00:00 jbwb8d8l28lg9z0xzl784wyb9vlbwss6-xgcc-12.3.0-libgcc
+dr-xr-xr-x root 0 82  Jan  1 00:00 k8ivghpggjrq1n49xp8sj116i4sh8lia-libidn2-2.3.4
+dr-xr-xr-x root 0 141 Jan  1 00:00 mdi7lvrn2mx7rfzv3fdq3v5yw8swiks6-hello-2.12.1
+dr-xr-xr-x root 0 5   Jan  1 00:00 s2gi8pfjszy6rq3ydx0z1vwbbskw994i-libunistring-1.1
+```
+
+Once you're done, run `poweroff` to turn off the VM.
+
+#### Execute a specific binary
+Run the VM like this:
+
+```
+hello_cmd=$(mg build //third_party/nixpkgs:hello)/bin/hello
+CH_CMDLINE=tvix.exec=$hello_cmd mg run //tvix/tests:runVM --
+```
+
+Observe it executing the file (and closure) from the tvix-store:
+
+```
+2023/09/24 13:06:13 Welcome to u-root!
+                              _
+   _   _      _ __ ___   ___ | |_
+  | | | |____| '__/ _ \ / _ \| __|
+  | |_| |____| | | (_) | (_) | |_
+   \__,_|    |_|  \___/ \___/ \__|
+
+2023/09/24 13:06:13 Running tvix-init…
+2023/09/24 13:06:13 Creating /nix/store
+2023/09/24 13:06:13 Mounting…
+2023/09/24 13:06:13 Invoking /nix/store/mdi7lvrn2mx7rfzv3fdq3v5yw8swiks6-hello-2.12.1/bin/hello
+…
+Hello, world!
+```
\ No newline at end of file
diff --git a/tvix/tests/default.nix b/tvix/tests/default.nix
new file mode 100644
index 000000000000..9e9a22a66221
--- /dev/null
+++ b/tvix/tests/default.nix
@@ -0,0 +1,137 @@
+{ depot, pkgs, ... }:
+
+rec {
+  # A binary that sets up /nix/store from virtiofs, lists all store paths, and
+  # powers off the machine.
+  tvix-init = depot.nix.buildGo.program {
+    name = "tvix-init";
+    srcs = [
+      ./tvix-init.go
+    ];
+  };
+
+  # A kernel with virtiofs support baked in
+  kernel = pkgs.buildLinux ({ } // {
+    inherit (pkgs.linuxPackages_latest.kernel) src version modDirVersion;
+    autoModules = false;
+    kernelPreferBuiltin = true;
+    ignoreConfigErrors = true;
+    kernelPatches = [ ];
+    structuredExtraConfig = with pkgs.lib.kernel; {
+      FUSE_FS = option yes;
+      DAX_DRIVER = option yes;
+      DAX = option yes;
+      FS_DAX = option yes;
+      VIRTIO_FS = option yes;
+      VIRTIO = option yes;
+      ZONE_DEVICE = option yes;
+    };
+  });
+
+  # A build framework for minimal initrds
+  uroot = pkgs.buildGoModule {
+    pname = "u-root";
+    version = "unstable-2023-09-20";
+    src = pkgs.fetchFromGitHub {
+      owner = "u-root";
+      repo = "u-root";
+      rev = "72921548ce2e88c4c5b62e83c717cbd834b58067";
+      hash = "sha256-fEoUGqh6ZXprtSpJ55MeuSFe7L5A/rkIIVLCwxbPHzE=";
+    };
+    vendorHash = null;
+
+    doCheck = false; # Some tests invoke /bin/bash
+  };
+
+  # Use u-root to build a initrd with our tvix-init inside.
+  initrd = pkgs.stdenv.mkDerivation {
+    name = "initrd.cpio";
+    nativeBuildInputs = [ pkgs.go ];
+    # https://github.com/u-root/u-root/issues/2466
+    buildCommand = ''
+      mkdir -p /tmp/go/src/github.com/u-root/
+      cp -R ${uroot.src} /tmp/go/src/github.com/u-root/u-root
+      cd /tmp/go/src/github.com/u-root/u-root
+      chmod +w .
+      cp ${tvix-init}/bin/tvix-init tvix-init
+
+      export HOME=$(mktemp -d)
+      export GOROOT="$(go env GOROOT)"
+
+      GO111MODULE=off GOPATH=/tmp/go GOPROXY=off ${uroot}/bin/u-root -files ./tvix-init -uinitcmd "/tvix-init" -o $out
+    '';
+  };
+
+  # Start a `tvix-store` virtiofs daemon from $PATH, then a cloud-hypervisor
+  # pointed to it.
+  # Supports the following env vars (and defaults)
+  # CH_NUM_CPUS=1
+  # CH_MEM_SIZE=512M
+  # CH_CMDLINE=""
+  runVM = pkgs.writers.writeBashBin "run-tvix-vm" ''
+    tempdir=$(mktemp -d)
+
+    cleanup() {
+      kill $virtiofsd_pid
+      if [[ -n ''${work_dir-} ]]; then
+        chmod -R u+rw "$tempdir"
+        rm -rf "$tempdir"
+      fi
+    }
+    trap cleanup EXIT
+
+    # Spin up the virtiofs daemon
+    tvix-store virtiofs -l $tempdir/tvix.sock &
+    virtiofsd_pid=$!
+
+    # Wait for the socket to exist.
+    until [ -e $tempdir/tvix.sock ]; do sleep 0.1; done
+
+    CH_NUM_CPUS="''${CH_NUM_CPUS:-1}"
+    CH_MEM_SIZE="''${CH_MEM_SIZE:-512M}"
+    CH_CMDLINE="''${CH_CMDLINE:-}"
+
+    # spin up cloud_hypervisor
+    ${pkgs.cloud-hypervisor}/bin/cloud-hypervisor \
+     --cpus boot=$CH_NUM_CPU \
+     --memory mergeable=on,shared=on,size=$CH_MEM_SIZE \
+     --console null \
+     --serial tty \
+     --kernel ${kernel.dev}/vmlinux \
+     --initramfs ${initrd} \
+     --cmdline "console=ttyS0 $CH_CMDLINE" \
+     --fs tag=tvix,socket=$tempdir/tvix.sock,num_queues=1,queue_size=512
+  '';
+
+  # Seed a tvix-store with the tvix docs, then start a VM and search for the
+  # store path in the output.
+  test-docs = pkgs.stdenv.mkDerivation {
+    name = "run-vm";
+    nativeBuildInputs = [
+      depot.tvix.store
+    ];
+    buildCommand = ''
+      touch $out
+
+      # Configure tvix to put data in the local working directory
+      export BLOB_SERVICE_ADDR=sled://$PWD/blobs.sled
+      export DIRECTORY_SERVICE_ADDR=sled://$PWD/directories.sled
+      export PATH_INFO_SERVICE_ADDR=sled://$PWD/pathinfo.sled
+
+      # Seed the tvix store with some data
+      # Create a `docs` directory with the contents from ../docs
+      # Make sure it still is called "docs" when calling import, so we can
+      # predict the store path.
+      cp -R ${../docs} docs
+      outpath=$(tvix-store import docs)
+
+      echo "Store contents imported to $outpath"
+
+      CH_CMDLINE="tvix.find" ${runVM}/bin/run-tvix-vm 2>&1 | tee output.txt
+      grep ${../docs} output.txt
+    '';
+    requiredSystemFeatures = [ "kvm" ];
+  };
+
+  meta.ci.targets = [ "test-docs" ];
+}
diff --git a/tvix/tests/tvix-init.go b/tvix/tests/tvix-init.go
new file mode 100644
index 000000000000..a81001159ae2
--- /dev/null
+++ b/tvix/tests/tvix-init.go
@@ -0,0 +1,85 @@
+package main
+
+import (
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+	"syscall"
+)
+
+// run the given command, connecting std{in,err,out} with the OS one.
+func run(args ...string) error {
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Stdin = os.Stdin
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stdout
+
+	return cmd.Run()
+}
+
+// parse the cmdline, return a map[string]string.
+func parseCmdline(cmdline string) map[string]string {
+	line := strings.TrimSuffix(cmdline, "\n")
+	fields := strings.Fields(line)
+	out := make(map[string]string, 0)
+
+	for _, arg := range fields {
+		kv := strings.SplitN(arg, "=", 2)
+		switch len(kv) {
+		case 1:
+			out[kv[0]] = ""
+		case 2:
+			out[kv[0]] = kv[1]
+		}
+	}
+
+	return out
+}
+
+func main() {
+	log.Println("Running tvix-init…")
+
+	log.Println("Creating /nix/store")
+	os.MkdirAll("/nix/store", os.ModePerm)
+
+	cmdline, err := os.ReadFile("/proc/cmdline")
+	if err != nil {
+		log.Printf("Failed to read cmdline: %s\n", err)
+	}
+	cmdlineFields := parseCmdline(string(cmdline))
+
+	log.Println("Mounting…")
+	if err := run("mount", "-t", "virtiofs", "tvix", "/nix/store", "-o", "ro"); err != nil {
+		log.Printf("Failed to run mount: %v\n", err)
+	}
+
+	// If tvix.find is set, invoke find /nix/store
+	if _, ok := cmdlineFields["tvix.find"]; ok {
+		log.Println("Listing…")
+		if err := run("find", "/nix/store"); err != nil {
+			log.Printf("Failed to run find command: %s\n", err)
+		}
+	}
+
+	// If tvix.shell is set, invoke the elvish shell
+	if v, ok := cmdlineFields["tvix.shell"]; ok {
+		log.Printf("Invoking shell%s\n…", v)
+		if err := run("elvish"); err != nil {
+			log.Printf("Failed to run shell: %s\n", err)
+		}
+	}
+
+	// If tvix.exec is set, invoke the binary specified
+	if v, ok := cmdlineFields["tvix.exec"]; ok {
+		log.Printf("Invoking %s\n…", v)
+		if err := syscall.Exec(v, []string{v}, []string{}); err != nil {
+			log.Printf("Failed to exec: %s\n", err)
+		}
+	}
+
+	log.Println("Powering off")
+	if err := run("poweroff"); err != nil {
+		log.Printf("Failed to run command: %v\n", err)
+	}
+}