From 1b3d6975ed8c2a19f1e7f4f39a564cdfa387f0b6 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Mon, 25 Sep 2023 13:59:25 +0300 Subject: chore(tvix/tests): rename to //tvix/boot This is mostly boot tooling, the integration test is just one instance making use of it. Expose initrd, kernel and runVM as a separate target to CI, and move the tests to a subdirectory. Change-Id: I1d22cd68bf5af095bc11dd9d7117b62956c7f7f2 Reviewed-on: https://cl.tvl.fyi/c/depot/+/9465 Reviewed-by: raitobezarius Autosubmit: flokli Tested-by: BuildkiteCI --- tvix/boot/README.md | 136 ++++++++++++++++++++++++++++++++++++++++++ tvix/boot/default.nix | 111 ++++++++++++++++++++++++++++++++++ tvix/boot/tests/default.nix | 35 +++++++++++ tvix/boot/tvix-init.go | 138 ++++++++++++++++++++++++++++++++++++++++++ tvix/tests/README.md | 142 -------------------------------------------- tvix/tests/default.nix | 137 ------------------------------------------ tvix/tests/tvix-init.go | 138 ------------------------------------------ 7 files changed, 420 insertions(+), 417 deletions(-) create mode 100644 tvix/boot/README.md create mode 100644 tvix/boot/default.nix create mode 100644 tvix/boot/tests/default.nix create mode 100644 tvix/boot/tvix-init.go delete mode 100644 tvix/tests/README.md delete mode 100644 tvix/tests/default.nix delete mode 100644 tvix/tests/tvix-init.go diff --git a/tvix/boot/README.md b/tvix/boot/README.md new file mode 100644 index 000000000000..79fca31ebc71 --- /dev/null +++ b/tvix/boot/README.md @@ -0,0 +1,136 @@ +# tvix/boot + +This directory provides tooling to boot VMs with /nix/store provided by +virtiofs. + +In the `tests/` subdirectory, there's some integration tests. + +## //tvix/tests:runVM +A script spinning up a `tvix-store virtiofs` daemon, then starting a cloud- +hypervisor VM. + +The cloud-hypervisor VM is using a (semi-)minimal kernel image with virtiofs +support, and a custom initrd (using u-root). It supports various command line +options, to be able to do VM tests, act as an interactive shell or exec a binary +from a closure. + +It supports the following env vars: + - `CH_NUM_CPUS=1` controls the number of CPUs available to the VM + - `CH_MEM_SIZE=512M` controls the memory availabe to the VM + - `CH_CMDLINE=` controls the kernel cmdline (which can be used to control the + boot) + +### Usage +First, ensure you have `tvix-store` in `$PATH`, as that's what `run-tvix-vm` +expects: + +Assuming you ran `cargo build --profile=release-with-debug` before, and are in +the `tvix` directory: + +``` +export PATH=$PATH:$PWD/target/release-with-debug +``` + +Secondly, configure tvix to use the local backend: + +``` +export BLOB_SERVICE_ADDR=sled://$PWD/blobs.sled +export DIRECTORY_SERVICE_ADDR=sled://$PWD/directories.sled +export PATH_INFO_SERVICE_ADDR=sled://$PWD/pathinfo.sled +``` + +Potentially copy some data into tvix-store (via nar-bridge): + +``` +mg run //tvix:store -- daemon & +mg run //tvix:nar-bridge -- & +rm -Rf ~/.cache/nix; nix copy --to http://localhost:9000\?compression\=none $(mg build //third_party/nixpkgs:hello) +pkill nar-bridge; pkill tvix-store +``` + +#### Interactive shell +Run the VM like this: + +``` +CH_CMDLINE=tvix.shell mg run //tvix/tests:runVM -- +``` + +You'll get dropped into an interactive shell, from which you can do things with +the store: + +``` + ______ _ ____ _ __ + /_ __/ __(_) __ / _/___ (_) /_ + / / | | / / / |/_/ / // __ \/ / __/ + / / | |/ / /> < _/ // / / / / /_ +/_/ |___/_/_/|_| /___/_/ /_/_/\__/ + +/# ls -la /nix/store/ +dr-xr-xr-x root 0 0 Jan 1 00:00 . +dr-xr-xr-x root 0 989 Jan 1 00:00 aw2fw9ag10wr9pf0qk4nk5sxi0q0bn56-glibc-2.37-8 +dr-xr-xr-x root 0 3 Jan 1 00:00 jbwb8d8l28lg9z0xzl784wyb9vlbwss6-xgcc-12.3.0-libgcc +dr-xr-xr-x root 0 82 Jan 1 00:00 k8ivghpggjrq1n49xp8sj116i4sh8lia-libidn2-2.3.4 +dr-xr-xr-x root 0 141 Jan 1 00:00 mdi7lvrn2mx7rfzv3fdq3v5yw8swiks6-hello-2.12.1 +dr-xr-xr-x root 0 5 Jan 1 00:00 s2gi8pfjszy6rq3ydx0z1vwbbskw994i-libunistring-1.1 +``` + +Once you exit the shell, the VM will power off itself. + +#### Execute a specific binary +Run the VM like this: + +``` +hello_cmd=$(mg build //third_party/nixpkgs:hello)/bin/hello +CH_CMDLINE=tvix.run=$hello_cmd mg run //tvix/tests:runVM -- +``` + +Observe it executing the file (and closure) from the tvix-store: + +``` +[ 0.277486] Run /init as init process + ______ _ ____ _ __ + /_ __/ __(_) __ / _/___ (_) /_ + / / | | / / / |/_/ / // __ \/ / __/ + / / | |/ / /> < _/ // / / / / /_ +/_/ |___/_/_/|_| /___/_/ /_/_/\__/ + +Hello, world! +2023/09/24 21:10:19 Nothing left to be done, powering off. +[ 0.299122] ACPI: PM: Preparing to enter system sleep state S5 +[ 0.299422] reboot: Power down +``` + +#### Execute a NixOS system closure +It's also possible to invoke a system closure. To do this, tvix-init honors the +init= cmdline option, and will switch_root to it. + + +``` +CH_CMDLINE=init=/nix/store/…-nixos-system-…/init mg run //tvix/tests:runVM -- +``` + +``` + ______ _ ____ _ __ + /_ __/ __(_) __ / _/___ (_) /_ + / / | | / / / |/_/ / // __ \/ / __/ + / / | |/ / /> < _/ // / / / / /_ +/_/ |___/_/_/|_| /___/_/ /_/_/\__/ + +2023/09/24 21:16:43 switch_root: moving mounts +2023/09/24 21:16:43 switch_root: Skipping "/run" as the dir does not exist +2023/09/24 21:16:43 switch_root: Changing directory +2023/09/24 21:16:43 switch_root: Moving / +2023/09/24 21:16:43 switch_root: Changing root! +2023/09/24 21:16:43 switch_root: Deleting old / +2023/09/24 21:16:43 switch_root: executing init + +<<< NixOS Stage 2 >>> + +[ 0.322096] booting system configuration /nix/store/g657sdxinpqfcdv0162zmb8vv9b5c4c5-nixos-system-client-23.11.git.82102fc37da +running activation script... +setting up /etc... +starting systemd... +[ 0.980740] systemd[1]: systemd 253.6 running in system mode (+PAM +AUDIT -SELINUX +APPARMOR +IMA +SMACK +SECCOMP +GCRYPT -GNUTLS +OPENSSL +ACL +BLKID +CURL +ELFUTILS +FIDO2 +IDN2 -IDN +IPTC +KMOD +LIBCRYPTSETUP +LIBFDISK +PCRE2 -PWQUALITY +P11KIT -QRENCODE +TPM2 +BZIP2 +LZ4 +XZ +ZLIB +ZSTD +BPF_FRAMEWORK -XKBCOMMON +UTMP -SYSVINIT default-hierarchy=unified) +``` + +This effectively replaces the NixOS Stage 1 entirely. \ No newline at end of file diff --git a/tvix/boot/default.nix b/tvix/boot/default.nix new file mode 100644 index 000000000000..8c20c35f6b80 --- /dev/null +++ b/tvix/boot/default.nix @@ -0,0 +1,111 @@ +{ depot, pkgs, ... }: + +rec { + # A binary that sets up /nix/store from virtiofs, lists all store paths, and + # powers off the machine. + tvix-init = depot.nix.buildGo.program { + name = "tvix-init"; + srcs = [ + ./tvix-init.go + ]; + }; + + # A kernel with virtiofs support baked in + kernel = pkgs.buildLinux ({ } // { + inherit (pkgs.linuxPackages_latest.kernel) src version modDirVersion; + autoModules = false; + kernelPreferBuiltin = true; + ignoreConfigErrors = true; + kernelPatches = [ ]; + structuredExtraConfig = with pkgs.lib.kernel; { + FUSE_FS = option yes; + DAX_DRIVER = option yes; + DAX = option yes; + FS_DAX = option yes; + VIRTIO_FS = option yes; + VIRTIO = option yes; + ZONE_DEVICE = option yes; + }; + }); + + # A build framework for minimal initrds + uroot = pkgs.buildGoModule { + pname = "u-root"; + version = "unstable-2023-09-20"; + src = pkgs.fetchFromGitHub { + owner = "u-root"; + repo = "u-root"; + rev = "72921548ce2e88c4c5b62e83c717cbd834b58067"; + hash = "sha256-fEoUGqh6ZXprtSpJ55MeuSFe7L5A/rkIIVLCwxbPHzE="; + }; + vendorHash = null; + + doCheck = false; # Some tests invoke /bin/bash + }; + + # Use u-root to build a initrd with our tvix-init inside. + initrd = pkgs.stdenv.mkDerivation { + name = "initrd.cpio"; + nativeBuildInputs = [ pkgs.go ]; + # https://github.com/u-root/u-root/issues/2466 + buildCommand = '' + mkdir -p /tmp/go/src/github.com/u-root/ + cp -R ${uroot.src} /tmp/go/src/github.com/u-root/u-root + cd /tmp/go/src/github.com/u-root/u-root + chmod +w . + cp ${tvix-init}/bin/tvix-init tvix-init + + export HOME=$(mktemp -d) + export GOROOT="$(go env GOROOT)" + + GO111MODULE=off GOPATH=/tmp/go GOPROXY=off ${uroot}/bin/u-root -files ./tvix-init -initcmd "/tvix-init" -o $out + ''; + }; + + # Start a `tvix-store` virtiofs daemon from $PATH, then a cloud-hypervisor + # pointed to it. + # Supports the following env vars (and defaults) + # CH_NUM_CPUS=1 + # CH_MEM_SIZE=512M + # CH_CMDLINE="" + runVM = pkgs.writers.writeBashBin "run-tvix-vm" '' + tempdir=$(mktemp -d) + + cleanup() { + kill $virtiofsd_pid + if [[ -n ''${work_dir-} ]]; then + chmod -R u+rw "$tempdir" + rm -rf "$tempdir" + fi + } + trap cleanup EXIT + + # Spin up the virtiofs daemon + tvix-store virtiofs -l $tempdir/tvix.sock & + virtiofsd_pid=$! + + # Wait for the socket to exist. + until [ -e $tempdir/tvix.sock ]; do sleep 0.1; done + + CH_NUM_CPUS="''${CH_NUM_CPUS:-1}" + CH_MEM_SIZE="''${CH_MEM_SIZE:-512M}" + CH_CMDLINE="''${CH_CMDLINE:-}" + + # spin up cloud_hypervisor + ${pkgs.cloud-hypervisor}/bin/cloud-hypervisor \ + --cpus boot=$CH_NUM_CPU \ + --memory mergeable=on,shared=on,size=$CH_MEM_SIZE \ + --console null \ + --serial tty \ + --kernel ${kernel.dev}/vmlinux \ + --initramfs ${initrd} \ + --cmdline "console=ttyS0 $CH_CMDLINE" \ + --fs tag=tvix,socket=$tempdir/tvix.sock,num_queues=1,queue_size=512 + ''; + + meta.ci.targets = [ + "initrd" + "kernel" + "runVM" + ]; +} diff --git a/tvix/boot/tests/default.nix b/tvix/boot/tests/default.nix new file mode 100644 index 000000000000..d725a7ac9bb7 --- /dev/null +++ b/tvix/boot/tests/default.nix @@ -0,0 +1,35 @@ +{ depot, pkgs, ... }: + +depot.nix.readTree.drvTargets { + # Seed a tvix-store with the tvix docs, then start a VM, ask it to list all + # files in /nix/store, and ensure the store path is present, which acts as a + # nice smoketest. + docs = pkgs.stdenv.mkDerivation { + name = "run-vm"; + nativeBuildInputs = [ + depot.tvix.store + depot.tvix.boot.runVM + ]; + buildCommand = '' + touch $out + + # Configure tvix to put data in the local working directory + export BLOB_SERVICE_ADDR=sled://$PWD/blobs.sled + export DIRECTORY_SERVICE_ADDR=sled://$PWD/directories.sled + export PATH_INFO_SERVICE_ADDR=sled://$PWD/pathinfo.sled + + # Seed the tvix store with some data + # Create a `docs` directory with the contents from ../docs + # Make sure it still is called "docs" when calling import, so we can + # predict the store path. + cp -R ${../../docs} docs + outpath=$(tvix-store import docs) + + echo "Store contents imported to $outpath" + + CH_CMDLINE="tvix.find" run-tvix-vm 2>&1 | tee output.txt + grep ${../../docs} output.txt + ''; + requiredSystemFeatures = [ "kvm" ]; + }; +} diff --git a/tvix/boot/tvix-init.go b/tvix/boot/tvix-init.go new file mode 100644 index 000000000000..97a24bab3547 --- /dev/null +++ b/tvix/boot/tvix-init.go @@ -0,0 +1,138 @@ +package main + +import ( + "fmt" + "log" + "os" + "os/exec" + "strings" + "syscall" +) + +// run the given command, connecting std{in,err,out} with the OS one. +func run(args ...string) error { + cmd := exec.Command(args[0], args[1:]...) + cmd.Stdin = os.Stdin + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + + return cmd.Run() +} + +// parse the cmdline, return a map[string]string. +func parseCmdline(cmdline string) map[string]string { + line := strings.TrimSuffix(cmdline, "\n") + fields := strings.Fields(line) + out := make(map[string]string, 0) + + for _, arg := range fields { + kv := strings.SplitN(arg, "=", 2) + switch len(kv) { + case 1: + out[kv[0]] = "" + case 2: + out[kv[0]] = kv[1] + } + } + + return out +} + +// mounts the nix store from the virtiofs tag to the given destination, +// creating the destination if it doesn't exist already. +func mountTvixStore(dest string) error { + if err := os.MkdirAll(dest, os.ModePerm); err != nil { + return fmt.Errorf("unable to mkdir dest: %w", err) + } + if err := run("mount", "-t", "virtiofs", "tvix", dest, "-o", "ro"); err != nil { + return fmt.Errorf("unable to run mount: %w", err) + } + + return nil +} + +func main() { + fmt.Print(` + ______ _ ____ _ __ + /_ __/ __(_) __ / _/___ (_) /_ + / / | | / / / |/_/ / // __ \/ / __/ + / / | |/ / /> < _/ // / / / / /_ +/_/ |___/_/_/|_| /___/_/ /_/_/\__/ + +`) + + // Set PATH to "/bbin", so we can find the u-root tools + os.Setenv("PATH", "/bbin") + + if err := run("mount", "-t", "proc", "none", "/proc"); err != nil { + log.Printf("Failed to mount /proc: %v\n", err) + } + if err := run("mount", "-t", "sysfs", "none", "/sys"); err != nil { + log.Printf("Failed to mount /sys: %v\n", err) + } + if err := run("mount", "-t", "devtmpfs", "devtmpfs", "/dev"); err != nil { + log.Printf("Failed to mount /dev: %v\n", err) + } + + cmdline, err := os.ReadFile("/proc/cmdline") + if err != nil { + log.Printf("Failed to read cmdline: %s\n", err) + } + cmdlineFields := parseCmdline(string(cmdline)) + + if _, ok := cmdlineFields["tvix.find"]; ok { + // If tvix.find is set, invoke find /nix/store + if err := mountTvixStore("/nix/store"); err != nil { + log.Printf("Failed to mount tvix store: %v\n", err) + } + + if err := run("find", "/nix/store"); err != nil { + log.Printf("Failed to run find command: %s\n", err) + } + } else if _, ok := cmdlineFields["tvix.shell"]; ok { + // If tvix.shell is set, mount the nix store to /nix/store directly, + // then invoke the elvish shell + if err := mountTvixStore("/nix/store"); err != nil { + log.Printf("Failed to mount tvix store: %v\n", err) + } + + if err := run("elvish"); err != nil { + log.Printf("Failed to run shell: %s\n", err) + } + } else if v, ok := cmdlineFields["tvix.run"]; ok { + // If tvix.run is set, mount the nix store to /nix/store directly, + // then invoke the command. + if err := mountTvixStore("/nix/store"); err != nil { + log.Printf("Failed to mount tvix store: %v\n", err) + } + + if err := run(v); err != nil { + log.Printf("Failed to run command: %s\n", err) + } + } else if v, ok := cmdlineFields["init"]; ok { + // If init is set, invoke the binary specified (with switch_root), + // and prepare /fs beforehand as well. + os.Mkdir("/fs", os.ModePerm) + if err := run("mount", "-t", "tmpfs", "none", "/fs"); err != nil { + log.Fatalf("Failed to mount /fs tmpfs: %s\n", err) + } + + // Mount /fs/nix/store + if err := mountTvixStore("/fs/nix/store"); err != nil { + log.Fatalf("Failed to mount tvix store: %v\n", err) + } + + // Invoke switch_root, which will take care of moving /proc, /sys and /dev. + if err := syscall.Exec("/bbin/switch_root", []string{"switch_root", "/fs", v}, []string{}); err != nil { + log.Printf("Failed to switch root: %s\n", err) + } + } else { + log.Printf("No command detected, not knowing what to do!") + } + + // This is only reached in the non switch_root case. + log.Printf("Nothing left to be done, powering off.") + if err := run("poweroff"); err != nil { + log.Printf("Failed to run poweroff command: %v\n", err) + } +} diff --git a/tvix/tests/README.md b/tvix/tests/README.md deleted file mode 100644 index 8cb4e9a6595c..000000000000 --- a/tvix/tests/README.md +++ /dev/null @@ -1,142 +0,0 @@ -# tvix/tests - -This directory provides a bunch of integration tests using tvix. - -The currently most interesting (and only) ones ;-) are using a cloud-hypervisor -VM. - -## //tvix/tests:test-docs -This is a test encapsulated in a nix build. -It seeds a tvix-store with the tvix docs, then starts a VM, asks it to list all -files in /nix/store, and ensures the store path is present, which acts as a -nice smoketest. - -## //tvix/tests:runVM -A script spinning up a `tvix-store virtiofs` daemon, then starting a cloud- -hypervisor VM. - -The cloud-hypervisor VM is using a (semi-)minimal kernel image with virtiofs -support, and a custom initrd (using u-root). It supports various command line -options, to be able to do VM tests, act as an interactive shell or exec a binary -from a closure. - -It supports the following env vars: - - `CH_NUM_CPUS=1` controls the number of CPUs available to the VM - - `CH_MEM_SIZE=512M` controls the memory availabe to the VM - - `CH_CMDLINE=` controls the kernel cmdline (which can be used to control the - boot) - -### Usage -First, ensure you have `tvix-store` in `$PATH`, as that's what `run-tvix-vm` -expects: - -Assuming you ran `cargo build --profile=release-with-debug` before, and are in -the `tvix` directory: - -``` -export PATH=$PATH:$PWD/target/release-with-debug -``` - -Secondly, configure tvix to use the local backend: - -``` -export BLOB_SERVICE_ADDR=sled://$PWD/blobs.sled -export DIRECTORY_SERVICE_ADDR=sled://$PWD/directories.sled -export PATH_INFO_SERVICE_ADDR=sled://$PWD/pathinfo.sled -``` - -Potentially copy some data into tvix-store (via nar-bridge): - -``` -mg run //tvix:store -- daemon & -mg run //tvix:nar-bridge -- & -rm -Rf ~/.cache/nix; nix copy --to http://localhost:9000\?compression\=none $(mg build //third_party/nixpkgs:hello) -pkill nar-bridge; pkill tvix-store -``` - -#### Interactive shell -Run the VM like this: - -``` -CH_CMDLINE=tvix.shell mg run //tvix/tests:runVM -- -``` - -You'll get dropped into an interactive shell, from which you can do things with -the store: - -``` - ______ _ ____ _ __ - /_ __/ __(_) __ / _/___ (_) /_ - / / | | / / / |/_/ / // __ \/ / __/ - / / | |/ / /> < _/ // / / / / /_ -/_/ |___/_/_/|_| /___/_/ /_/_/\__/ - -/# ls -la /nix/store/ -dr-xr-xr-x root 0 0 Jan 1 00:00 . -dr-xr-xr-x root 0 989 Jan 1 00:00 aw2fw9ag10wr9pf0qk4nk5sxi0q0bn56-glibc-2.37-8 -dr-xr-xr-x root 0 3 Jan 1 00:00 jbwb8d8l28lg9z0xzl784wyb9vlbwss6-xgcc-12.3.0-libgcc -dr-xr-xr-x root 0 82 Jan 1 00:00 k8ivghpggjrq1n49xp8sj116i4sh8lia-libidn2-2.3.4 -dr-xr-xr-x root 0 141 Jan 1 00:00 mdi7lvrn2mx7rfzv3fdq3v5yw8swiks6-hello-2.12.1 -dr-xr-xr-x root 0 5 Jan 1 00:00 s2gi8pfjszy6rq3ydx0z1vwbbskw994i-libunistring-1.1 -``` - -Once you exit the shell, the VM will power off itself. - -#### Execute a specific binary -Run the VM like this: - -``` -hello_cmd=$(mg build //third_party/nixpkgs:hello)/bin/hello -CH_CMDLINE=tvix.run=$hello_cmd mg run //tvix/tests:runVM -- -``` - -Observe it executing the file (and closure) from the tvix-store: - -``` -[ 0.277486] Run /init as init process - ______ _ ____ _ __ - /_ __/ __(_) __ / _/___ (_) /_ - / / | | / / / |/_/ / // __ \/ / __/ - / / | |/ / /> < _/ // / / / / /_ -/_/ |___/_/_/|_| /___/_/ /_/_/\__/ - -Hello, world! -2023/09/24 21:10:19 Nothing left to be done, powering off. -[ 0.299122] ACPI: PM: Preparing to enter system sleep state S5 -[ 0.299422] reboot: Power down -``` - -#### Execute a NixOS system closure -It's also possible to invoke a system closure. To do this, tvix-init honors the -init= cmdline option, and will switch_root to it. - - -``` -CH_CMDLINE=init=/nix/store/…-nixos-system-…/init mg run //tvix/tests:runVM -- -``` - -``` - ______ _ ____ _ __ - /_ __/ __(_) __ / _/___ (_) /_ - / / | | / / / |/_/ / // __ \/ / __/ - / / | |/ / /> < _/ // / / / / /_ -/_/ |___/_/_/|_| /___/_/ /_/_/\__/ - -2023/09/24 21:16:43 switch_root: moving mounts -2023/09/24 21:16:43 switch_root: Skipping "/run" as the dir does not exist -2023/09/24 21:16:43 switch_root: Changing directory -2023/09/24 21:16:43 switch_root: Moving / -2023/09/24 21:16:43 switch_root: Changing root! -2023/09/24 21:16:43 switch_root: Deleting old / -2023/09/24 21:16:43 switch_root: executing init - -<<< NixOS Stage 2 >>> - -[ 0.322096] booting system configuration /nix/store/g657sdxinpqfcdv0162zmb8vv9b5c4c5-nixos-system-client-23.11.git.82102fc37da -running activation script... -setting up /etc... -starting systemd... -[ 0.980740] systemd[1]: systemd 253.6 running in system mode (+PAM +AUDIT -SELINUX +APPARMOR +IMA +SMACK +SECCOMP +GCRYPT -GNUTLS +OPENSSL +ACL +BLKID +CURL +ELFUTILS +FIDO2 +IDN2 -IDN +IPTC +KMOD +LIBCRYPTSETUP +LIBFDISK +PCRE2 -PWQUALITY +P11KIT -QRENCODE +TPM2 +BZIP2 +LZ4 +XZ +ZLIB +ZSTD +BPF_FRAMEWORK -XKBCOMMON +UTMP -SYSVINIT default-hierarchy=unified) -``` - -This effectively replaces the NixOS Stage 1 entirely. \ No newline at end of file diff --git a/tvix/tests/default.nix b/tvix/tests/default.nix deleted file mode 100644 index 875fe0b9f1bf..000000000000 --- a/tvix/tests/default.nix +++ /dev/null @@ -1,137 +0,0 @@ -{ depot, pkgs, ... }: - -rec { - # A binary that sets up /nix/store from virtiofs, lists all store paths, and - # powers off the machine. - tvix-init = depot.nix.buildGo.program { - name = "tvix-init"; - srcs = [ - ./tvix-init.go - ]; - }; - - # A kernel with virtiofs support baked in - kernel = pkgs.buildLinux ({ } // { - inherit (pkgs.linuxPackages_latest.kernel) src version modDirVersion; - autoModules = false; - kernelPreferBuiltin = true; - ignoreConfigErrors = true; - kernelPatches = [ ]; - structuredExtraConfig = with pkgs.lib.kernel; { - FUSE_FS = option yes; - DAX_DRIVER = option yes; - DAX = option yes; - FS_DAX = option yes; - VIRTIO_FS = option yes; - VIRTIO = option yes; - ZONE_DEVICE = option yes; - }; - }); - - # A build framework for minimal initrds - uroot = pkgs.buildGoModule { - pname = "u-root"; - version = "unstable-2023-09-20"; - src = pkgs.fetchFromGitHub { - owner = "u-root"; - repo = "u-root"; - rev = "72921548ce2e88c4c5b62e83c717cbd834b58067"; - hash = "sha256-fEoUGqh6ZXprtSpJ55MeuSFe7L5A/rkIIVLCwxbPHzE="; - }; - vendorHash = null; - - doCheck = false; # Some tests invoke /bin/bash - }; - - # Use u-root to build a initrd with our tvix-init inside. - initrd = pkgs.stdenv.mkDerivation { - name = "initrd.cpio"; - nativeBuildInputs = [ pkgs.go ]; - # https://github.com/u-root/u-root/issues/2466 - buildCommand = '' - mkdir -p /tmp/go/src/github.com/u-root/ - cp -R ${uroot.src} /tmp/go/src/github.com/u-root/u-root - cd /tmp/go/src/github.com/u-root/u-root - chmod +w . - cp ${tvix-init}/bin/tvix-init tvix-init - - export HOME=$(mktemp -d) - export GOROOT="$(go env GOROOT)" - - GO111MODULE=off GOPATH=/tmp/go GOPROXY=off ${uroot}/bin/u-root -files ./tvix-init -initcmd "/tvix-init" -o $out - ''; - }; - - # Start a `tvix-store` virtiofs daemon from $PATH, then a cloud-hypervisor - # pointed to it. - # Supports the following env vars (and defaults) - # CH_NUM_CPUS=1 - # CH_MEM_SIZE=512M - # CH_CMDLINE="" - runVM = pkgs.writers.writeBashBin "run-tvix-vm" '' - tempdir=$(mktemp -d) - - cleanup() { - kill $virtiofsd_pid - if [[ -n ''${work_dir-} ]]; then - chmod -R u+rw "$tempdir" - rm -rf "$tempdir" - fi - } - trap cleanup EXIT - - # Spin up the virtiofs daemon - tvix-store virtiofs -l $tempdir/tvix.sock & - virtiofsd_pid=$! - - # Wait for the socket to exist. - until [ -e $tempdir/tvix.sock ]; do sleep 0.1; done - - CH_NUM_CPUS="''${CH_NUM_CPUS:-1}" - CH_MEM_SIZE="''${CH_MEM_SIZE:-512M}" - CH_CMDLINE="''${CH_CMDLINE:-}" - - # spin up cloud_hypervisor - ${pkgs.cloud-hypervisor}/bin/cloud-hypervisor \ - --cpus boot=$CH_NUM_CPU \ - --memory mergeable=on,shared=on,size=$CH_MEM_SIZE \ - --console null \ - --serial tty \ - --kernel ${kernel.dev}/vmlinux \ - --initramfs ${initrd} \ - --cmdline "console=ttyS0 $CH_CMDLINE" \ - --fs tag=tvix,socket=$tempdir/tvix.sock,num_queues=1,queue_size=512 - ''; - - # Seed a tvix-store with the tvix docs, then start a VM and search for the - # store path in the output. - test-docs = pkgs.stdenv.mkDerivation { - name = "run-vm"; - nativeBuildInputs = [ - depot.tvix.store - ]; - buildCommand = '' - touch $out - - # Configure tvix to put data in the local working directory - export BLOB_SERVICE_ADDR=sled://$PWD/blobs.sled - export DIRECTORY_SERVICE_ADDR=sled://$PWD/directories.sled - export PATH_INFO_SERVICE_ADDR=sled://$PWD/pathinfo.sled - - # Seed the tvix store with some data - # Create a `docs` directory with the contents from ../docs - # Make sure it still is called "docs" when calling import, so we can - # predict the store path. - cp -R ${../docs} docs - outpath=$(tvix-store import docs) - - echo "Store contents imported to $outpath" - - CH_CMDLINE="tvix.find" ${runVM}/bin/run-tvix-vm 2>&1 | tee output.txt - grep ${../docs} output.txt - ''; - requiredSystemFeatures = [ "kvm" ]; - }; - - meta.ci.targets = [ "test-docs" ]; -} diff --git a/tvix/tests/tvix-init.go b/tvix/tests/tvix-init.go deleted file mode 100644 index 97a24bab3547..000000000000 --- a/tvix/tests/tvix-init.go +++ /dev/null @@ -1,138 +0,0 @@ -package main - -import ( - "fmt" - "log" - "os" - "os/exec" - "strings" - "syscall" -) - -// run the given command, connecting std{in,err,out} with the OS one. -func run(args ...string) error { - cmd := exec.Command(args[0], args[1:]...) - cmd.Stdin = os.Stdin - cmd.Stderr = os.Stderr - cmd.Stdout = os.Stdout - - return cmd.Run() -} - -// parse the cmdline, return a map[string]string. -func parseCmdline(cmdline string) map[string]string { - line := strings.TrimSuffix(cmdline, "\n") - fields := strings.Fields(line) - out := make(map[string]string, 0) - - for _, arg := range fields { - kv := strings.SplitN(arg, "=", 2) - switch len(kv) { - case 1: - out[kv[0]] = "" - case 2: - out[kv[0]] = kv[1] - } - } - - return out -} - -// mounts the nix store from the virtiofs tag to the given destination, -// creating the destination if it doesn't exist already. -func mountTvixStore(dest string) error { - if err := os.MkdirAll(dest, os.ModePerm); err != nil { - return fmt.Errorf("unable to mkdir dest: %w", err) - } - if err := run("mount", "-t", "virtiofs", "tvix", dest, "-o", "ro"); err != nil { - return fmt.Errorf("unable to run mount: %w", err) - } - - return nil -} - -func main() { - fmt.Print(` - ______ _ ____ _ __ - /_ __/ __(_) __ / _/___ (_) /_ - / / | | / / / |/_/ / // __ \/ / __/ - / / | |/ / /> < _/ // / / / / /_ -/_/ |___/_/_/|_| /___/_/ /_/_/\__/ - -`) - - // Set PATH to "/bbin", so we can find the u-root tools - os.Setenv("PATH", "/bbin") - - if err := run("mount", "-t", "proc", "none", "/proc"); err != nil { - log.Printf("Failed to mount /proc: %v\n", err) - } - if err := run("mount", "-t", "sysfs", "none", "/sys"); err != nil { - log.Printf("Failed to mount /sys: %v\n", err) - } - if err := run("mount", "-t", "devtmpfs", "devtmpfs", "/dev"); err != nil { - log.Printf("Failed to mount /dev: %v\n", err) - } - - cmdline, err := os.ReadFile("/proc/cmdline") - if err != nil { - log.Printf("Failed to read cmdline: %s\n", err) - } - cmdlineFields := parseCmdline(string(cmdline)) - - if _, ok := cmdlineFields["tvix.find"]; ok { - // If tvix.find is set, invoke find /nix/store - if err := mountTvixStore("/nix/store"); err != nil { - log.Printf("Failed to mount tvix store: %v\n", err) - } - - if err := run("find", "/nix/store"); err != nil { - log.Printf("Failed to run find command: %s\n", err) - } - } else if _, ok := cmdlineFields["tvix.shell"]; ok { - // If tvix.shell is set, mount the nix store to /nix/store directly, - // then invoke the elvish shell - if err := mountTvixStore("/nix/store"); err != nil { - log.Printf("Failed to mount tvix store: %v\n", err) - } - - if err := run("elvish"); err != nil { - log.Printf("Failed to run shell: %s\n", err) - } - } else if v, ok := cmdlineFields["tvix.run"]; ok { - // If tvix.run is set, mount the nix store to /nix/store directly, - // then invoke the command. - if err := mountTvixStore("/nix/store"); err != nil { - log.Printf("Failed to mount tvix store: %v\n", err) - } - - if err := run(v); err != nil { - log.Printf("Failed to run command: %s\n", err) - } - } else if v, ok := cmdlineFields["init"]; ok { - // If init is set, invoke the binary specified (with switch_root), - // and prepare /fs beforehand as well. - os.Mkdir("/fs", os.ModePerm) - if err := run("mount", "-t", "tmpfs", "none", "/fs"); err != nil { - log.Fatalf("Failed to mount /fs tmpfs: %s\n", err) - } - - // Mount /fs/nix/store - if err := mountTvixStore("/fs/nix/store"); err != nil { - log.Fatalf("Failed to mount tvix store: %v\n", err) - } - - // Invoke switch_root, which will take care of moving /proc, /sys and /dev. - if err := syscall.Exec("/bbin/switch_root", []string{"switch_root", "/fs", v}, []string{}); err != nil { - log.Printf("Failed to switch root: %s\n", err) - } - } else { - log.Printf("No command detected, not knowing what to do!") - } - - // This is only reached in the non switch_root case. - log.Printf("Nothing left to be done, powering off.") - if err := run("poweroff"); err != nil { - log.Printf("Failed to run poweroff command: %v\n", err) - } -} -- cgit 1.4.1