feat(tools/tvlc): pass list of derivations to depot-scanner r/1711

You can now provide a list of Nix derivations to tvlc to get a git worktree + sparse-checkout containing only the paths needed to build the specified derivations.

Known bugs: even though //third_party is only passed to readdir(), git doesn't know this and includes all of //third_party/*.

Change-Id: I9dccebd3fbff4bb04ebd568175cf0a7e37d71ab3
Reviewed-on: https://cl.tvl.fyi/c/depot/+/1826
Tested-by: BuildkiteCI
Reviewed-by: tazjin <mail@tazj.in>

4 files changed, 36 insertions, 7 deletions

diff --git a/tools/depot-scanner/main.go b/tools/depot-scanner/main.go
index 8d69981c0d7e..273190258958 100644
--- a/tools/depot-scanner/main.go
+++ b/tools/depot-scanner/main.go
@@ -18,6 +18,7 @@ var nixStoreRoot = flag.String("store-path", "/nix/store/", "prefix for all vali
 
 var modeFlag = flag.String("mode", modeArchive, "operation mode. valid values: tar, print")
 var onlyFlag = flag.String("only", "", "only enable the listed output types, comma separated. valid values: DEPOT, STORE, CORE, UNKNOWN")
+var relativeFlag = flag.Bool("relpath", false, "when printing paths, print them relative to the root of their path type")
 
 const (
 	modeArchive = "tar"
@@ -164,16 +165,25 @@ func main() {
 	if *modeFlag == "print" {
 		if enabledPathTypes[pb.PathType_STORE] {
 			for k, _ := range results[nixStorePath] {
+				if *relativePath {
+					k = strings.TrimPrefix(k, *nixStoreRoot)
+					k = strings.TrimPrefix(k, "/")
+				}
 				fmt.Println(k)
 			}
 		}
 		if enabledPathTypes[pb.PathType_DEPOT] {
 			for k, _ := range results[depotPath] {
+				if *relativeFlag {
+					k = strings.TrimPrefix(k, *depotRoot)
+					k = strings.TrimPrefix(k, "/")
+				}
 				fmt.Println(k)
 			}
 		}
 		if enabledPathTypes[pb.PathType_CORE] {
 			for k, _ := range results[corePkgsPath] {
+				// TODO relativeFlag
 				fmt.Println(k)
 			}
 		}
diff --git a/tools/tvlc/common.sh b/tools/tvlc/common.sh
index 2c8cc603cb97..fe7605857fd3 100644
--- a/tools/tvlc/common.sh
+++ b/tools/tvlc/common.sh
@@ -3,9 +3,12 @@
 set -eu
 set -o pipefail
 
+source path-scripts
+
 XDG_DATA_HOME="${XDG_DATA_HOME:-$HOME/.local/share}"
 tvlc_root="$XDG_DATA_HOME/tvlc"
 
+nice_checkout_root=
 if [ -f "$tvlc_root"/nice_checkout_root ]; then
   nice_checkout_root="$(cat "$tvlc_root"/nice_checkout_root)"
 fi
diff --git a/tools/tvlc/default.nix b/tools/tvlc/default.nix
index de18af236fdd..ef0c69927fef 100644
--- a/tools/tvlc/default.nix
+++ b/tools/tvlc/default.nix
@@ -1,17 +1,31 @@
 { pkgs, depot, ... }:
 
 let
-  commonsh = ./common.sh;
-
-  # TODO(riking): path deduction
-  #tvix-instantiate="${third_party.nix}/bin/nix-instantiate"
   pathScripts = pkgs.writeShellScript "imports" ''
+    export tvix_instantiate="${depot.third_party.nix}/bin/nix-instantiate"
+    export depot_scanner="${depot.tools.depot-scanner}/bin/depot-scanner"
   '';
 
   # setup: git rev-parse --show-toplevel > $tvlc_root/depot_root
   # setup: mkdir $tvlc_root/clients
   # setup: echo 1 > $tvlc_root/next_clientid
 
+  commonsh = pkgs.stdenv.mkDerivation {
+    name = "common.sh";
+    src = ./common.sh;
+    doCheck = true;
+    unpackPhase = "true";
+    buildPhase = ''
+      substitute ${./common.sh} $out --replace path-scripts ${pathScripts}
+    '';
+    checkPhase = ''
+      ${pkgs.shellcheck}/bin/shellcheck $out ${pathScripts} && echo "SHELLCHECK OK"
+    '';
+    installPhase = ''
+      chmod +x $out
+    '';
+  };
+
   tvlcNew = pkgs.stdenv.mkDerivation {
     name = "tvlc-new";
     src = ./tvlc-new;
@@ -22,7 +36,7 @@ let
       substitute ${./tvlc-new} $out --replace common.sh ${commonsh}
     '';
     checkPhase = ''
-      ${pkgs.shellcheck}/bin/shellcheck $out ${commonsh} && echo "SHELLCHECK OK"
+      ${pkgs.shellcheck}/bin/shellcheck $out ${commonsh} ${pathScripts} && echo "SHELLCHECK OK"
     '';
     installPhase = ''
       chmod +x $out
@@ -30,6 +44,7 @@ let
   };
 
 in pkgs.stdenv.mkDerivation rec {
+  inherit pathScripts;
   inherit commonsh;
   inherit tvlcNew;
 }
diff --git a/tools/tvlc/tvlc-new b/tools/tvlc/tvlc-new
index e3e65b4f642c..4ef0df5d33b2 100755
--- a/tools/tvlc/tvlc-new
+++ b/tools/tvlc/tvlc-new
@@ -79,7 +79,8 @@ if [ -f "$DEPOT_ROOT/.git/refs/heads/$branch_name" ]; then
   exit 1
 fi
 
-# TODO(riking): tvlc-get-depends
+# The big one: call into Nix to figure out what paths the desired derivations depend on.
+readarray -t includedPaths < <("$depot_scanner" --mode 'print' --only 'DEPOT' --relpath --depot "$DEPOT_ROOT" --nix-bin "$tvix_instantiate" "$@")
 
 # bash math
 checkout_id=$(("$(cat "$tvlc_root/next_clientid")"))
@@ -95,7 +96,7 @@ mkdir "$DEPOT_ROOT/.git/worktrees/$checkout_id/info"
 
 cd "$checkout_dir"
 git sparse-checkout init --cone
-git sparse-checkout set "$@"
+git sparse-checkout set "${includedPaths[@]}"
 
 ln -s "$checkout_dir" "$nice_checkout_root"/"$checkout_name"