chore(3p/gerrit): create buildBazelPackageNG and migrate gerrit to it r/8358

This bumps Gerrit to 3.10.0, and also introduces a new mechanism for
building it that should hopefully have some more stable hashes than the
previous bodgery.

In this world, we only cache what we explicitly want to. There are some
hooks implemented for `rules_java` and `rules_nodejs` (before version
6) that force use of local binaries; this means we can drop the use of
the FHSUserEnv and use the java and nodejs binaries provided by nixpkgs
instead.

detzip is deleted; it hasn't been used in yonks.

We also add https://gerrit-review.googlesource.com/c/gerrit/+/431977,
which bumps the SSHd version so that we can have U2F-based SSH keys.

Change-Id: Ie12a9a33bbb1e4bd96aa252580aca3b8bc4a1205
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11963
Reviewed-by: lukegb <lukegb@tvl.fyi>
Autosubmit: lukegb <lukegb@tvl.fyi>
Tested-by: BuildkiteCI

16 files changed, 286 insertions, 0 deletions

diff --git a/nix/buildBazelPackageNG/.skip-subtree b/nix/buildBazelPackageNG/.skip-subtree
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/nix/buildBazelPackageNG/.skip-subtree
diff --git a/nix/buildBazelPackageNG/bazelRulesJavaHook/default.nix b/nix/buildBazelPackageNG/bazelRulesJavaHook/default.nix
new file mode 100644
index 000000000000..eb8332e44eef
--- /dev/null
+++ b/nix/buildBazelPackageNG/bazelRulesJavaHook/default.nix
@@ -0,0 +1,8 @@
+{ makeSetupHook }:
+
+makeSetupHook {
+  name = "rules_java_bazel_hook";
+  substitutions = {
+    local_java = ./local_java;
+  };
+} ./setup-hook.sh
diff --git a/nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/BUILD.bazel b/nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/BUILD.bazel
new file mode 100644
index 000000000000..8bea4954cd54
--- /dev/null
+++ b/nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/BUILD.bazel
@@ -0,0 +1,3 @@
+alias(name = "jdk", actual = "@local_jdk//:jdk")
+alias(name = "toolchain", actual = "@local_jdk//:toolchain")
+alias(name = "bootstrap_runtime_toolchain", actual = "@local_jdk//:bootstrap_runtime_toolchain")
diff --git a/nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/WORKSPACE b/nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/WORKSPACE
new file mode 100644
index 000000000000..5b3107898d75
--- /dev/null
+++ b/nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/WORKSPACE
@@ -0,0 +1 @@
+workspace(name = "local_java")
diff --git a/nix/buildBazelPackageNG/bazelRulesJavaHook/setup-hook.sh b/nix/buildBazelPackageNG/bazelRulesJavaHook/setup-hook.sh
new file mode 100644
index 000000000000..f7f7e3afe5bf
--- /dev/null
+++ b/nix/buildBazelPackageNG/bazelRulesJavaHook/setup-hook.sh
@@ -0,0 +1,17 @@
+prePatchHooks+=(_setupLocalJavaRepo)
+
+javaVersions=(11 17 21)
+javaPlatforms=(
+  "linux" "linux_aarch64" "linux_ppc64le" "linux_s390x"
+  "macos" "macos_aarch64"
+  "win" "win_arm64")
+
+_setupLocalJavaRepo() {
+	for javaVersion in ${javaVersions[@]}; do
+		for javaPlatform in ${javaPlatforms[@]}; do
+			bazelFlagsArray+=(
+				"--override_repository=remotejdk${javaVersion}_${javaPlatform}=@local_java@"
+			)
+		done
+	done
+}
diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/default.nix b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/default.nix
new file mode 100644
index 000000000000..c99cc39e9e4c
--- /dev/null
+++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/default.nix
@@ -0,0 +1,53 @@
+{ stdenvNoCC
+, lib
+, makeSetupHook
+, fetchFromGitHub
+, coreutils
+, gnugrep
+, nodejs
+, yarn
+, git
+, cacert
+}:
+let
+  rulesNodeJS = stdenvNoCC.mkDerivation rec {
+    pname = "bazelbuild-rules_nodejs";
+    version = "5.8.5";
+
+    src = fetchFromGitHub {
+      owner = "bazelbuild";
+      repo = "rules_nodejs";
+      rev = version;
+      hash = "sha256-6UbYRrOnS93+pK4VI016gQZv2jLCzkJn6wJ4vZNCNjY=";
+    };
+
+    dontBuild = true;
+
+    postPatch = ''
+      shopt -s globstar
+      for i in **/*.bzl **/*.sh **/*.cjs; do
+        substituteInPlace "$i" \
+          --replace-quiet '#!/usr/bin/env bash' '#!${stdenvNoCC.shell}' \
+          --replace-quiet '#!/bin/bash' '#!${stdenvNoCC.shell}'
+      done
+      sed -i '/^#!/a export PATH=${lib.makeBinPath [ coreutils gnugrep ]}:$PATH' internal/node/launcher.sh
+    '';
+
+    installPhase = ''
+      cp -R . $out
+    '';
+  };
+in makeSetupHook {
+  name = "bazelbuild-rules_nodejs-5-hook";
+  propagatedBuildInputs = [
+    nodejs
+    yarn
+    git
+    cacert
+  ];
+  substitutions = {
+    inherit nodejs yarn cacert rulesNodeJS;
+    local_node = ./local_node;
+    local_yarn = ./local_yarn;
+  };
+} ./setup-hook.sh
diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/BUILD b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/BUILD
new file mode 100644
index 000000000000..d764d23ffd1a
--- /dev/null
+++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/BUILD
@@ -0,0 +1,20 @@
+load("@build_bazel_rules_nodejs//nodejs:toolchain.bzl", _node_toolchain = "node_toolchain")
+
+package(default_visibility = ["//visibility:public"])
+
+exports_files([
+    "bin/node",
+    "bin/npm",
+])
+
+_node_toolchain(
+    name = "node_toolchain",
+    target_tool_path = "__NODEJS__/bin/node",
+    npm_path = "__NODEJS__/bin/npm",
+)
+
+toolchain(
+    name = "nodejs",
+    toolchain = ":node_toolchain",
+    toolchain_type = "@build_bazel_rules_nodejs//nodejs:toolchain_type",
+)
diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/WORKSPACE b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/WORKSPACE
new file mode 100644
index 000000000000..5bc1698b62d5
--- /dev/null
+++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/WORKSPACE
@@ -0,0 +1 @@
+workspace(name = "nodejs")
diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/node b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/node
new file mode 100644
index 000000000000..ef1f010f0bf3
--- /dev/null
+++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/node
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec "__NODEJS__/bin/node" "$@"
diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/npm b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/npm
new file mode 100644
index 000000000000..63a985dbde20
--- /dev/null
+++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/npm
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec "__NODEJS__/bin/npm" "$@"
diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/BUILD b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/BUILD
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/BUILD
diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/WORKSPACE b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/WORKSPACE
new file mode 100644
index 000000000000..2a1b7d4653a1
--- /dev/null
+++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/WORKSPACE
@@ -0,0 +1 @@
+workspace(name = "yarn")
diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/bin/yarn b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/bin/yarn
new file mode 100644
index 000000000000..2009572e4eff
--- /dev/null
+++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/bin/yarn
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec "__YARN__/bin/yarn" "$@"
diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/setup-hook.sh b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/setup-hook.sh
new file mode 100644
index 000000000000..5e3cf1eb94c2
--- /dev/null
+++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/setup-hook.sh
@@ -0,0 +1,63 @@
+prePatchHooks+=(_setupLocalNodeRepos)
+preBuildHooks+=(_setupYarnCache)
+
+case "$bazelPhase" in
+	cache)
+		postInstallHooks+=(_copyYarnCache)
+		;;
+	build)
+		preBuildHooks+=(_linkYarnCache)
+		;;
+	*)
+		echo "Unexpected bazelPhase '$bazelPhase' (want cache or build)" >&2
+		exit 1
+		;;
+esac
+
+
+_setupLocalNodeRepos() {
+	cp -R @local_node@ $HOME/local_node
+	chmod -R +w $HOME/local_node
+	substituteInPlace $HOME/local_node/bin/node \
+		--replace-fail '__NODEJS__' '@nodejs@'
+	substituteInPlace $HOME/local_node/bin/npm \
+		--replace-fail '__NODEJS__' '@nodejs@'
+	substituteInPlace $HOME/local_node/BUILD \
+		--replace-fail '__NODEJS__' '@nodejs@'
+	chmod -R +x $HOME/local_node/bin/*
+
+	cp -R @local_yarn@ $HOME/local_yarn
+	chmod -R +w $HOME/local_yarn
+	substituteInPlace $HOME/local_yarn/bin/yarn \
+		--replace-fail '__YARN__' '@yarn@'
+	chmod -R +x $HOME/local_yarn/bin/*
+
+	bazelFlagsArray+=(
+		"--override_repository=build_bazel_rules_nodejs=@rulesNodeJS@"
+
+		"--override_repository=nodejs_linux_amd64=$HOME/local_node"
+		"--override_repository=nodejs_linux_arm64=$HOME/local_node"
+		"--override_repository=nodejs_linux_s390x=$HOME/local_node"
+		"--override_repository=nodejs_linux_ppc64le=$HOME/local_node"
+		"--override_repository=nodejs_darwin_amd64=$HOME/local_node"
+		"--override_repository=nodejs_darwin_arm64=$HOME/local_node"
+		"--override_repository=nodejs_windows_amd64=$HOME/local_node"
+		"--override_repository=nodejs_windows_arm64=$HOME/local_node"
+		"--override_repository=nodejs=$HOME/local_node"
+
+		"--override_repository=yarn=$HOME/local_yarn"
+	)
+}
+
+_setupYarnCache() {
+	@yarn@/bin/yarn config set cafile "@cacert@/etc/ssl/certs/ca-bundle.crt"
+	@yarn@/bin/yarn config set yarn-offline-mirror "$HOME/yarn-offline-mirror"
+}
+
+_copyYarnCache() {
+	cp -R "$HOME/yarn-offline-mirror" "$out/yarn-offline-mirror"
+}
+
+_linkYarnCache() {
+	ln -sf "$cache/yarn-offline-mirror" "$HOME/yarn-offline-mirror"
+}
diff --git a/nix/buildBazelPackageNG/buildBazelPackageNG.nix b/nix/buildBazelPackageNG/buildBazelPackageNG.nix
new file mode 100644
index 000000000000..5195d3e89fa8
--- /dev/null
+++ b/nix/buildBazelPackageNG/buildBazelPackageNG.nix
@@ -0,0 +1,105 @@
+{ stdenv
+, lib
+, pkgs
+, coreutils
+}:
+
+{ name ? "${baseAttrs.pname}-${baseAttrs.version}"
+, bazelTargets
+, bazel ? pkgs.bazel
+, depsHash
+, extraCacheInstall ? ""
+, extraBuildSetup ? ""
+, extraBuildInstall ? ""
+, ...
+}@baseAttrs:
+
+let
+  cleanAttrs = lib.flip removeAttrs [
+    "bazelTargets" "depsHash" "extraCacheInstall" "extraBuildSetup" "extraBuildInstall"
+  ];
+  attrs = cleanAttrs baseAttrs;
+
+  base = stdenv.mkDerivation (attrs // {
+    nativeBuildInputs = (attrs.nativeBuildInputs or []) ++ [
+      bazel
+    ];
+
+    preUnpack = ''
+      if [[ ! -d $HOME ]]; then
+        export HOME=$NIX_BUILD_TOP/home
+        mkdir -p $HOME
+      fi
+    '';
+
+    bazelTargetNames = builtins.attrNames bazelTargets;
+  });
+
+  cache = base.overrideAttrs (base: {
+    name = "${name}-deps";
+
+    bazelPhase = "cache";
+
+    buildPhase = ''
+      runHook preBuild
+
+      bazel sync --repository_cache=repository-cache $bazelFlags "''${bazelFlagsArray[@]}"
+      bazel build --repository_cache=repository-cache --nobuild $bazelFlags "''${bazelFlagsArray[@]}" $bazelTargetNames
+
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      runHook preInstall
+
+      mkdir $out
+      echo "${bazel.version}" > $out/bazel_version
+      cp -R repository-cache $out/repository-cache
+      ${extraCacheInstall}
+
+      runHook postInstall
+    '';
+
+    outputHashMode = "recursive";
+    outputHash = depsHash;
+  });
+
+  build = base.overrideAttrs (base: {
+    bazelPhase = "build";
+
+    inherit cache;
+
+    nativeBuildInputs = (base.nativeBuildInputs or []) ++ [
+      coreutils
+    ];
+
+    buildPhase = ''
+      runHook preBuild
+
+      ${extraBuildSetup}
+      bazel build --repository_cache=$cache/repository-cache $bazelFlags "''${bazelFlagsArray[@]}" $bazelTargetNames
+
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      runHook preInstall
+
+      ${builtins.concatStringsSep "\n" (lib.mapAttrsToList (target: outPath: lib.optionalString (outPath != null) ''
+        TARGET_OUTPUTS="$(bazel cquery --repository_cache=$cache/repository-cache $bazelFlags "''${bazelFlagsArray[@]}" --output=files "${target}")"
+        if [[ "$(echo "$TARGET_OUTPUTS" | wc -l)" -gt 1 ]]; then
+          echo "Installing ${target}'s outputs ($TARGET_OUTPUTS) into ${outPath} as a directory"
+          mkdir -p "${outPath}"
+          cp $TARGET_OUTPUTS "${outPath}"
+        else
+          echo "Installing ${target}'s output ($TARGET_OUTPUTS) to ${outPath}"
+          mkdir -p "${dirOf outPath}"
+          cp "$TARGET_OUTPUTS" "${outPath}"
+        fi
+      '') bazelTargets)}
+      ${extraBuildInstall}
+
+      runHook postInstall
+    '';
+  });
+in build
diff --git a/nix/buildBazelPackageNG/default.nix b/nix/buildBazelPackageNG/default.nix
new file mode 100644
index 000000000000..c1584e66a5ef
--- /dev/null
+++ b/nix/buildBazelPackageNG/default.nix
@@ -0,0 +1,6 @@
+{ pkgs, ... }:
+
+(pkgs.callPackage ./buildBazelPackageNG.nix { }) // {
+  bazelRulesJavaHook = pkgs.callPackage ./bazelRulesJavaHook { };
+  bazelRulesNodeJS5Hook = pkgs.callPackage ./bazelRulesNodeJS5Hook { };
+}