feat(gerrit): Create Gerrit derivation. r/948

This uses the actual Bazel build, using a variety of tricks and hacks to
make it actually work.

Bazel really wants to download linux binaries from the internet and run
them. In lieu of trying to fix the build system to not do this, we
instead put bazel inside an FHS environment, which allows the binaries
to find their dependencies.

We also have to patch a few things:

* We use build --nobuild instead of fetch, so we only fetch the
  dependencies we actually need for the build and not, say, Windows
  binaries.
* We don't remove rules_cc, because we need it as an external
  dependency, not bundled.
* We do some manual fixes on the cache before packing, because we need
  to remove some in-tree sources (so they don't cause the hash to break,
  since the hashes differ each time they're generated), and also remove
  some extraneous files.
* We explicitly turn off the repository and disk caches, because the
  .bazelrc at the root of the Gerrit tree turns them on, with paths
  pointing into the user's home directory.
* detzip is used instead of the zip binary for packing bower_components
  into an archive. detzip doesn't create entries for directories, and
  also doesn't store most metadata (timestamps, etc.), and uses store
  (i.e. uncompressed) compression only. It also sorts the file tree
  before writing them into the file.

Change-Id: I572c43f7175067ecb1b85cdf40dda13a52de1439
Reviewed-on: https://cl.tvl.fyi/c/depot/+/252
Reviewed-by: tazjin <mail@tazj.in>

3 files changed, 167 insertions, 1 deletions

diff --git a/third_party/default.nix b/third_party/default.nix
index 52faff0279..a72e04adce 100644
--- a/third_party/default.nix
+++ b/third_party/default.nix
@@ -33,6 +33,8 @@ let
       autoreconfHook
       bashInteractive
       bat
+      buildBazelPackage
+      buildFHSUserEnv
       buildGoModule
       buildGoPackage
       buildPackages
@@ -51,6 +53,7 @@ let
       darwin
       dockerTools
       fetchFromGitHub
+      fetchgit
       fetchurl
       fetchzip
       fira
@@ -58,7 +61,6 @@ let
       fira-mono
       fontconfig
       freetype
-      gerrit
       gettext
       glibc
       gmock
@@ -70,6 +72,7 @@ let
       haskell
       iana-etc
       imagemagickBig
+      jdk
       jetbrains-mono
       jq
       kontemplate
diff --git a/third_party/gerrit/default.nix b/third_party/gerrit/default.nix
new file mode 100644
index 0000000000..de2fb71576
--- /dev/null
+++ b/third_party/gerrit/default.nix
@@ -0,0 +1,150 @@
+{ depot, pkgs, ... }:
+
+let
+  detzip = depot.nix.buildGo.program {
+    name = "detzip";
+    srcs = [ ./detzip.go ];
+  };
+  bazelRunScript = pkgs.writeShellScriptBin "bazel-run" ''
+    yarn config set cache-folder "$bazelOut/external/yarn_cache"
+    export HOME="$bazelOut/external/home"
+    mkdir -p "$bazelOut/external/home"
+    exec /bin/bazel "$@"
+  '';
+  bazelTop = pkgs.buildFHSUserEnv {
+    name = "bazel";
+    targetPkgs = pkgs: [
+      (pkgs.bazel.override { enableNixHacks = true; })
+      detzip
+      pkgs.jdk
+      pkgs.zlib
+      pkgs.python
+      pkgs.curl
+      pkgs.nodejs
+      pkgs.yarn
+      pkgs.git
+      bazelRunScript
+    ];
+    runScript = "/bin/bazel-run";
+  };
+  bazel = bazelTop // { override = x: bazelTop; };
+  version = "3.2.1";
+in
+pkgs.buildBazelPackage {
+  name = "gerrit";
+
+  src = pkgs.fetchgit {
+    url = "https://gerrit.googlesource.com/gerrit";
+    rev = "v${version}";
+    sha256 = "1xrckiqc2l07dib22rn6iicgvph8iyxjrzr15bs5x6jdg9cic1s8";
+    fetchSubmodules = true;
+  };
+  patches = [ ./use_detzip.patch ];
+
+  bazelTarget = "release";
+  inherit bazel;
+
+  bazelFlags = [
+    "--repository_cache="
+    "--disk_cache="
+  ];
+  removeRulesCC = false;
+
+  fetchAttrs = {
+    sha256 = "18cglkmfi1nwcs8n0j62bkh8l9bcgp1g52ackbzs57v70cg93nq7";
+    preBuild = ''
+      rm .bazelversion
+    '';
+    buildPhase = ''
+      runHook preBuild
+
+      BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 \
+      USER=homeless-shelter \
+      bazel \
+        --output_base="$bazelOut" \
+        --output_user_root="$bazelUserRoot" \
+        build --nobuild \
+        --loading_phase_threads=1 \
+        $bazelFlags \
+        $bazelFetchFlags \
+        $bazelTarget
+
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      runHook preInstall
+
+      # Remove all built in external workspaces, Bazel will recreate them when building
+      rm -rf $bazelOut/external/{bazel_tools,\@bazel_tools.marker}
+      rm -rf $bazelOut/external/{embedded_jdk,\@embedded_jdk.marker}
+      rm -rf $bazelOut/external/{local_config_cc,\@local_config_cc.marker}
+      rm -rf $bazelOut/external/{local_*,\@local_*.marker}
+
+      # Clear markers
+      find $bazelOut/external -name '@*\.marker' -exec sh -c 'echo > {}' \;
+
+      # Remove all vcs files
+      rm -rf $(find $bazelOut/external -type d -name .git)
+      rm -rf $(find $bazelOut/external -type d -name .svn)
+      rm -rf $(find $bazelOut/external -type d -name .hg)
+
+      # Removing top-level symlinks along with their markers.
+      # This is needed because they sometimes point to temporary paths (?).
+      # For example, in Tensorflow-gpu build:
+      # platforms -> NIX_BUILD_TOP/tmp/install/35282f5123611afa742331368e9ae529/_embedded_binaries/platforms
+      find $bazelOut/external -maxdepth 1 -type l | while read symlink; do
+        name="$(basename "$symlink")"
+        rm -rf "$symlink" "$bazelOut/external/@$name.marker"
+      done
+
+      # Patching symlinks to remove build directory reference
+      find $bazelOut/external -type l | while read symlink; do
+        new_target="$(readlink "$symlink" | sed "s,$NIX_BUILD_TOP,NIX_BUILD_TOP,")"
+        rm "$symlink"
+        ln -sf "$new_target" "$symlink"
+      done
+
+      echo '${bazel.name}' > $bazelOut/external/.nix-bazel-version
+
+      # Gerrit fixups:
+      # Remove polymer-bridges and ba-linkify, they're in-repo
+      rm -rf $bazelOut/external/yarn_cache/v6/npm-polymer-bridges-*
+      rm -rf $bazelOut/external/yarn_cache/v6/npm-ba-linkify-*
+      # Normalize permissions on .yarn-{tarball,metadata} files
+      find $bazelOut/external/yarn_cache \( -name .yarn-tarball.tgz -or -name .yarn-metadata.json \) -exec chmod 644 {} +
+
+      (cd $bazelOut/ && tar czf $out --sort=name --mtime='@1' --owner=0 --group=0 --numeric-owner external/)
+
+      runHook postInstall
+    '';
+  };
+
+  buildAttrs = {
+    preConfigure = ''
+      rm .bazelversion
+    '';
+    installPhase = ''
+      mkdir -p "$out"/webapps/
+      cp bazel-bin/release.war "$out"/webapps/gerrit-${version}.war
+    '';
+  };
+
+  passthru = {
+    # A list of plugins that are part of the gerrit.war file.
+    # Use `java -jar gerrit.war ls | grep -Po '(?<=plugins/)[^.]+' | sed -e 's,^,",' -e 's,$,",' | sort` to generate that list.
+    plugins = [
+      "codemirror-editor"
+      "commit-message-length-validator"
+      "delete-project"
+      "download-commands"
+      "gitiles"
+      "hooks"
+      "plugin-manager"
+      "replication"
+      "reviewnotes"
+      "singleusergroup"
+      "webhooks"
+    ];
+  };
+}
diff --git a/third_party/gerrit/use_detzip.patch b/third_party/gerrit/use_detzip.patch
new file mode 100644
index 0000000000..068bce7717
--- /dev/null
+++ b/third_party/gerrit/use_detzip.patch
@@ -0,0 +1,13 @@
+diff --git a/tools/js/download_bower.py b/tools/js/download_bower.py
+index 1df4b826bc..65bda74082 100755
+--- a/tools/js/download_bower.py
++++ b/tools/js/download_bower.py
+@@ -106,7 +106,7 @@ def main():
+                 args.b, '--quiet', 'install', '%s#%s' % (args.p, args.v)))
+         bc = os.path.join(cwd, 'bower_components')
+         subprocess.check_call(
+-            ['zip', '-q', '--exclude', '.bower.json', '-r', cached, args.n],
++            ['detzip', '--exclude', '.bower.json', cached, args.n],
+             cwd=bc)
+ 
+         if args.s: