From a73ca3f43dbf6b179509d6e5e3933be4cf99d989 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Sun, 14 Jun 2020 12:23:47 +0100 Subject: feat(gerrit): Create Gerrit derivation. This uses the actual Bazel build, using a variety of tricks and hacks to make it actually work. Bazel really wants to download linux binaries from the internet and run them. In lieu of trying to fix the build system to not do this, we instead put bazel inside an FHS environment, which allows the binaries to find their dependencies. We also have to patch a few things: * We use build --nobuild instead of fetch, so we only fetch the dependencies we actually need for the build and not, say, Windows binaries. * We don't remove rules_cc, because we need it as an external dependency, not bundled. * We do some manual fixes on the cache before packing, because we need to remove some in-tree sources (so they don't cause the hash to break, since the hashes differ each time they're generated), and also remove some extraneous files. * We explicitly turn off the repository and disk caches, because the .bazelrc at the root of the Gerrit tree turns them on, with paths pointing into the user's home directory. * detzip is used instead of the zip binary for packing bower_components into an archive. detzip doesn't create entries for directories, and also doesn't store most metadata (timestamps, etc.), and uses store (i.e. uncompressed) compression only. It also sorts the file tree before writing them into the file. Change-Id: I572c43f7175067ecb1b85cdf40dda13a52de1439 Reviewed-on: https://cl.tvl.fyi/c/depot/+/252 Reviewed-by: tazjin --- third_party/default.nix | 5 +- third_party/gerrit/default.nix | 150 ++++++++++++++++++++++++++++++++++++ third_party/gerrit/use_detzip.patch | 13 ++++ 3 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 third_party/gerrit/default.nix create mode 100644 third_party/gerrit/use_detzip.patch (limited to 'third_party') diff --git a/third_party/default.nix b/third_party/default.nix index 52faff0279..a72e04adce 100644 --- a/third_party/default.nix +++ b/third_party/default.nix @@ -33,6 +33,8 @@ let autoreconfHook bashInteractive bat + buildBazelPackage + buildFHSUserEnv buildGoModule buildGoPackage buildPackages @@ -51,6 +53,7 @@ let darwin dockerTools fetchFromGitHub + fetchgit fetchurl fetchzip fira @@ -58,7 +61,6 @@ let fira-mono fontconfig freetype - gerrit gettext glibc gmock @@ -70,6 +72,7 @@ let haskell iana-etc imagemagickBig + jdk jetbrains-mono jq kontemplate diff --git a/third_party/gerrit/default.nix b/third_party/gerrit/default.nix new file mode 100644 index 0000000000..de2fb71576 --- /dev/null +++ b/third_party/gerrit/default.nix @@ -0,0 +1,150 @@ +{ depot, pkgs, ... }: + +let + detzip = depot.nix.buildGo.program { + name = "detzip"; + srcs = [ ./detzip.go ]; + }; + bazelRunScript = pkgs.writeShellScriptBin "bazel-run" '' + yarn config set cache-folder "$bazelOut/external/yarn_cache" + export HOME="$bazelOut/external/home" + mkdir -p "$bazelOut/external/home" + exec /bin/bazel "$@" + ''; + bazelTop = pkgs.buildFHSUserEnv { + name = "bazel"; + targetPkgs = pkgs: [ + (pkgs.bazel.override { enableNixHacks = true; }) + detzip + pkgs.jdk + pkgs.zlib + pkgs.python + pkgs.curl + pkgs.nodejs + pkgs.yarn + pkgs.git + bazelRunScript + ]; + runScript = "/bin/bazel-run"; + }; + bazel = bazelTop // { override = x: bazelTop; }; + version = "3.2.1"; +in +pkgs.buildBazelPackage { + name = "gerrit"; + + src = pkgs.fetchgit { + url = "https://gerrit.googlesource.com/gerrit"; + rev = "v${version}"; + sha256 = "1xrckiqc2l07dib22rn6iicgvph8iyxjrzr15bs5x6jdg9cic1s8"; + fetchSubmodules = true; + }; + patches = [ ./use_detzip.patch ]; + + bazelTarget = "release"; + inherit bazel; + + bazelFlags = [ + "--repository_cache=" + "--disk_cache=" + ]; + removeRulesCC = false; + + fetchAttrs = { + sha256 = "18cglkmfi1nwcs8n0j62bkh8l9bcgp1g52ackbzs57v70cg93nq7"; + preBuild = '' + rm .bazelversion + ''; + buildPhase = '' + runHook preBuild + + BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 \ + USER=homeless-shelter \ + bazel \ + --output_base="$bazelOut" \ + --output_user_root="$bazelUserRoot" \ + build --nobuild \ + --loading_phase_threads=1 \ + $bazelFlags \ + $bazelFetchFlags \ + $bazelTarget + + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + # Remove all built in external workspaces, Bazel will recreate them when building + rm -rf $bazelOut/external/{bazel_tools,\@bazel_tools.marker} + rm -rf $bazelOut/external/{embedded_jdk,\@embedded_jdk.marker} + rm -rf $bazelOut/external/{local_config_cc,\@local_config_cc.marker} + rm -rf $bazelOut/external/{local_*,\@local_*.marker} + + # Clear markers + find $bazelOut/external -name '@*\.marker' -exec sh -c 'echo > {}' \; + + # Remove all vcs files + rm -rf $(find $bazelOut/external -type d -name .git) + rm -rf $(find $bazelOut/external -type d -name .svn) + rm -rf $(find $bazelOut/external -type d -name .hg) + + # Removing top-level symlinks along with their markers. + # This is needed because they sometimes point to temporary paths (?). + # For example, in Tensorflow-gpu build: + # platforms -> NIX_BUILD_TOP/tmp/install/35282f5123611afa742331368e9ae529/_embedded_binaries/platforms + find $bazelOut/external -maxdepth 1 -type l | while read symlink; do + name="$(basename "$symlink")" + rm -rf "$symlink" "$bazelOut/external/@$name.marker" + done + + # Patching symlinks to remove build directory reference + find $bazelOut/external -type l | while read symlink; do + new_target="$(readlink "$symlink" | sed "s,$NIX_BUILD_TOP,NIX_BUILD_TOP,")" + rm "$symlink" + ln -sf "$new_target" "$symlink" + done + + echo '${bazel.name}' > $bazelOut/external/.nix-bazel-version + + # Gerrit fixups: + # Remove polymer-bridges and ba-linkify, they're in-repo + rm -rf $bazelOut/external/yarn_cache/v6/npm-polymer-bridges-* + rm -rf $bazelOut/external/yarn_cache/v6/npm-ba-linkify-* + # Normalize permissions on .yarn-{tarball,metadata} files + find $bazelOut/external/yarn_cache \( -name .yarn-tarball.tgz -or -name .yarn-metadata.json \) -exec chmod 644 {} + + + (cd $bazelOut/ && tar czf $out --sort=name --mtime='@1' --owner=0 --group=0 --numeric-owner external/) + + runHook postInstall + ''; + }; + + buildAttrs = { + preConfigure = '' + rm .bazelversion + ''; + installPhase = '' + mkdir -p "$out"/webapps/ + cp bazel-bin/release.war "$out"/webapps/gerrit-${version}.war + ''; + }; + + passthru = { + # A list of plugins that are part of the gerrit.war file. + # Use `java -jar gerrit.war ls | grep -Po '(?<=plugins/)[^.]+' | sed -e 's,^,",' -e 's,$,",' | sort` to generate that list. + plugins = [ + "codemirror-editor" + "commit-message-length-validator" + "delete-project" + "download-commands" + "gitiles" + "hooks" + "plugin-manager" + "replication" + "reviewnotes" + "singleusergroup" + "webhooks" + ]; + }; +} diff --git a/third_party/gerrit/use_detzip.patch b/third_party/gerrit/use_detzip.patch new file mode 100644 index 0000000000..068bce7717 --- /dev/null +++ b/third_party/gerrit/use_detzip.patch @@ -0,0 +1,13 @@ +diff --git a/tools/js/download_bower.py b/tools/js/download_bower.py +index 1df4b826bc..65bda74082 100755 +--- a/tools/js/download_bower.py ++++ b/tools/js/download_bower.py +@@ -106,7 +106,7 @@ def main(): + args.b, '--quiet', 'install', '%s#%s' % (args.p, args.v))) + bc = os.path.join(cwd, 'bower_components') + subprocess.check_call( +- ['zip', '-q', '--exclude', '.bower.json', '-r', cached, args.n], ++ ['detzip', '--exclude', '.bower.json', cached, args.n], + cwd=bc) + + if args.s: -- cgit 1.4.1