diff options
author | Luke Granger-Brown <git@lukegb.com> | 2024-07-07T18·12+0100 |
---|---|---|
committer | clbot <clbot@tvl.fyi> | 2024-07-08T00·17+0000 |
commit | c05bf02a856121cdf40f77a21cdb26667d449615 (patch) | |
tree | 09b59054c7a8d47dccd40609bf9880112f7bcace | |
parent | d17c3d96b61a38b8a1900ca3b08bafff8e863cd2 (diff) |
chore(3p/gerrit): create buildBazelPackageNG and migrate gerrit to it r/8358
This bumps Gerrit to 3.10.0, and also introduces a new mechanism for building it that should hopefully have some more stable hashes than the previous bodgery. In this world, we only cache what we explicitly want to. There are some hooks implemented for `rules_java` and `rules_nodejs` (before version 6) that force use of local binaries; this means we can drop the use of the FHSUserEnv and use the java and nodejs binaries provided by nixpkgs instead. detzip is deleted; it hasn't been used in yonks. We also add https://gerrit-review.googlesource.com/c/gerrit/+/431977, which bumps the SSHd version so that we can have U2F-based SSH keys. Change-Id: Ie12a9a33bbb1e4bd96aa252580aca3b8bc4a1205 Reviewed-on: https://cl.tvl.fyi/c/depot/+/11963 Reviewed-by: lukegb <lukegb@tvl.fyi> Autosubmit: lukegb <lukegb@tvl.fyi> Tested-by: BuildkiteCI
27 files changed, 458 insertions, 263 deletions
diff --git a/nix/buildBazelPackageNG/.skip-subtree b/nix/buildBazelPackageNG/.skip-subtree new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/nix/buildBazelPackageNG/.skip-subtree diff --git a/nix/buildBazelPackageNG/bazelRulesJavaHook/default.nix b/nix/buildBazelPackageNG/bazelRulesJavaHook/default.nix new file mode 100644 index 000000000000..eb8332e44eef --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesJavaHook/default.nix @@ -0,0 +1,8 @@ +{ makeSetupHook }: + +makeSetupHook { + name = "rules_java_bazel_hook"; + substitutions = { + local_java = ./local_java; + }; +} ./setup-hook.sh diff --git a/nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/BUILD.bazel b/nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/BUILD.bazel new file mode 100644 index 000000000000..8bea4954cd54 --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/BUILD.bazel @@ -0,0 +1,3 @@ +alias(name = "jdk", actual = "@local_jdk//:jdk") +alias(name = "toolchain", actual = "@local_jdk//:toolchain") +alias(name = "bootstrap_runtime_toolchain", actual = "@local_jdk//:bootstrap_runtime_toolchain") diff --git a/nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/WORKSPACE b/nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/WORKSPACE new file mode 100644 index 000000000000..5b3107898d75 --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/WORKSPACE @@ -0,0 +1 @@ +workspace(name = "local_java") diff --git a/nix/buildBazelPackageNG/bazelRulesJavaHook/setup-hook.sh b/nix/buildBazelPackageNG/bazelRulesJavaHook/setup-hook.sh new file mode 100644 index 000000000000..f7f7e3afe5bf --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesJavaHook/setup-hook.sh @@ -0,0 +1,17 @@ +prePatchHooks+=(_setupLocalJavaRepo) + +javaVersions=(11 17 21) +javaPlatforms=( + "linux" "linux_aarch64" "linux_ppc64le" "linux_s390x" + "macos" "macos_aarch64" + "win" "win_arm64") + +_setupLocalJavaRepo() { + for javaVersion in ${javaVersions[@]}; do + for javaPlatform in ${javaPlatforms[@]}; do + bazelFlagsArray+=( + "--override_repository=remotejdk${javaVersion}_${javaPlatform}=@local_java@" + ) + done + done +} diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/default.nix b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/default.nix new file mode 100644 index 000000000000..c99cc39e9e4c --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/default.nix @@ -0,0 +1,53 @@ +{ stdenvNoCC +, lib +, makeSetupHook +, fetchFromGitHub +, coreutils +, gnugrep +, nodejs +, yarn +, git +, cacert +}: +let + rulesNodeJS = stdenvNoCC.mkDerivation rec { + pname = "bazelbuild-rules_nodejs"; + version = "5.8.5"; + + src = fetchFromGitHub { + owner = "bazelbuild"; + repo = "rules_nodejs"; + rev = version; + hash = "sha256-6UbYRrOnS93+pK4VI016gQZv2jLCzkJn6wJ4vZNCNjY="; + }; + + dontBuild = true; + + postPatch = '' + shopt -s globstar + for i in **/*.bzl **/*.sh **/*.cjs; do + substituteInPlace "$i" \ + --replace-quiet '#!/usr/bin/env bash' '#!${stdenvNoCC.shell}' \ + --replace-quiet '#!/bin/bash' '#!${stdenvNoCC.shell}' + done + sed -i '/^#!/a export PATH=${lib.makeBinPath [ coreutils gnugrep ]}:$PATH' internal/node/launcher.sh + ''; + + installPhase = '' + cp -R . $out + ''; + }; +in makeSetupHook { + name = "bazelbuild-rules_nodejs-5-hook"; + propagatedBuildInputs = [ + nodejs + yarn + git + cacert + ]; + substitutions = { + inherit nodejs yarn cacert rulesNodeJS; + local_node = ./local_node; + local_yarn = ./local_yarn; + }; +} ./setup-hook.sh diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/BUILD b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/BUILD new file mode 100644 index 000000000000..d764d23ffd1a --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/BUILD @@ -0,0 +1,20 @@ +load("@build_bazel_rules_nodejs//nodejs:toolchain.bzl", _node_toolchain = "node_toolchain") + +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "bin/node", + "bin/npm", +]) + +_node_toolchain( + name = "node_toolchain", + target_tool_path = "__NODEJS__/bin/node", + npm_path = "__NODEJS__/bin/npm", +) + +toolchain( + name = "nodejs", + toolchain = ":node_toolchain", + toolchain_type = "@build_bazel_rules_nodejs//nodejs:toolchain_type", +) diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/WORKSPACE b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/WORKSPACE new file mode 100644 index 000000000000..5bc1698b62d5 --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/WORKSPACE @@ -0,0 +1 @@ +workspace(name = "nodejs") diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/node b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/node new file mode 100644 index 000000000000..ef1f010f0bf3 --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/node @@ -0,0 +1,3 @@ +#!/bin/sh + +exec "__NODEJS__/bin/node" "$@" diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/npm b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/npm new file mode 100644 index 000000000000..63a985dbde20 --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/npm @@ -0,0 +1,3 @@ +#!/bin/sh + +exec "__NODEJS__/bin/npm" "$@" diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/BUILD b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/BUILD new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/BUILD diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/WORKSPACE b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/WORKSPACE new file mode 100644 index 000000000000..2a1b7d4653a1 --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/WORKSPACE @@ -0,0 +1 @@ +workspace(name = "yarn") diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/bin/yarn b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/bin/yarn new file mode 100644 index 000000000000..2009572e4eff --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/bin/yarn @@ -0,0 +1,2 @@ +#!/bin/sh +exec "__YARN__/bin/yarn" "$@" diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/setup-hook.sh b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/setup-hook.sh new file mode 100644 index 000000000000..5e3cf1eb94c2 --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/setup-hook.sh @@ -0,0 +1,63 @@ +prePatchHooks+=(_setupLocalNodeRepos) +preBuildHooks+=(_setupYarnCache) + +case "$bazelPhase" in + cache) + postInstallHooks+=(_copyYarnCache) + ;; + build) + preBuildHooks+=(_linkYarnCache) + ;; + *) + echo "Unexpected bazelPhase '$bazelPhase' (want cache or build)" >&2 + exit 1 + ;; +esac + + +_setupLocalNodeRepos() { + cp -R @local_node@ $HOME/local_node + chmod -R +w $HOME/local_node + substituteInPlace $HOME/local_node/bin/node \ + --replace-fail '__NODEJS__' '@nodejs@' + substituteInPlace $HOME/local_node/bin/npm \ + --replace-fail '__NODEJS__' '@nodejs@' + substituteInPlace $HOME/local_node/BUILD \ + --replace-fail '__NODEJS__' '@nodejs@' + chmod -R +x $HOME/local_node/bin/* + + cp -R @local_yarn@ $HOME/local_yarn + chmod -R +w $HOME/local_yarn + substituteInPlace $HOME/local_yarn/bin/yarn \ + --replace-fail '__YARN__' '@yarn@' + chmod -R +x $HOME/local_yarn/bin/* + + bazelFlagsArray+=( + "--override_repository=build_bazel_rules_nodejs=@rulesNodeJS@" + + "--override_repository=nodejs_linux_amd64=$HOME/local_node" + "--override_repository=nodejs_linux_arm64=$HOME/local_node" + "--override_repository=nodejs_linux_s390x=$HOME/local_node" + "--override_repository=nodejs_linux_ppc64le=$HOME/local_node" + "--override_repository=nodejs_darwin_amd64=$HOME/local_node" + "--override_repository=nodejs_darwin_arm64=$HOME/local_node" + "--override_repository=nodejs_windows_amd64=$HOME/local_node" + "--override_repository=nodejs_windows_arm64=$HOME/local_node" + "--override_repository=nodejs=$HOME/local_node" + + "--override_repository=yarn=$HOME/local_yarn" + ) +} + +_setupYarnCache() { + @yarn@/bin/yarn config set cafile "@cacert@/etc/ssl/certs/ca-bundle.crt" + @yarn@/bin/yarn config set yarn-offline-mirror "$HOME/yarn-offline-mirror" +} + +_copyYarnCache() { + cp -R "$HOME/yarn-offline-mirror" "$out/yarn-offline-mirror" +} + +_linkYarnCache() { + ln -sf "$cache/yarn-offline-mirror" "$HOME/yarn-offline-mirror" +} diff --git a/nix/buildBazelPackageNG/buildBazelPackageNG.nix b/nix/buildBazelPackageNG/buildBazelPackageNG.nix new file mode 100644 index 000000000000..5195d3e89fa8 --- /dev/null +++ b/nix/buildBazelPackageNG/buildBazelPackageNG.nix @@ -0,0 +1,105 @@ +{ stdenv +, lib +, pkgs +, coreutils +}: + +{ name ? "${baseAttrs.pname}-${baseAttrs.version}" +, bazelTargets +, bazel ? pkgs.bazel +, depsHash +, extraCacheInstall ? "" +, extraBuildSetup ? "" +, extraBuildInstall ? "" +, ... +}@baseAttrs: + +let + cleanAttrs = lib.flip removeAttrs [ + "bazelTargets" "depsHash" "extraCacheInstall" "extraBuildSetup" "extraBuildInstall" + ]; + attrs = cleanAttrs baseAttrs; + + base = stdenv.mkDerivation (attrs // { + nativeBuildInputs = (attrs.nativeBuildInputs or []) ++ [ + bazel + ]; + + preUnpack = '' + if [[ ! -d $HOME ]]; then + export HOME=$NIX_BUILD_TOP/home + mkdir -p $HOME + fi + ''; + + bazelTargetNames = builtins.attrNames bazelTargets; + }); + + cache = base.overrideAttrs (base: { + name = "${name}-deps"; + + bazelPhase = "cache"; + + buildPhase = '' + runHook preBuild + + bazel sync --repository_cache=repository-cache $bazelFlags "''${bazelFlagsArray[@]}" + bazel build --repository_cache=repository-cache --nobuild $bazelFlags "''${bazelFlagsArray[@]}" $bazelTargetNames + + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + mkdir $out + echo "${bazel.version}" > $out/bazel_version + cp -R repository-cache $out/repository-cache + ${extraCacheInstall} + + runHook postInstall + ''; + + outputHashMode = "recursive"; + outputHash = depsHash; + }); + + build = base.overrideAttrs (base: { + bazelPhase = "build"; + + inherit cache; + + nativeBuildInputs = (base.nativeBuildInputs or []) ++ [ + coreutils + ]; + + buildPhase = '' + runHook preBuild + + ${extraBuildSetup} + bazel build --repository_cache=$cache/repository-cache $bazelFlags "''${bazelFlagsArray[@]}" $bazelTargetNames + + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + ${builtins.concatStringsSep "\n" (lib.mapAttrsToList (target: outPath: lib.optionalString (outPath != null) '' + TARGET_OUTPUTS="$(bazel cquery --repository_cache=$cache/repository-cache $bazelFlags "''${bazelFlagsArray[@]}" --output=files "${target}")" + if [[ "$(echo "$TARGET_OUTPUTS" | wc -l)" -gt 1 ]]; then + echo "Installing ${target}'s outputs ($TARGET_OUTPUTS) into ${outPath} as a directory" + mkdir -p "${outPath}" + cp $TARGET_OUTPUTS "${outPath}" + else + echo "Installing ${target}'s output ($TARGET_OUTPUTS) to ${outPath}" + mkdir -p "${dirOf outPath}" + cp "$TARGET_OUTPUTS" "${outPath}" + fi + '') bazelTargets)} + ${extraBuildInstall} + + runHook postInstall + ''; + }); +in build diff --git a/nix/buildBazelPackageNG/default.nix b/nix/buildBazelPackageNG/default.nix new file mode 100644 index 000000000000..c1584e66a5ef --- /dev/null +++ b/nix/buildBazelPackageNG/default.nix @@ -0,0 +1,6 @@ +{ pkgs, ... }: + +(pkgs.callPackage ./buildBazelPackageNG.nix { }) // { + bazelRulesJavaHook = pkgs.callPackage ./bazelRulesJavaHook { }; + bazelRulesNodeJS5Hook = pkgs.callPackage ./bazelRulesNodeJS5Hook { }; +} diff --git a/third_party/gerrit/0001-Syntax-highlight-nix.patch b/third_party/gerrit/0001-Syntax-highlight-nix.patch index bdc3fd3b5510..d17dc27db094 100644 --- a/third_party/gerrit/0001-Syntax-highlight-nix.patch +++ b/third_party/gerrit/0001-Syntax-highlight-nix.patch @@ -1,4 +1,4 @@ -From 084e4f92fb58f7cd85303ba602fb3c40133c8fcc Mon Sep 17 00:00:00 2001 +From 216843cff4a8e41ad9887118751a412c1a22ce72 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown <git@lukegb.com> Date: Thu, 2 Jul 2020 23:02:32 +0100 Subject: [PATCH 1/3] Syntax highlight nix @@ -9,23 +9,23 @@ Subject: [PATCH 1/3] Syntax highlight nix 2 files changed, 2 insertions(+) diff --git a/polygerrit-ui/app/embed/diff/gr-syntax-layer/gr-syntax-layer-worker.ts b/polygerrit-ui/app/embed/diff/gr-syntax-layer/gr-syntax-layer-worker.ts -index a9f88bdd81..385249f280 100644 +index 50742903de..d1e89920cc 100644 --- a/polygerrit-ui/app/embed/diff/gr-syntax-layer/gr-syntax-layer-worker.ts +++ b/polygerrit-ui/app/embed/diff/gr-syntax-layer/gr-syntax-layer-worker.ts -@@ -93,6 +93,7 @@ const LANGUAGE_MAP = new Map<string, string>([ +@@ -98,6 +98,7 @@ const LANGUAGE_MAP = new Map<string, string>([ ['text/x-vhdl', 'vhdl'], ['text/x-yaml', 'yaml'], ['text/vbscript', 'vbscript'], + ['text/x-nix', 'nix'], ]); - const CLASS_PREFIX = 'gr-diff gr-syntax gr-syntax-'; + const CLASS_PREFIX = 'gr-syntax gr-syntax-'; diff --git a/resources/com/google/gerrit/server/mime/mime-types.properties b/resources/com/google/gerrit/server/mime/mime-types.properties -index 2f9561ba2e..739818ec05 100644 +index 642ef474a5..97f1ff835b 100644 --- a/resources/com/google/gerrit/server/mime/mime-types.properties +++ b/resources/com/google/gerrit/server/mime/mime-types.properties -@@ -149,6 +149,7 @@ mscin = text/x-mscgen - msgenny = text/x-msgenny +@@ -154,6 +154,7 @@ msgenny = text/x-msgenny + mts = application/typescript nb = text/x-mathematica nginx.conf = text/x-nginx-conf +nix = text/x-nix @@ -33,5 +33,5 @@ index 2f9561ba2e..739818ec05 100644 nsi = text/x-nsis nt = text/n-triples -- -2.37.3 +2.45.1 diff --git a/third_party/gerrit/0002-Syntax-highlight-rules.pl.patch b/third_party/gerrit/0002-Syntax-highlight-rules.pl.patch index 4b91e2c3541f..d0da61dd0f21 100644 --- a/third_party/gerrit/0002-Syntax-highlight-rules.pl.patch +++ b/third_party/gerrit/0002-Syntax-highlight-rules.pl.patch @@ -1,4 +1,4 @@ -From aedf8ac8fa5113843bcd83ff85e2d9f3bffdb16c Mon Sep 17 00:00:00 2001 +From 63f1ff6ea749ae2af29a53463bca81bc3f4bf25b Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown <git@lukegb.com> Date: Thu, 2 Jul 2020 23:02:43 +0100 Subject: [PATCH 2/3] Syntax highlight rules.pl @@ -9,10 +9,10 @@ Subject: [PATCH 2/3] Syntax highlight rules.pl 2 files changed, 2 insertions(+) diff --git a/polygerrit-ui/app/embed/diff/gr-syntax-layer/gr-syntax-layer-worker.ts b/polygerrit-ui/app/embed/diff/gr-syntax-layer/gr-syntax-layer-worker.ts -index 385249f280..7cb3068494 100644 +index d1e89920cc..5d62af1c64 100644 --- a/polygerrit-ui/app/embed/diff/gr-syntax-layer/gr-syntax-layer-worker.ts +++ b/polygerrit-ui/app/embed/diff/gr-syntax-layer/gr-syntax-layer-worker.ts -@@ -68,6 +68,7 @@ const LANGUAGE_MAP = new Map<string, string>([ +@@ -72,6 +72,7 @@ const LANGUAGE_MAP = new Map<string, string>([ ['text/x-perl', 'perl'], ['text/x-pgsql', 'pgsql'], // postgresql ['text/x-php', 'php'], @@ -21,10 +21,10 @@ index 385249f280..7cb3068494 100644 ['text/x-protobuf', 'protobuf'], ['text/x-puppet', 'puppet'], diff --git a/resources/com/google/gerrit/server/mime/mime-types.properties b/resources/com/google/gerrit/server/mime/mime-types.properties -index 739818ec05..58eb727bf9 100644 +index 97f1ff835b..85d630340f 100644 --- a/resources/com/google/gerrit/server/mime/mime-types.properties +++ b/resources/com/google/gerrit/server/mime/mime-types.properties -@@ -200,6 +200,7 @@ rq = application/sparql-query +@@ -208,6 +208,7 @@ rq = application/sparql-query rs = text/x-rustsrc rss = application/xml rst = text/x-rst @@ -33,5 +33,5 @@ index 739818ec05..58eb727bf9 100644 s = text/x-gas sas = text/x-sas -- -2.37.3 +2.45.1 diff --git a/third_party/gerrit/0003-Add-titles-to-CLs-over-HTTP.patch b/third_party/gerrit/0003-Add-titles-to-CLs-over-HTTP.patch index c4edee3a40c3..a5881e5a6c27 100644 --- a/third_party/gerrit/0003-Add-titles-to-CLs-over-HTTP.patch +++ b/third_party/gerrit/0003-Add-titles-to-CLs-over-HTTP.patch @@ -1,19 +1,19 @@ -From f49c50ca9a84ca374b7bd91c171bbea0457f2c7a Mon Sep 17 00:00:00 2001 +From ca2df6d7f53441d443d42908e30bf60fbfc15392 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown <git@lukegb.com> Date: Thu, 2 Jul 2020 23:03:02 +0100 Subject: [PATCH 3/3] Add titles to CLs over HTTP --- - .../gerrit/httpd/raw/IndexHtmlUtil.java | 13 +++- + .../gerrit/httpd/raw/IndexHtmlUtil.java | 12 +++- .../google/gerrit/httpd/raw/IndexServlet.java | 8 ++- .../google/gerrit/httpd/raw/StaticModule.java | 5 +- .../gerrit/httpd/raw/TitleComputer.java | 67 +++++++++++++++++++ .../gerrit/httpd/raw/PolyGerritIndexHtml.soy | 4 +- - 5 files changed, 89 insertions(+), 8 deletions(-) + 5 files changed, 88 insertions(+), 8 deletions(-) create mode 100644 java/com/google/gerrit/httpd/raw/TitleComputer.java diff --git a/java/com/google/gerrit/httpd/raw/IndexHtmlUtil.java b/java/com/google/gerrit/httpd/raw/IndexHtmlUtil.java -index 72bfe40c3b..439bd73b44 100644 +index a92dd18f04..f87c46d321 100644 --- a/java/com/google/gerrit/httpd/raw/IndexHtmlUtil.java +++ b/java/com/google/gerrit/httpd/raw/IndexHtmlUtil.java @@ -41,6 +41,7 @@ import java.util.Collections; @@ -24,7 +24,7 @@ index 72bfe40c3b..439bd73b44 100644 import java.util.Set; import java.util.function.Function; -@@ -62,13 +63,14 @@ public class IndexHtmlUtil { +@@ -63,13 +64,14 @@ public class IndexHtmlUtil { String faviconPath, Map<String, String[]> urlParameterMap, Function<String, SanitizedContent> urlInScriptTagOrdainer, @@ -36,22 +36,21 @@ index 72bfe40c3b..439bd73b44 100644 data.putAll( staticTemplateData( canonicalURL, cdnPath, faviconPath, urlParameterMap, urlInScriptTagOrdainer)) -- .putAll(dynamicTemplateData(gerritApi, requestedURL)); -+ .putAll(dynamicTemplateData(gerritApi, requestedURL, titleComputer)); +- .putAll(dynamicTemplateData(gerritApi, requestedURL, canonicalURL)); ++ .putAll(dynamicTemplateData(gerritApi, requestedURL, canonicalURL, titleComputer)); Set<String> enabledExperiments = new HashSet<>(); enabledExperiments.addAll(experimentFeatures.getEnabledExperimentFeatures()); // Add all experiments enabled through url -@@ -81,7 +83,8 @@ public class IndexHtmlUtil { +@@ -82,7 +84,7 @@ public class IndexHtmlUtil { /** Returns dynamic parameters of {@code index.html}. */ public static ImmutableMap<String, Object> dynamicTemplateData( -- GerritApi gerritApi, String requestedURL) throws RestApiException, URISyntaxException { -+ GerritApi gerritApi, String requestedURL, TitleComputer titleComputer) -+ throws RestApiException, URISyntaxException { +- GerritApi gerritApi, String requestedURL, String canonicalURL) ++ GerritApi gerritApi, String requestedURL, String canonicalURL, TitleComputer titleComputer) + throws RestApiException, URISyntaxException { ImmutableMap.Builder<String, Object> data = ImmutableMap.builder(); Map<String, SanitizedContent> initialData = new HashMap<>(); - Server serverApi = gerritApi.config().server(); -@@ -129,6 +132,10 @@ public class IndexHtmlUtil { +@@ -141,6 +143,10 @@ public class IndexHtmlUtil { } data.put("gerritInitialData", initialData); @@ -102,17 +101,17 @@ index fcb821e5ae..e1464b992b 100644 } catch (URISyntaxException | RestApiException e) { throw new IOException(e); diff --git a/java/com/google/gerrit/httpd/raw/StaticModule.java b/java/com/google/gerrit/httpd/raw/StaticModule.java -index 15dcf42e0e..9f56bf33ce 100644 +index b00294f73e..f1c1aae12c 100644 --- a/java/com/google/gerrit/httpd/raw/StaticModule.java +++ b/java/com/google/gerrit/httpd/raw/StaticModule.java -@@ -241,10 +241,11 @@ public class StaticModule extends ServletModule { +@@ -224,10 +224,11 @@ public class StaticModule extends ServletModule { @CanonicalWebUrl @Nullable String canonicalUrl, @GerritServerConfig Config cfg, GerritApi gerritApi, - ExperimentFeatures experimentFeatures) { + ExperimentFeatures experimentFeatures, + TitleComputer titleComputer) { - String cdnPath = options.devCdn().orElse(cfg.getString("gerrit", null, "cdnPath")); + String cdnPath = options.devCdn().orElseGet(() -> cfg.getString("gerrit", null, "cdnPath")); String faviconPath = cfg.getString("gerrit", null, "faviconPath"); - return new IndexServlet(canonicalUrl, cdnPath, faviconPath, gerritApi, experimentFeatures); + return new IndexServlet(canonicalUrl, cdnPath, faviconPath, gerritApi, experimentFeatures, titleComputer); @@ -193,7 +192,7 @@ index 0000000000..8fd2053ad0 + } +} diff --git a/resources/com/google/gerrit/httpd/raw/PolyGerritIndexHtml.soy b/resources/com/google/gerrit/httpd/raw/PolyGerritIndexHtml.soy -index dbfef44dfe..347ee75aab 100644 +index 5ff1822cd9..81c3cdf0e1 100644 --- a/resources/com/google/gerrit/httpd/raw/PolyGerritIndexHtml.soy +++ b/resources/com/google/gerrit/httpd/raw/PolyGerritIndexHtml.soy @@ -33,10 +33,12 @@ @@ -211,5 +210,5 @@ index dbfef44dfe..347ee75aab 100644 <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=0">{\n} -- -2.37.3 +2.45.1 diff --git a/third_party/gerrit/bazelrc b/third_party/gerrit/bazelrc new file mode 100644 index 000000000000..f77da77b46ee --- /dev/null +++ b/third_party/gerrit/bazelrc @@ -0,0 +1,11 @@ +# Not using common --repository_cache because Gerrit's bazelrc overrides this... +build --repository_cache=repository-cache +build --action_env=SSL_CERT_FILE +build --action_env=GERRIT_CACHE_HOME +build --tool_java_runtime_version=local_jdk --java_runtime_version=local_jdk +build --workspace_status_command="cat .version" + +# Disable errorprone +build --javacopt="-XepDisableAllChecks" + +sync --repository_cache=repository-cache diff --git a/third_party/gerrit/default.nix b/third_party/gerrit/default.nix index a137946264ed..94d68a1dcc04 100644 --- a/third_party/gerrit/default.nix +++ b/third_party/gerrit/default.nix @@ -1,139 +1,88 @@ { depot, pkgs, ... }: let - bazelRunScript = pkgs.writeShellScriptBin "bazel-run" '' - yarn config set cache-folder "$bazelOut/external/yarn_cache" - export HOME="$bazelOut/external/home" - mkdir -p "$bazelOut/external/home" - exec /bin/bazel "$@" - ''; - bazelTop = pkgs.buildFHSUserEnv { - name = "bazel"; - targetPkgs = pkgs: [ - (pkgs.bazel_5.override { enableNixHacks = true; }) - pkgs.jdk17_headless - pkgs.zlib - pkgs.python3 - pkgs.curl - pkgs.nodejs - pkgs.yarn - pkgs.git - bazelRunScript - ]; - runScript = "/bin/bazel-run"; - }; - bazel = bazelTop // { override = x: bazelTop; }; - version = "3.9.1"; + inherit (depot.nix) buildBazelPackageNG; + inherit (buildBazelPackageNG) bazelRulesJavaHook bazelRulesNodeJS5Hook; in -pkgs.lib.makeOverridable pkgs.buildBazelPackage { +pkgs.lib.makeOverridable depot.nix.buildBazelPackageNG rec { pname = "gerrit"; - inherit version; + version = "3.10.0"; - src = pkgs.fetchgit { + bazel = pkgs.bazel_7; + + src = (pkgs.fetchgit { url = "https://gerrit.googlesource.com/gerrit"; - rev = "620a819cbf3c64fff7a66798822775ad42c91d8e"; - branchName = "v${version}"; - sha256 = "sha256:1mdxbgnx3mpxand4wq96ic38bb4yh45q271h40jrk7dk23sgmz02"; + rev = "v${version}"; fetchSubmodules = true; - }; + deepClone = true; + hash = "sha256-FpKuzityHuHNYBIOL8YUjCLlkuVBfxjvHECb26NsZNE="; + }).overrideAttrs (_: { + env.NIX_PREFETCH_GIT_CHECKOUT_HOOK = '' + pushd "$dir" >/dev/null + ${pkgs.python3}/bin/python tools/workspace_status_release.py | sort > .version + popd >/dev/null + + # delete all the .git; we can't do this using fetchgit if deepClone is on, + # but our mischief has already been achieved by the python command above :) + find "$dir" -name .git -print0 | xargs -0 rm -rf + ''; + }); + depsHash = "sha256-Z6pvnL8bv09K4wKaatUCsnXAtSno+BJO6rTKhDsHYv4="; patches = [ ./0001-Syntax-highlight-nix.patch ./0002-Syntax-highlight-rules.pl.patch ./0003-Add-titles-to-CLs-over-HTTP.patch + + ./gerrit-cl-431977-bump-sshd.patch ]; - bazelTargets = [ "release" "api-skip-javadoc" ]; - inherit bazel; + nativeBuildInputs = with pkgs; [ + bazelRulesJavaHook + bazelRulesNodeJS5Hook - bazelFlags = [ - "--repository_cache=" - "--disk_cache=" + curl + jdk + python3 + unzip ]; - removeRulesCC = false; - fetchConfigured = true; + prePatch = '' + rm .bazelversion - fetchAttrs = { - sha256 = "sha256:119mqli75c9fy05ddrlh2brjxb354yvv1ijjkk1y1yqcaqppwwb8"; - preBuild = '' - rm .bazelversion - ''; + ln -sf ${./bazelrc} user.bazelrc - installPhase = '' - runHook preInstall - - # Remove all built in external workspaces, Bazel will recreate them when building - rm -rf $bazelOut/external/{bazel_tools,\@bazel_tools.marker} - rm -rf $bazelOut/external/{embedded_jdk,\@embedded_jdk.marker} - rm -rf $bazelOut/external/{local_config_cc,\@local_config_cc.marker} - rm -rf $bazelOut/external/{local_*,\@local_*.marker} - - # Clear markers - find $bazelOut/external -name '@*\.marker' -exec sh -c 'echo > {}' \; - - # Remove all vcs files - rm -rf $(find $bazelOut/external -type d -name .git) - rm -rf $(find $bazelOut/external -type d -name .svn) - rm -rf $(find $bazelOut/external -type d -name .hg) - - # Removing top-level symlinks along with their markers. - # This is needed because they sometimes point to temporary paths (?). - # For example, in Tensorflow-gpu build: - #sha256:06bmzbcb9717s4b016kcbn8nr9pgaz04i8bnzg7ybkbdwpl8vxvv"; platforms -> NIX_BUILD_TOP/tmp/install/35282f5123611afa742331368e9ae529/_embedded_binaries/platforms - find $bazelOut/external -maxdepth 1 -type l | while read symlink; do - name="$(basename "$symlink")" - rm -rf "$symlink" "$bazelOut/external/@$name.marker" - done - - # Patching symlinks to remove build directory reference - find $bazelOut/external -type l | while read symlink; do - new_target="$(readlink "$symlink" | sed "s,$NIX_BUILD_TOP,NIX_BUILD_TOP,")" - rm "$symlink" - ln -sf "$new_target" "$symlink" - done - - echo '${bazel.name}' > $bazelOut/external/.nix-bazel-version - - # Gerrit fixups: - # Normalize permissions on .yarn-{tarball,metadata} files - test -d $bazelOut/external/yarn_cache && find $bazelOut/external/yarn_cache \( -name .yarn-tarball.tgz -or -name .yarn-metadata.json \) -exec chmod 644 {} + - - mkdir $bazelOut/_bits/ - find . -name node_modules -prune -print | while read d; do - echo "$d" "$(dirname $d)" - mkdir -p $bazelOut/_bits/$(dirname $d) - cp -R "$d" "$bazelOut/_bits/$(dirname $d)/node_modules" - done - - (cd $bazelOut/ && tar czf $out --sort=name --mtime='@1' --owner=0 --group=0 --numeric-owner external/ _bits/) - - runHook postInstall - ''; - }; + ln -sf ${./workspace_overrides.bzl} workspace_overrides.bzl + substituteInPlace WORKSPACE \ + --replace-fail 'load("@io_bazel_rules_webtesting//web:repositories.bzl"' 'load("//:workspace_overrides.bzl"' \ + --replace-fail 'load("@io_bazel_rules_webtesting//web/versioned:browsers-0.3.3.bzl"' 'load("//:workspace_overrides.bzl"' - buildAttrs = { - preConfigure = '' - rm .bazelversion + patchShebangs Documentation/replace_macros.py + ''; - [ "$(ls -A $bazelOut/_bits)" ] && cp -R $bazelOut/_bits/* ./ || true - ''; - postPatch = '' - # Disable all errorprone checks, since we might be using a different version. - sed -i \ - -e '/-Xep:/d' \ - -e '/-XepExcludedPaths:/a "-XepDisableAllChecks",' \ - tools/BUILD - ''; - installPhase = '' - mkdir -p "$out"/webapps/ "$out"/share/api/ - cp bazel-bin/release.war "$out"/webapps/gerrit-${version}.war - unzip bazel-bin/api-skip-javadoc.zip -d "$out"/share/api - ''; + postPatch = '' + sed -Ei 's,^(STABLE_BUILD_GERRIT_LABEL.*)$,\1-dirty-nix,' .version + ''; - nativeBuildInputs = with pkgs; [ - unzip - ]; + preBuild = '' + export GERRIT_CACHE_HOME=$HOME/gerrit-cache + ''; + + extraCacheInstall = '' + cp -R $GERRIT_CACHE_HOME $out/gerrit-cache + ''; + + extraBuildSetup = '' + ln -sf $cache/gerrit-cache $GERRIT_CACHE_HOME + ''; + extraBuildInstall = '' + mkdir -p "$out"/share/api/ + unzip bazel-bin/api-skip-javadoc.zip -d "$out"/share/api + ''; + + bazelTargets = { + "//:release" = "$out/webapps/gerrit-${version}.war"; + "//:api-skip-javadoc" = null; }; passthru = { @@ -154,5 +103,5 @@ pkgs.lib.makeOverridable pkgs.buildBazelPackage { ]; }; - meta.ci.targets = [ "deps" ]; + meta.ci.targets = [ "cache" ]; } diff --git a/third_party/gerrit/detzip.go b/third_party/gerrit/detzip.go deleted file mode 100644 index 511c18ecfe17..000000000000 --- a/third_party/gerrit/detzip.go +++ /dev/null @@ -1,97 +0,0 @@ -package main - -import ( - "archive/zip" - "flag" - "fmt" - "io" - "log" - "os" - "path/filepath" - "sort" - "strings" -) - -var ( - exclude = flag.String("exclude", "", "comma-separated list of filenames to exclude (in any directory)") -) - -func init() { - flag.Usage = func() { - fmt.Fprintf(flag.CommandLine.Output(), "Usage of %s [zip file] [directory]:\n", os.Args[0]) - flag.PrintDefaults() - } -} - -func listToMap(ss []string) map[string]bool { - m := make(map[string]bool) - for _, s := range ss { - m[s] = true - } - return m -} - -func main() { - flag.Parse() - if flag.NArg() != 2 { - flag.Usage() - os.Exit(1) - } - - outPath := flag.Arg(0) - dirPath := flag.Arg(1) - - excludeFiles := listToMap(strings.Split(*exclude, ",")) - - // Aggregate all files first. - var files []string - filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - if excludeFiles[info.Name()] { - return nil - } - files = append(files, path) - return nil - }) - - // Create zip - outW, err := os.Create(outPath) - if err != nil { - log.Fatalf("Create(%q): %v", outPath, err) - } - - zipW := zip.NewWriter(outW) - - // Output files in alphabetical order - sort.Strings(files) - for _, f := range files { - fw, err := zipW.CreateHeader(&zip.FileHeader{ - Name: f, - Method: zip.Store, - }) - if err != nil { - log.Fatalf("creating %q in zip: %v", f, err) - } - - ff, err := os.Open(f) - if err != nil { - log.Fatalf("opening %q: %v", f, err) - } - if _, err := io.Copy(fw, ff); err != nil { - log.Fatalf("copying %q to zip: %v", f, err) - } - ff.Close() - } - - if err := zipW.Close(); err != nil { - log.Fatalf("writing ZIP central directory: %v", err) - } - if err := outW.Close(); err != nil { - log.Fatalf("closing ZIP file: %v", err) - } -} diff --git a/third_party/gerrit/gerrit-cl-431977-bump-sshd.patch b/third_party/gerrit/gerrit-cl-431977-bump-sshd.patch new file mode 100644 index 000000000000..3a08df21ba7c --- /dev/null +++ b/third_party/gerrit/gerrit-cl-431977-bump-sshd.patch @@ -0,0 +1,40 @@ +Bump SSHD version to 2.13.1 + +Release-Notes: Update SSHD version to 2.13.1 +Change-Id: Ib7bc185bfd9e7eda0cc04230da8bd87ee1bb2358 + +diff --git a/tools/nongoogle.bzl b/tools/nongoogle.bzl +index 91caf31..d9b90d8 100644 +--- a/tools/nongoogle.bzl ++++ b/tools/nongoogle.bzl +@@ -137,18 +137,18 @@ + sha1 = "cb2f351bf4463751201f43bb99865235d5ba07ca", + ) + +- SSHD_VERS = "2.12.0" ++ SSHD_VERS = "2.13.1" + + maven_jar( + name = "sshd-osgi", + artifact = "org.apache.sshd:sshd-osgi:" + SSHD_VERS, +- sha1 = "32b8de1cbb722ba75bdf9898e0c41d42af00ce57", ++ sha1 = "50958cc44076749e790d7332021cff546707624c", + ) + + maven_jar( + name = "sshd-sftp", + artifact = "org.apache.sshd:sshd-sftp:" + SSHD_VERS, +- sha1 = "0f96f00a07b186ea62838a6a4122e8f4cad44df6", ++ sha1 = "e1b6da4ef604718e32cad59ef32618610da7a170", + ) + + maven_jar( +@@ -166,7 +166,7 @@ + maven_jar( + name = "sshd-mina", + artifact = "org.apache.sshd:sshd-mina:" + SSHD_VERS, +- sha1 = "8b202f7d4c0d7b714fd0c93a1352af52aa031149", ++ sha1 = "ff4a9fac41a111d806f6a058d23278b0819da7ce", + ) + + maven_jar( diff --git a/third_party/gerrit/workspace_overrides.bzl b/third_party/gerrit/workspace_overrides.bzl new file mode 100644 index 000000000000..839368606557 --- /dev/null +++ b/third_party/gerrit/workspace_overrides.bzl @@ -0,0 +1,5 @@ +def web_test_repositories(): + pass + +def browser_repositories(*args, **kwargs): + pass diff --git a/third_party/gerrit_plugins/builder.nix b/third_party/gerrit_plugins/builder.nix index 50a4e78ae7cf..84515a9f68d2 100644 --- a/third_party/gerrit_plugins/builder.nix +++ b/third_party/gerrit_plugins/builder.nix @@ -2,30 +2,33 @@ { buildGerritBazelPlugin = { name + , version , src - , depsOutputHash + , depsHash ? null , overlayPluginCmd ? '' cp -R "${src}" "$out/plugins/${name}" + echo "STABLE_BUILD_${lib.toUpper name}_LABEL v${version}-nix${if patches != [] then "-dirty" else ""}" >> $out/.version '' + , postOverlayPlugin ? "" , postPatch ? "" , patches ? [ ] - }: ((depot.third_party.gerrit.override { + }: ((depot.third_party.gerrit.override (old: { name = "${name}.jar"; src = pkgs.runCommandLocal "${name}-src" { } '' cp -R "${depot.third_party.gerrit.src}" "$out" - chmod +w "$out/plugins" + chmod -R +w "$out" ${overlayPluginCmd} + ${postOverlayPlugin} ''; + depsHash = (if depsHash != null then depsHash else old.depsHash); - bazelTargets = [ "//plugins/${name}" ]; - }).overrideAttrs (super: { - deps = super.deps.overrideAttrs (superDeps: { - outputHash = depsOutputHash; - }); - installPhase = '' - cp "bazel-bin/plugins/${name}/${name}.jar" "$out" - ''; + bazelTargets = { + "//plugins/${name}" = "$out"; + }; + + extraBuildInstall = ""; + })).overrideAttrs (super: { postPatch = '' ${super.postPatch or ""} pushd "plugins/${name}" diff --git a/third_party/gerrit_plugins/code-owners/default.nix b/third_party/gerrit_plugins/code-owners/default.nix index 0dc3ef83ae7f..c849bfb52d41 100644 --- a/third_party/gerrit_plugins/code-owners/default.nix +++ b/third_party/gerrit_plugins/code-owners/default.nix @@ -5,11 +5,11 @@ let in buildGerritBazelPlugin rec { name = "code-owners"; - depsOutputHash = "sha256:0jv62cc1kpgsmwk119i9njmqn6w6k8frlbgcw87y8nfbpprmcf01"; + version = "7de40d8"; src = pkgs.fetchgit { url = "https://gerrit.googlesource.com/plugins/code-owners"; - rev = "e654ae5bda2085bce9a99942bec440e004a114f3"; - sha256 = "sha256:14d3x3iqskgw16pvyaa0swh252agj84p9pzlf24l8lgx9d7y4biz"; + rev = "7de40d8b30e55eb64316b6fc3d0d00da9caddade"; + hash = "sha256-0sLwUcG9RN1o9vZGW8ErwL7UgJapgYoo8XMGsWLO25Q="; }; patches = [ ./using-usernames.patch diff --git a/third_party/gerrit_plugins/oauth/default.nix b/third_party/gerrit_plugins/oauth/default.nix index e7626fa88c38..a949596b0b1f 100644 --- a/third_party/gerrit_plugins/oauth/default.nix +++ b/third_party/gerrit_plugins/oauth/default.nix @@ -5,15 +5,14 @@ let in buildGerritBazelPlugin rec { name = "oauth"; - depsOutputHash = "sha256:01z7rn8hnms3cp7mq27yk063lpy4pmqwpfrcc3cfl0r43k889zz3"; + version = "982316"; src = pkgs.fetchgit { url = "https://gerrit.googlesource.com/plugins/oauth"; - rev = "b27cf3ea820eec2ddd22d217fc839261692ccdb0"; - sha256 = "1m654ibgzprrhcl0wpzqrmq8drpgx6rzlw0ha16l1fi2zv5idkk2"; + rev = "98231604d60788bb43490f1a301d792817ac8008"; + hash = "sha256-AuVO1Yys8BYqGHZI/adszCUg0JM2v4Td4fe26LdOPLM="; }; - overlayPluginCmd = '' - chmod +w "$out" "$out/plugins/external_plugin_deps.bzl" - cp -R "${src}" "$out/plugins/${name}" + depsHash = "sha256-qEeKQPyt3uIS+BStrputBUnE3ft5c4oeazgsWd+lbN8="; + postOverlayPlugin = '' cp "${src}/external_plugin_deps.bzl" "$out/plugins/external_plugin_deps.bzl" ''; } |