From f4609b896fac842433bd495c166d5987852a6a73 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sat, 21 Nov 2020 19:20:35 +0100 Subject: merge(3p/git): Merge git subtree at v2.29.2 This also bumps the stable nixpkgs to 20.09 as of 2020-11-21, because there is some breakage in the git build related to the netrc credentials helper which someone has taken care of in nixpkgs. The stable channel is not used for anything other than git, so this should be fine. Change-Id: I3575a19dab09e1e9556cf8231d717de9890484fb --- third_party/git/t/perf/README | 9 +++++ third_party/git/t/perf/aggregate.perl | 21 +++++----- third_party/git/t/perf/bisect_regression | 2 +- third_party/git/t/perf/p1400-update-ref.sh | 39 ++++++++++++++++++ third_party/git/t/perf/p5302-pack-index.sh | 47 +++++++++++----------- third_party/git/t/perf/p5303-many-packs.sh | 23 +++++++++++ third_party/git/t/perf/p5310-pack-bitmaps.sh | 36 +++++++++++++++-- third_party/git/t/perf/p5600-clone-reference.sh | 27 ------------- third_party/git/t/perf/p5601-clone-reference.sh | 27 +++++++++++++ third_party/git/t/perf/p9300-fast-import-export.sh | 23 +++++++++++ third_party/git/t/perf/perf-lib.sh | 6 ++- 11 files changed, 192 insertions(+), 68 deletions(-) create mode 100755 third_party/git/t/perf/p1400-update-ref.sh delete mode 100755 third_party/git/t/perf/p5600-clone-reference.sh create mode 100755 third_party/git/t/perf/p5601-clone-reference.sh create mode 100755 third_party/git/t/perf/p9300-fast-import-export.sh (limited to 'third_party/git/t/perf') diff --git a/third_party/git/t/perf/README b/third_party/git/t/perf/README index c7b70e2d28ba..bd649afa9783 100644 --- a/third_party/git/t/perf/README +++ b/third_party/git/t/perf/README @@ -84,6 +84,15 @@ You can set the following variables (also in your config.mak): probably be about linux.git size for optimal results. Both default to the git.git you are running from. + GIT_PERF_EXTRA + Boolean to enable additional tests. Most test scripts are + written to detect regressions between two versions of Git, and + the output will compare timings for individual tests between + those versions. Some scripts have additional tests which are not + run by default, that show patterns within a single version of + Git (e.g., performance of index-pack as the number of threads + changes). These can be enabled with GIT_PERF_EXTRA. + You can also pass the options taken by ordinary git tests; the most useful one is: diff --git a/third_party/git/t/perf/aggregate.perl b/third_party/git/t/perf/aggregate.perl index 66554d216122..14e4cda287d7 100755 --- a/third_party/git/t/perf/aggregate.perl +++ b/third_party/git/t/perf/aggregate.perl @@ -4,7 +4,6 @@ use lib '../../perl/build/lib'; use strict; use warnings; use Getopt::Long; -use Git; use Cwd qw(realpath); sub get_times { @@ -85,6 +84,11 @@ sub format_size { return $out; } +sub sane_backticks { + open(my $fh, '-|', @_); + return <$fh>; +} + my (@dirs, %dirnames, %dirabbrevs, %prefixes, @tests, $codespeed, $sortby, $subsection, $reponame); @@ -102,7 +106,8 @@ while (scalar @ARGV) { my $prefix = ''; last if -f $arg or $arg eq "--"; if (! -d $arg) { - my $rev = Git::command_oneline(qw(rev-parse --verify), $arg); + my $rev = sane_backticks(qw(git rev-parse --verify), $arg); + chomp $rev; $dir = "build/".$rev; } elsif ($arg eq '.') { $dir = '.'; @@ -219,13 +224,7 @@ sub print_default_results { for my $i (0..$#dirs) { my $d = $dirs[$i]; my $base = "$resultsdir/$prefixes{$d}$t"; - $times{$prefixes{$d}.$t} = []; - foreach my $type (qw(times size)) { - if (-e "$base.$type") { - $times{$prefixes{$d}.$t} = [get_times("$base.$type")]; - last; - } - } + $times{$prefixes{$d}.$t} = [get_times("$base.result")]; my ($r,$u,$s) = @{$times{$prefixes{$d}.$t}}; my $w = length format_times($r,$u,$s,$firstr); $colwidth[$i] = $w if $w > $colwidth[$i]; @@ -267,7 +266,7 @@ sub print_sorted_results { my ($prevr, $prevu, $prevs, $prevrev); for my $i (0..$#dirs) { my $d = $dirs[$i]; - my ($r, $u, $s) = get_times("$resultsdir/$prefixes{$d}$t.times"); + my ($r, $u, $s) = get_times("$resultsdir/$prefixes{$d}$t.result"); if ($i > 0 and defined $r and defined $prevr and $prevr > 0) { my $percent = 100.0 * ($r - $prevr) / $prevr; push @evolutions, { "percent" => $percent, @@ -327,7 +326,7 @@ sub print_codespeed_results { my $commitid = $prefixes{$d}; $commitid =~ s/^build_//; $commitid =~ s/\.$//; - my ($result_value, $u, $s) = get_times("$resultsdir/$prefixes{$d}$t.times"); + my ($result_value, $u, $s) = get_times("$resultsdir/$prefixes{$d}$t.result"); my %vals = ( "commitid" => $commitid, diff --git a/third_party/git/t/perf/bisect_regression b/third_party/git/t/perf/bisect_regression index a94d9955d012..ce47e1662a92 100755 --- a/third_party/git/t/perf/bisect_regression +++ b/third_party/git/t/perf/bisect_regression @@ -51,7 +51,7 @@ oldtime=$(echo "$oldtime" | sed -e 's/^\([0-9]\+\.[0-9]\+\).*$/\1/') newtime=$(echo "$newtime" | sed -e 's/^\([0-9]\+\.[0-9]\+\).*$/\1/') test $(echo "$newtime" "$oldtime" | awk '{ print ($1 > $2) }') = 1 || - die "New time '$newtime' shoud be greater than old time '$oldtime'" + die "New time '$newtime' should be greater than old time '$oldtime'" tmpdir=$(mktemp -d -t bisect_regression_XXXXXX) || die "Failed to create temp directory" echo "$oldtime" >"$tmpdir/oldtime" || die "Failed to write to '$tmpdir/oldtime'" diff --git a/third_party/git/t/perf/p1400-update-ref.sh b/third_party/git/t/perf/p1400-update-ref.sh new file mode 100755 index 000000000000..ce5ac3ed85e1 --- /dev/null +++ b/third_party/git/t/perf/p1400-update-ref.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +test_description="Tests performance of update-ref" + +. ./perf-lib.sh + +test_perf_fresh_repo + +test_expect_success "setup" ' + git init --bare target-repo.git && + test_commit PRE && + test_commit POST && + printf "create refs/heads/%d PRE\n" $(test_seq 1000) >create && + printf "update refs/heads/%d POST PRE\n" $(test_seq 1000) >update && + printf "delete refs/heads/%d POST\n" $(test_seq 1000) >delete && + git update-ref --stdin /dev/null ' + test_perf "abbrev-commit ($nr_packs)" ' + git rev-list --abbrev-commit HEAD >/dev/null + ' + # This simulates the interesting part of the repack, which is the # actual pack generation, without smudging the on-disk setup # between trials. test_perf "repack ($nr_packs)" ' + GIT_TEST_FULL_IN_PACK_ARRAY=1 \ git pack-objects --keep-true-parents \ --honor-pack-keep --non-empty --all \ --reflog --indexed-objects --delta-base-offset \ @@ -84,4 +89,22 @@ do ' done +# Measure pack loading with 10,000 packs. +test_expect_success 'generate lots of packs' ' + for i in $(test_seq 10000); do + echo "blob" + echo "data </dev/null ' -test_perf 'pack to file' ' - git pack-objects --all pack1 /dev/null -' - test_perf 'pack to file (bitmap)' ' git pack-objects --use-bitmap-index --all pack1b /dev/null ' +test_perf 'rev-list (commits)' ' + git rev-list --all --use-bitmap-index >/dev/null +' + +test_perf 'rev-list (objects)' ' + git rev-list --all --use-bitmap-index --objects >/dev/null +' + +test_perf 'rev-list count with blob:none' ' + git rev-list --use-bitmap-index --count --objects --all \ + --filter=blob:none >/dev/null +' + +test_perf 'rev-list count with blob:limit=1k' ' + git rev-list --use-bitmap-index --count --objects --all \ + --filter=blob:limit=1k >/dev/null +' + +test_perf 'rev-list count with tree:0' ' + git rev-list --use-bitmap-index --count --objects --all \ + --filter=tree:0 >/dev/null +' + +test_perf 'simulated partial clone' ' + git pack-objects --stdout --all --filter=blob:none /dev/null +' + test_expect_success 'create partial bitmap state' ' # pick a commit to represent the repo tip in the past cutoff=$(git rev-list HEAD~100 -1) && @@ -68,4 +91,9 @@ test_perf 'pack to file (partial bitmap)' ' git pack-objects --use-bitmap-index --all pack2b /dev/null ' +test_perf 'rev-list with tree filter (partial bitmap)' ' + git rev-list --use-bitmap-index --count --objects --all \ + --filter=tree:0 >/dev/null +' + test_done diff --git a/third_party/git/t/perf/p5600-clone-reference.sh b/third_party/git/t/perf/p5600-clone-reference.sh deleted file mode 100755 index 68fed663479d..000000000000 --- a/third_party/git/t/perf/p5600-clone-reference.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -test_description='speed of clone --reference' -. ./perf-lib.sh - -test_perf_default_repo - -test_expect_success 'create shareable repository' ' - git clone --bare . shared.git -' - -test_expect_success 'advance base repository' ' - # Do not use test_commit here; its test_tick will - # use some ancient hard-coded date. The resulting clock - # skew will cause pack-objects to traverse in a very - # sub-optimal order, skewing the results. - echo content >new-file-that-does-not-exist && - git add new-file-that-does-not-exist && - git commit -m "new commit" -' - -test_perf 'clone --reference' ' - rm -rf dst.git && - git clone --no-local --bare --reference shared.git . dst.git -' - -test_done diff --git a/third_party/git/t/perf/p5601-clone-reference.sh b/third_party/git/t/perf/p5601-clone-reference.sh new file mode 100755 index 000000000000..68fed663479d --- /dev/null +++ b/third_party/git/t/perf/p5601-clone-reference.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +test_description='speed of clone --reference' +. ./perf-lib.sh + +test_perf_default_repo + +test_expect_success 'create shareable repository' ' + git clone --bare . shared.git +' + +test_expect_success 'advance base repository' ' + # Do not use test_commit here; its test_tick will + # use some ancient hard-coded date. The resulting clock + # skew will cause pack-objects to traverse in a very + # sub-optimal order, skewing the results. + echo content >new-file-that-does-not-exist && + git add new-file-that-does-not-exist && + git commit -m "new commit" +' + +test_perf 'clone --reference' ' + rm -rf dst.git && + git clone --no-local --bare --reference shared.git . dst.git +' + +test_done diff --git a/third_party/git/t/perf/p9300-fast-import-export.sh b/third_party/git/t/perf/p9300-fast-import-export.sh new file mode 100755 index 000000000000..586161e9adcd --- /dev/null +++ b/third_party/git/t/perf/p9300-fast-import-export.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +test_description='test fast-import and fast-export performance' +. ./perf-lib.sh + +test_perf_default_repo + +# Use --no-data here to produce a vastly smaller export file. +# This is much cheaper to work with but should still exercise +# fast-import pretty well (we'll still process all commits and +# trees, which account for 60% or more of objects in most repos). +# +# Use --reencode to avoid the default of aborting on non-utf8 commits, +# which lets this test run against a wider variety of sample repos. +test_perf 'export (no-blobs)' ' + git fast-export --reencode=yes --no-data HEAD >export +' + +test_perf 'import (no-blobs)' ' + git fast-import --force "$base".times + "$TEST_DIRECTORY"/perf/min_time.perl test_time.* >"$base".result } test_perf () { @@ -223,7 +223,7 @@ test_perf () { test_size_ () { say >&3 "running: $2" - if test_eval_ "$2" 3>"$base".size; then + if test_eval_ "$2" 3>"$base".result; then test_ok_ "$1" else test_failure_ "$@" @@ -245,3 +245,5 @@ test_at_end_hook_ () { test_export () { export "$@" } + +test_lazy_prereq PERF_EXTRA 'test_bool_env GIT_PERF_EXTRA false' -- cgit 1.4.1