merge(3p/git): Merge git subtree at v2.29.2 r/1890

This also bumps the stable nixpkgs to 20.09 as of 2020-11-21, because
there is some breakage in the git build related to the netrc
credentials helper which someone has taken care of in nixpkgs.

The stable channel is not used for anything other than git, so this
should be fine.

Change-Id: I3575a19dab09e1e9556cf8231d717de9890484fb

1 files changed, 175 insertions, 96 deletions

diff --git a/third_party/git/builtin/commit-graph.c b/third_party/git/builtin/commit-graph.c
index 38027b83d9..78fa08f43a 100644
--- a/third_party/git/builtin/commit-graph.c
+++ b/third_party/git/builtin/commit-graph.c
@@ -6,27 +6,28 @@
 #include "repository.h"
 #include "commit-graph.h"
 #include "object-store.h"
+#include "progress.h"
+#include "tag.h"
 
 static char const * const builtin_commit_graph_usage[] = {
-	N_("git commit-graph [--object-dir <objdir>]"),
-	N_("git commit-graph read [--object-dir <objdir>]"),
-	N_("git commit-graph verify [--object-dir <objdir>] [--shallow]"),
-	N_("git commit-graph write [--object-dir <objdir>] [--append|--split] [--reachable|--stdin-packs|--stdin-commits] <split options>"),
+	N_("git commit-graph verify [--object-dir <objdir>] [--shallow] [--[no-]progress]"),
+	N_("git commit-graph write [--object-dir <objdir>] [--append] "
+	   "[--split[=<strategy>]] [--reachable|--stdin-packs|--stdin-commits] "
+	   "[--changed-paths] [--[no-]max-new-filters <n>] [--[no-]progress] "
+	   "<split options>"),
 	NULL
 };
 
 static const char * const builtin_commit_graph_verify_usage[] = {
-	N_("git commit-graph verify [--object-dir <objdir>] [--shallow]"),
-	NULL
-};
-
-static const char * const builtin_commit_graph_read_usage[] = {
-	N_("git commit-graph read [--object-dir <objdir>]"),
+	N_("git commit-graph verify [--object-dir <objdir>] [--shallow] [--[no-]progress]"),
 	NULL
 };
 
 static const char * const builtin_commit_graph_write_usage[] = {
-	N_("git commit-graph write [--object-dir <objdir>] [--append|--split] [--reachable|--stdin-packs|--stdin-commits] <split options>"),
+	N_("git commit-graph write [--object-dir <objdir>] [--append] "
+	   "[--split[=<strategy>]] [--reachable|--stdin-packs|--stdin-commits] "
+	   "[--changed-paths] [--[no-]max-new-filters <n>] [--[no-]progress] "
+	   "<split options>"),
 	NULL
 };
 
@@ -38,11 +39,36 @@ static struct opts_commit_graph {
 	int append;
 	int split;
 	int shallow;
+	int progress;
+	int enable_changed_paths;
 } opts;
 
+static struct object_directory *find_odb(struct repository *r,
+					 const char *obj_dir)
+{
+	struct object_directory *odb;
+	char *obj_dir_real = real_pathdup(obj_dir, 1);
+	struct strbuf odb_path_real = STRBUF_INIT;
+
+	prepare_alt_odb(r);
+	for (odb = r->objects->odb; odb; odb = odb->next) {
+		strbuf_realpath(&odb_path_real, odb->path, 1);
+		if (!strcmp(obj_dir_real, odb_path_real.buf))
+			break;
+	}
+
+	free(obj_dir_real);
+	strbuf_release(&odb_path_real);
+
+	if (!odb)
+		die(_("could not find object directory matching %s"), obj_dir);
+	return odb;
+}
+
 static int graph_verify(int argc, const char **argv)
 {
 	struct commit_graph *graph = NULL;
+	struct object_directory *odb = NULL;
 	char *graph_name;
 	int open_ok;
 	int fd;
@@ -55,9 +81,13 @@ static int graph_verify(int argc, const char **argv)
 			   N_("The object directory to store the graph")),
 		OPT_BOOL(0, "shallow", &opts.shallow,
 			 N_("if the commit-graph is split, only verify the tip file")),
+		OPT_BOOL(0, "progress", &opts.progress, N_("force progress reporting")),
 		OPT_END(),
 	};
 
+	trace2_cmd_mode("verify");
+
+	opts.progress = isatty(2);
 	argc = parse_options(argc, argv, NULL,
 			     builtin_commit_graph_verify_options,
 			     builtin_commit_graph_verify_usage, 0);
@@ -66,8 +96,11 @@ static int graph_verify(int argc, const char **argv)
 		opts.obj_dir = get_object_directory();
 	if (opts.shallow)
 		flags |= COMMIT_GRAPH_VERIFY_SHALLOW;
+	if (opts.progress)
+		flags |= COMMIT_GRAPH_WRITE_PROGRESS;
 
-	graph_name = get_commit_graph_filename(opts.obj_dir);
+	odb = find_odb(the_repository, opts.obj_dir);
+	graph_name = get_commit_graph_filename(odb);
 	open_ok = open_commit_graph(graph_name, &fd, &st);
 	if (!open_ok && errno != ENOENT)
 		die_errno(_("Could not open commit-graph '%s'"), graph_name);
@@ -75,9 +108,9 @@ static int graph_verify(int argc, const char **argv)
 	FREE_AND_NULL(graph_name);
 
 	if (open_ok)
-		graph = load_commit_graph_one_fd_st(fd, &st);
-	 else
-		graph = read_commit_graph_one(the_repository, opts.obj_dir);
+		graph = load_commit_graph_one_fd_st(the_repository, fd, &st, odb);
+	else
+		graph = read_commit_graph_one(the_repository, odb);
 
 	/* Return failure if open_ok predicted success */
 	if (!graph)
@@ -87,74 +120,90 @@ static int graph_verify(int argc, const char **argv)
 	return verify_commit_graph(the_repository, graph, flags);
 }
 
-static int graph_read(int argc, const char **argv)
+extern int read_replace_refs;
+static struct commit_graph_opts write_opts;
+
+static int write_option_parse_split(const struct option *opt, const char *arg,
+				    int unset)
 {
-	struct commit_graph *graph = NULL;
-	char *graph_name;
-	int open_ok;
-	int fd;
-	struct stat st;
+	enum commit_graph_split_flags *flags = opt->value;
 
-	static struct option builtin_commit_graph_read_options[] = {
-		OPT_STRING(0, "object-dir", &opts.obj_dir,
-			N_("dir"),
-			N_("The object directory to store the graph")),
-		OPT_END(),
-	};
+	BUG_ON_OPT_NEG(unset);
 
-	argc = parse_options(argc, argv, NULL,
-			     builtin_commit_graph_read_options,
-			     builtin_commit_graph_read_usage, 0);
+	opts.split = 1;
+	if (!arg)
+		return 0;
 
-	if (!opts.obj_dir)
-		opts.obj_dir = get_object_directory();
+	if (!strcmp(arg, "no-merge"))
+		*flags = COMMIT_GRAPH_SPLIT_MERGE_PROHIBITED;
+	else if (!strcmp(arg, "replace"))
+		*flags = COMMIT_GRAPH_SPLIT_REPLACE;
+	else
+		die(_("unrecognized --split argument, %s"), arg);
 
-	graph_name = get_commit_graph_filename(opts.obj_dir);
+	return 0;
+}
 
-	open_ok = open_commit_graph(graph_name, &fd, &st);
-	if (!open_ok)
-		die_errno(_("Could not open commit-graph '%s'"), graph_name);
+static int read_one_commit(struct oidset *commits, struct progress *progress,
+			   const char *hash)
+{
+	struct object *result;
+	struct object_id oid;
+	const char *end;
 
-	graph = load_commit_graph_one_fd_st(fd, &st);
-	if (!graph)
-		return 1;
+	if (parse_oid_hex(hash, &oid, &end))
+		return error(_("unexpected non-hex object ID: %s"), hash);
 
-	FREE_AND_NULL(graph_name);
+	result = deref_tag(the_repository, parse_object(the_repository, &oid),
+			   NULL, 0);
+	if (!result)
+		return error(_("invalid object: %s"), hash);
+	else if (object_as_type(result, OBJ_COMMIT, 1))
+		oidset_insert(commits, &result->oid);
 
-	printf("header: %08x %d %d %d %d\n",
-		ntohl(*(uint32_t*)graph->data),
-		*(unsigned char*)(graph->data + 4),
-		*(unsigned char*)(graph->data + 5),
-		*(unsigned char*)(graph->data + 6),
-		*(unsigned char*)(graph->data + 7));
-	printf("num_commits: %u\n", graph->num_commits);
-	printf("chunks:");
-
-	if (graph->chunk_oid_fanout)
-		printf(" oid_fanout");
-	if (graph->chunk_oid_lookup)
-		printf(" oid_lookup");
-	if (graph->chunk_commit_data)
-		printf(" commit_metadata");
-	if (graph->chunk_extra_edges)
-		printf(" extra_edges");
-	printf("\n");
+	display_progress(progress, oidset_size(commits));
 
-	UNLEAK(graph);
+	return 0;
+}
 
+static int write_option_max_new_filters(const struct option *opt,
+					const char *arg,
+					int unset)
+{
+	int *to = opt->value;
+	if (unset)
+		*to = -1;
+	else {
+		const char *s;
+		*to = strtol(arg, (char **)&s, 10);
+		if (*s)
+			return error(_("%s expects a numerical value"),
+				     optname(opt, opt->flags));
+	}
 	return 0;
 }
 
-extern int read_replace_refs;
-static struct split_commit_graph_opts split_opts;
+static int git_commit_graph_write_config(const char *var, const char *value,
+					 void *cb)
+{
+	if (!strcmp(var, "commitgraph.maxnewfilters"))
+		write_opts.max_new_filters = git_config_int(var, value);
+	/*
+	 * No need to fall-back to 'git_default_config', since this was already
+	 * called in 'cmd_commit_graph()'.
+	 */
+	return 0;
+}
 
 static int graph_write(int argc, const char **argv)
 {
-	struct string_list *pack_indexes = NULL;
-	struct string_list *commit_hex = NULL;
-	struct string_list lines;
+	struct string_list pack_indexes = STRING_LIST_INIT_NODUP;
+	struct strbuf buf = STRBUF_INIT;
+	struct oidset commits = OIDSET_INIT;
+	struct object_directory *odb = NULL;
 	int result = 0;
-	unsigned int flags = COMMIT_GRAPH_PROGRESS;
+	enum commit_graph_write_flags flags = 0;
+	struct progress *progress = NULL;
 
 	static struct option builtin_commit_graph_write_options[] = {
 		OPT_STRING(0, "object-dir", &opts.obj_dir,
@@ -168,20 +217,35 @@ static int graph_write(int argc, const char **argv)
 			N_("start walk at commits listed by stdin")),
 		OPT_BOOL(0, "append", &opts.append,
 			N_("include all commits already in the commit-graph file")),
-		OPT_BOOL(0, "split", &opts.split,
-			N_("allow writing an incremental commit-graph file")),
-		OPT_INTEGER(0, "max-commits", &split_opts.max_commits,
+		OPT_BOOL(0, "changed-paths", &opts.enable_changed_paths,
+			N_("enable computation for changed paths")),
+		OPT_BOOL(0, "progress", &opts.progress, N_("force progress reporting")),
+		OPT_CALLBACK_F(0, "split", &write_opts.split_flags, NULL,
+			N_("allow writing an incremental commit-graph file"),
+			PARSE_OPT_OPTARG | PARSE_OPT_NONEG,
+			write_option_parse_split),
+		OPT_INTEGER(0, "max-commits", &write_opts.max_commits,
 			N_("maximum number of commits in a non-base split commit-graph")),
-		OPT_INTEGER(0, "size-multiple", &split_opts.size_multiple,
+		OPT_INTEGER(0, "size-multiple", &write_opts.size_multiple,
 			N_("maximum ratio between two levels of a split commit-graph")),
-		OPT_EXPIRY_DATE(0, "expire-time", &split_opts.expire_time,
-			N_("maximum number of commits in a non-base split commit-graph")),
+		OPT_EXPIRY_DATE(0, "expire-time", &write_opts.expire_time,
+			N_("only expire files older than a given date-time")),
+		OPT_CALLBACK_F(0, "max-new-filters", &write_opts.max_new_filters,
+			NULL, N_("maximum number of changed-path Bloom filters to compute"),
+			0, write_option_max_new_filters),
 		OPT_END(),
 	};
 
-	split_opts.size_multiple = 2;
-	split_opts.max_commits = 0;
-	split_opts.expire_time = 0;
+	opts.progress = isatty(2);
+	opts.enable_changed_paths = -1;
+	write_opts.size_multiple = 2;
+	write_opts.max_commits = 0;
+	write_opts.expire_time = 0;
+	write_opts.max_new_filters = -1;
+
+	trace2_cmd_mode("write");
+
+	git_config(git_commit_graph_write_config, &opts);
 
 	argc = parse_options(argc, argv, NULL,
 			     builtin_commit_graph_write_options,
@@ -192,41 +256,56 @@ static int graph_write(int argc, const char **argv)
 	if (!opts.obj_dir)
 		opts.obj_dir = get_object_directory();
 	if (opts.append)
-		flags |= COMMIT_GRAPH_APPEND;
+		flags |= COMMIT_GRAPH_WRITE_APPEND;
 	if (opts.split)
-		flags |= COMMIT_GRAPH_SPLIT;
+		flags |= COMMIT_GRAPH_WRITE_SPLIT;
+	if (opts.progress)
+		flags |= COMMIT_GRAPH_WRITE_PROGRESS;
+	if (!opts.enable_changed_paths)
+		flags |= COMMIT_GRAPH_NO_WRITE_BLOOM_FILTERS;
+	if (opts.enable_changed_paths == 1 ||
+	    git_env_bool(GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS, 0))
+		flags |= COMMIT_GRAPH_WRITE_BLOOM_FILTERS;
 
 	read_replace_refs = 0;
+	odb = find_odb(the_repository, opts.obj_dir);
 
 	if (opts.reachable) {
-		if (write_commit_graph_reachable(opts.obj_dir, flags, &split_opts))
+		if (write_commit_graph_reachable(odb, flags, &write_opts))
 			return 1;
 		return 0;
 	}
 
-	string_list_init(&lines, 0);
-	if (opts.stdin_packs || opts.stdin_commits) {
-		struct strbuf buf = STRBUF_INIT;
-
+	if (opts.stdin_packs) {
 		while (strbuf_getline(&buf, stdin) != EOF)
-			string_list_append(&lines, strbuf_detach(&buf, NULL));
-
-		if (opts.stdin_packs)
-			pack_indexes = &lines;
-		if (opts.stdin_commits)
-			commit_hex = &lines;
-
-		UNLEAK(buf);
+			string_list_append(&pack_indexes,
+					   strbuf_detach(&buf, NULL));
+	} else if (opts.stdin_commits) {
+		oidset_init(&commits, 0);
+		if (opts.progress)
+			progress = start_delayed_progress(
+				_("Collecting commits from input"), 0);
+
+		while (strbuf_getline(&buf, stdin) != EOF) {
+			if (read_one_commit(&commits, progress, buf.buf)) {
+				result = 1;
+				goto cleanup;
+			}
+		}
+
+		stop_progress(&progress);
 	}
 
-	if (write_commit_graph(opts.obj_dir,
-			       pack_indexes,
-			       commit_hex,
+	if (write_commit_graph(odb,
+			       opts.stdin_packs ? &pack_indexes : NULL,
+			       opts.stdin_commits ? &commits : NULL,
 			       flags,
-			       &split_opts))
+			       &write_opts))
 		result = 1;
 
-	UNLEAK(lines);
+cleanup:
+	string_list_clear(&pack_indexes, 0);
+	strbuf_release(&buf);
 	return result;
 }
 
@@ -249,9 +328,9 @@ int cmd_commit_graph(int argc, const char **argv, const char *prefix)
 			     builtin_commit_graph_usage,
 			     PARSE_OPT_STOP_AT_NON_OPTION);
 
+	save_commit_buffer = 0;
+
 	if (argc > 0) {
-		if (!strcmp(argv[0], "read"))
-			return graph_read(argc, argv);
 		if (!strcmp(argv[0], "verify"))
 			return graph_verify(argc, argv);
 		if (!strcmp(argv[0], "write"))