From 93ba78d6f4632ef1c5228965e3edc8c0faf88c1e Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Tue, 26 May 2020 00:06:52 +0100 Subject: revert(3p/git): Revert merge of git upstream at v2.26.2 This causes cgit to serve error pages, which is undesirable. This reverts commit 5229c9b232de5bfa959ad6ebbb4c8192ac513352, reversing changes made to f2b211131f2347342dde63975b09cf603149f1a3. --- third_party/git/packfile.c | 198 ++++++++++++++++++++++----------------------- 1 file changed, 99 insertions(+), 99 deletions(-) (limited to 'third_party/git/packfile.c') diff --git a/third_party/git/packfile.c b/third_party/git/packfile.c index f4e752996d..fc43a6c52c 100644 --- a/third_party/git/packfile.c +++ b/third_party/git/packfile.c @@ -6,6 +6,7 @@ #include "mergesort.h" #include "packfile.h" #include "delta.h" +#include "list.h" #include "streaming.h" #include "sha1-lookup.h" #include "commit.h" @@ -16,15 +17,14 @@ #include "object-store.h" #include "midx.h" #include "commit-graph.h" -#include "promisor-remote.h" char *odb_pack_name(struct strbuf *buf, - const unsigned char *hash, + const unsigned char *sha1, const char *ext) { strbuf_reset(buf); strbuf_addf(buf, "%s/pack/pack-%s.%s", get_object_directory(), - hash_to_hex(hash), ext); + sha1_to_hex(sha1), ext); return buf->buf; } @@ -287,6 +287,13 @@ static int unuse_one_window(struct packed_git *current) return 0; } +void release_pack_memory(size_t need) +{ + size_t cur = pack_mapped; + while (need >= (cur - pack_mapped) && unuse_one_window(NULL)) + ; /* nothing */ +} + void close_pack_windows(struct packed_git *p) { while (p->windows) { @@ -510,6 +517,7 @@ static int open_packed_git_1(struct packed_git *p) struct pack_header hdr; unsigned char hash[GIT_MAX_RAWSZ]; unsigned char *idx_hash; + long fd_flag; ssize_t read_result; const unsigned hashsz = the_hash_algo->rawsz; @@ -553,6 +561,16 @@ static int open_packed_git_1(struct packed_git *p) } else if (p->pack_size != st.st_size) return error("packfile %s size changed", p->pack_name); + /* We leave these file descriptors open with sliding mmap; + * there is no point keeping them open across exec(), though. + */ + fd_flag = fcntl(p->pack_fd, F_GETFD, 0); + if (fd_flag < 0) + return error("cannot determine file descriptor flags"); + fd_flag |= FD_CLOEXEC; + if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1) + return error("cannot set FD_CLOEXEC"); + /* Verify we recognize this pack file format. */ read_result = read_in_full(p->pack_fd, &hdr, sizeof(hdr)); if (read_result < 0) @@ -576,8 +594,9 @@ static int open_packed_git_1(struct packed_git *p) " while index indicates %"PRIu32" objects", p->pack_name, ntohl(hdr.hdr_entries), p->num_objects); - read_result = pread_in_full(p->pack_fd, hash, hashsz, - p->pack_size - hashsz); + if (lseek(p->pack_fd, p->pack_size - hashsz, SEEK_SET) == -1) + return error("end of packfile %s is unavailable", p->pack_name); + read_result = read_in_full(p->pack_fd, hash, hashsz); if (read_result < 0) return error_errno("error reading from %s", p->pack_name); if (read_result != hashsz) @@ -691,12 +710,23 @@ void unuse_pack(struct pack_window **w_cursor) } } +static void try_to_free_pack_memory(size_t size) +{ + release_pack_memory(size); +} + struct packed_git *add_packed_git(const char *path, size_t path_len, int local) { + static int have_set_try_to_free_routine; struct stat st; size_t alloc; struct packed_git *p; + if (!have_set_try_to_free_routine) { + have_set_try_to_free_routine = 1; + set_try_to_free_routine(try_to_free_pack_memory); + } + /* * Make sure a corresponding .pack file exists and that * the index looks sane. @@ -745,9 +775,6 @@ void install_packed_git(struct repository *r, struct packed_git *pack) pack->next = r->objects->packed_git; r->objects->packed_git = pack; - - hashmap_entry_init(&pack->packmap_ent, strhash(pack->pack_name)); - hashmap_add(&r->objects->pack_map, &pack->packmap_ent); } void (*report_garbage)(unsigned seen_bits, const char *path); @@ -847,18 +874,20 @@ static void prepare_pack(const char *full_name, size_t full_name_len, if (strip_suffix_mem(full_name, &base_len, ".idx") && !(data->m && midx_contains_pack(data->m, file_name))) { - struct hashmap_entry hent; - char *pack_name = xstrfmt("%.*s.pack", (int)base_len, full_name); - unsigned int hash = strhash(pack_name); - hashmap_entry_init(&hent, hash); - /* Don't reopen a pack we already have. */ - if (!hashmap_get(&data->r->objects->pack_map, &hent, pack_name)) { + for (p = data->r->objects->packed_git; p; p = p->next) { + size_t len; + if (strip_suffix(p->pack_name, ".pack", &len) && + len == base_len && + !memcmp(p->pack_name, full_name, len)) + break; + } + + if (!p) { p = add_packed_git(full_name, full_name_len, data->local); if (p) install_packed_git(data->r, p); } - free(pack_name); } if (!report_garbage) @@ -1004,14 +1033,12 @@ void reprepare_packed_git(struct repository *r) { struct object_directory *odb; - obj_read_lock(); for (odb = r->objects->odb; odb; odb = odb->next) odb_clear_loose_cache(odb); r->objects->approximate_object_count_valid = 0; r->objects->packed_git_initialized = 0; prepare_packed_git(r); - obj_read_unlock(); } struct packed_git *get_packed_git(struct repository *r) @@ -1088,23 +1115,7 @@ unsigned long get_size_from_delta(struct packed_git *p, do { in = use_pack(p, w_curs, curpos, &stream.avail_in); stream.next_in = in; - /* - * Note: the window section returned by use_pack() must be - * available throughout git_inflate()'s unlocked execution. To - * ensure no other thread will modify the window in the - * meantime, we rely on the packed_window.inuse_cnt. This - * counter is incremented before window reading and checked - * before window disposal. - * - * Other worrying sections could be the call to close_pack_fd(), - * which can close packs even with in-use windows, and to - * reprepare_packed_git(). Regarding the former, mmap doc says: - * "closing the file descriptor does not unmap the region". And - * for the latter, it won't re-open already available packs. - */ - obj_read_unlock(); st = git_inflate(&stream, Z_FINISH); - obj_read_lock(); curpos += stream.next_in - in; } while ((st == Z_OK || st == Z_BUF_ERROR) && stream.total_out < sizeof(delta_head)); @@ -1180,11 +1191,11 @@ const struct packed_git *has_packed_and_bad(struct repository *r, return NULL; } -off_t get_delta_base(struct packed_git *p, - struct pack_window **w_curs, - off_t *curpos, - enum object_type type, - off_t delta_obj_offset) +static off_t get_delta_base(struct packed_git *p, + struct pack_window **w_curs, + off_t *curpos, + enum object_type type, + off_t delta_obj_offset) { unsigned char *base_info = use_pack(p, w_curs, *curpos, NULL); off_t base_offset; @@ -1225,32 +1236,30 @@ off_t get_delta_base(struct packed_git *p, * the final object lookup), but more expensive for OFS deltas (we * have to load the revidx to convert the offset back into a sha1). */ -static int get_delta_base_oid(struct packed_git *p, - struct pack_window **w_curs, - off_t curpos, - struct object_id *oid, - enum object_type type, - off_t delta_obj_offset) +static const unsigned char *get_delta_base_sha1(struct packed_git *p, + struct pack_window **w_curs, + off_t curpos, + enum object_type type, + off_t delta_obj_offset) { if (type == OBJ_REF_DELTA) { unsigned char *base = use_pack(p, w_curs, curpos, NULL); - oidread(oid, base); - return 0; + return base; } else if (type == OBJ_OFS_DELTA) { struct revindex_entry *revidx; off_t base_offset = get_delta_base(p, w_curs, &curpos, type, delta_obj_offset); if (!base_offset) - return -1; + return NULL; revidx = find_pack_revindex(p, base_offset); if (!revidx) - return -1; + return NULL; - return nth_packed_object_id(oid, p, revidx->nr); + return nth_packed_object_sha1(p, revidx->nr); } else - return -1; + return NULL; } static int retry_bad_packed_offset(struct repository *r, @@ -1263,7 +1272,7 @@ static int retry_bad_packed_offset(struct repository *r, revidx = find_pack_revindex(p, obj_offset); if (!revidx) return OBJ_BAD; - nth_packed_object_id(&oid, p, revidx->nr); + nth_packed_object_oid(&oid, p, revidx->nr); mark_bad_packed_object(p, oid.hash); type = oid_object_info(r, &oid, NULL); if (type <= OBJ_NONE) @@ -1352,7 +1361,7 @@ struct delta_base_cache_key { }; struct delta_base_cache_entry { - struct hashmap_entry ent; + struct hashmap hash; struct delta_base_cache_key key; struct list_head lru; void *data; @@ -1372,7 +1381,7 @@ static unsigned int pack_entry_hash(struct packed_git *p, off_t base_offset) static struct delta_base_cache_entry * get_delta_base_cache_entry(struct packed_git *p, off_t base_offset) { - struct hashmap_entry entry, *e; + struct hashmap_entry entry; struct delta_base_cache_key key; if (!delta_base_cache.cmpfn) @@ -1381,8 +1390,7 @@ get_delta_base_cache_entry(struct packed_git *p, off_t base_offset) hashmap_entry_init(&entry, pack_entry_hash(p, base_offset)); key.p = p; key.base_offset = base_offset; - e = hashmap_get(&delta_base_cache, &entry, &key); - return e ? container_of(e, struct delta_base_cache_entry, ent) : NULL; + return hashmap_get(&delta_base_cache, &entry, &key); } static int delta_base_cache_key_eq(const struct delta_base_cache_key *a, @@ -1392,16 +1400,11 @@ static int delta_base_cache_key_eq(const struct delta_base_cache_key *a, } static int delta_base_cache_hash_cmp(const void *unused_cmp_data, - const struct hashmap_entry *va, - const struct hashmap_entry *vb, + const void *va, const void *vb, const void *vkey) { - const struct delta_base_cache_entry *a, *b; + const struct delta_base_cache_entry *a = va, *b = vb; const struct delta_base_cache_key *key = vkey; - - a = container_of(va, const struct delta_base_cache_entry, ent); - b = container_of(vb, const struct delta_base_cache_entry, ent); - if (key) return !delta_base_cache_key_eq(&a->key, key); else @@ -1420,7 +1423,7 @@ static int in_delta_base_cache(struct packed_git *p, off_t base_offset) */ static void detach_delta_base_cache_entry(struct delta_base_cache_entry *ent) { - hashmap_remove(&delta_base_cache, &ent->ent, &ent->key); + hashmap_remove(&delta_base_cache, ent, &ent->key); list_del(&ent->lru); delta_base_cached -= ent->size; free(ent); @@ -1465,14 +1468,6 @@ static void add_delta_base_cache(struct packed_git *p, off_t base_offset, struct delta_base_cache_entry *ent = xmalloc(sizeof(*ent)); struct list_head *lru, *tmp; - /* - * Check required to avoid redundant entries when more than one thread - * is unpacking the same object, in unpack_entry() (since its phases I - * and III might run concurrently across multiple threads). - */ - if (in_delta_base_cache(p, base_offset)) - return; - delta_base_cached += base_size; list_for_each_safe(lru, tmp, &delta_base_cache_lru) { @@ -1492,8 +1487,8 @@ static void add_delta_base_cache(struct packed_git *p, off_t base_offset, if (!delta_base_cache.cmpfn) hashmap_init(&delta_base_cache, delta_base_cache_hash_cmp, NULL, 0); - hashmap_entry_init(&ent->ent, pack_entry_hash(p, base_offset)); - hashmap_add(&delta_base_cache, &ent->ent); + hashmap_entry_init(ent, pack_entry_hash(p, base_offset)); + hashmap_add(&delta_base_cache, ent); } int packed_object_info(struct repository *r, struct packed_git *p, @@ -1558,16 +1553,20 @@ int packed_object_info(struct repository *r, struct packed_git *p, } } - if (oi->delta_base_oid) { + if (oi->delta_base_sha1) { if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) { - if (get_delta_base_oid(p, &w_curs, curpos, - oi->delta_base_oid, - type, obj_offset) < 0) { + const unsigned char *base; + + base = get_delta_base_sha1(p, &w_curs, curpos, + type, obj_offset); + if (!base) { type = OBJ_BAD; goto out; } + + hashcpy(oi->delta_base_sha1, base); } else - oidclr(oi->delta_base_oid); + hashclr(oi->delta_base_sha1); } oi->whence = in_delta_base_cache(p, obj_offset) ? OI_DBCACHED : @@ -1598,15 +1597,7 @@ static void *unpack_compressed_entry(struct packed_git *p, do { in = use_pack(p, w_curs, curpos, &stream.avail_in); stream.next_in = in; - /* - * Note: we must ensure the window section returned by - * use_pack() will be available throughout git_inflate()'s - * unlocked execution. Please refer to the comment at - * get_size_from_delta() to see how this is done. - */ - obj_read_unlock(); st = git_inflate(&stream, Z_FINISH); - obj_read_lock(); if (!stream.avail_out) break; /* the payload is larger than it should be */ curpos += stream.next_in - in; @@ -1691,7 +1682,7 @@ void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset, off_t len = revidx[1].offset - obj_offset; if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) { struct object_id oid; - nth_packed_object_id(&oid, p, revidx->nr); + nth_packed_object_oid(&oid, p, revidx->nr); error("bad packed object CRC for %s", oid_to_hex(&oid)); mark_bad_packed_object(p, oid.hash); @@ -1780,7 +1771,7 @@ void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset, struct object_id base_oid; revidx = find_pack_revindex(p, obj_offset); if (revidx) { - nth_packed_object_id(&base_oid, p, revidx->nr); + nth_packed_object_oid(&base_oid, p, revidx->nr); error("failed to read delta base object %s" " at offset %"PRIuMAX" from %s", oid_to_hex(&base_oid), (uintmax_t)obj_offset, @@ -1867,27 +1858,36 @@ int bsearch_pack(const struct object_id *oid, const struct packed_git *p, uint32 index_lookup, index_lookup_width, result); } -int nth_packed_object_id(struct object_id *oid, - struct packed_git *p, - uint32_t n) +const unsigned char *nth_packed_object_sha1(struct packed_git *p, + uint32_t n) { const unsigned char *index = p->index_data; const unsigned int hashsz = the_hash_algo->rawsz; if (!index) { if (open_pack_index(p)) - return -1; + return NULL; index = p->index_data; } if (n >= p->num_objects) - return -1; + return NULL; index += 4 * 256; if (p->index_version == 1) { - oidread(oid, index + (hashsz + 4) * n + 4); + return index + (hashsz + 4) * n + 4; } else { index += 8; - oidread(oid, index + hashsz * n); + return index + hashsz * n; } - return 0; +} + +const struct object_id *nth_packed_object_oid(struct object_id *oid, + struct packed_git *p, + uint32_t n) +{ + const unsigned char *hash = nth_packed_object_sha1(p, n); + if (!hash) + return NULL; + hashcpy(oid->hash, hash); + return oid; } void check_pack_index_ptr(const struct packed_git *p, const void *vptr) @@ -2066,7 +2066,7 @@ int for_each_object_in_pack(struct packed_git *p, else pos = i; - if (nth_packed_object_id(&oid, p, pos) < 0) + if (!nth_packed_object_oid(&oid, p, pos)) return error("unable to get sha1 of object %u in %s", pos, p->pack_name); @@ -2139,7 +2139,7 @@ static int add_promisor_object(const struct object_id *oid, oidset_insert(set, &parents->item->object.oid); } else if (obj->type == OBJ_TAG) { struct tag *tag = (struct tag *) obj; - oidset_insert(set, get_tagged_oid(tag)); + oidset_insert(set, &tag->tagged->oid); } return 0; } @@ -2150,7 +2150,7 @@ int is_promisor_object(const struct object_id *oid) static int promisor_objects_prepared; if (!promisor_objects_prepared) { - if (has_promisor_remote()) { + if (repository_format_partial_clone) { for_each_packed_object(add_promisor_object, &promisor_objects, FOR_EACH_OBJECT_PROMISOR_ONLY); -- cgit 1.4.1