@matheusbsbs / matheus-bagliano / commits / ec48540fe8
Fork 0

packfile.c: speed up loading lots of packfiles

When loading packfiles on start-up, we traverse the internal packfile list once per file to avoid reloading packfiles that have already been loaded. This check runs in quadratic time, so for poorly maintained repos with a large number of packfiles, it can be pretty slow. Add a hashmap containing the packfile names as we load them so that the average runtime cost of checking for already-loaded packs becomes constant. Add a perf test to p5303 to show speed-up. The existing p5303 test runtimes are dominated by other factors and do not show an appreciable speed-up. The new test in p5303 clearly exposes a speed-up in bad cases. In this test we create 10,000 packfiles and measure the start-up time of git rev-parse, which does little else besides load in the packs. Here are the numbers for the new p5303 test: Test HEAD^ HEAD --------------------------------------------------------------------- 5303.12: load 10,000 packs 1.03(0.92+0.10) 0.12(0.02+0.09) -88.3% Signed-off-by: Colin Stolley <cstolley@runbox.com> Helped-by: Jeff King <peff@peff.net> [jc: squashed the change to call hashmap in install_packed_git() by peff] Signed-off-by: Junio C Hamano <gitster@pobox.com>

Colin Stolley committed Nov 27, 2019 at 16:24 UTC ec48540fe8c387cf7424d5387ddbd53e89bb9d51
4 files changed +52 -9
object-store.h
+21
@@ -60,6 +60,7 @@ struct oid_array *odb_loose_cache(struct object_directory *odb,
60 void odb_clear_loose_cache(struct object_directory *odb);
61
62 struct packed_git {
63 + struct hashmap_entry packmap_ent;
64 struct packed_git *next;
65 struct list_head mru;
66 struct pack_window *windows;
@@ -88,6 +89,20 @@ struct packed_git {
89
90 struct multi_pack_index;
91
92 +static inline int pack_map_entry_cmp(const void *unused_cmp_data,
93 + const struct hashmap_entry *entry,
94 + const struct hashmap_entry *entry2,
95 + const void *keydata)
96 +{
97 + const char *key = keydata;
98 + const struct packed_git *pg1, *pg2;
99 +
100 + pg1 = container_of(entry, const struct packed_git, packmap_ent);
101 + pg2 = container_of(entry2, const struct packed_git, packmap_ent);
102 +
103 + return strcmp(pg1->pack_name, key ? key : pg2->pack_name);
104 +}
105 +
106 struct raw_object_store {
107 /*
108 * Set of all object directories; the main directory is first (and
@@ -131,6 +146,12 @@ struct raw_object_store {
146 /* A most-recently-used ordered version of the packed_git list. */
147 struct list_head packed_git_mru;
148
149 + /*
150 + * A map of packfiles to packed_git structs for tracking which
151 + * packs have been loaded already.
152 + */
153 + struct hashmap pack_map;
154 +
155 /*
156 * A fast, rough count of the number of objects in the repository.
157 * These two fields are not meant for direct access. Use
object.c
+3
@@ -479,6 +479,7 @@ struct raw_object_store *raw_object_store_new(void)
479
480 memset(o, 0, sizeof(*o));
481 INIT_LIST_HEAD(&o->packed_git_mru);
482 + hashmap_init(&o->pack_map, pack_map_entry_cmp, NULL, 0);
483 return o;
484 }
485
@@ -518,6 +519,8 @@ void raw_object_store_clear(struct raw_object_store *o)
519 INIT_LIST_HEAD(&o->packed_git_mru);
520 close_object_store(o);
521 o->packed_git = NULL;
522 +
523 + hashmap_free(&o->pack_map);
524 }
525
526 void parsed_object_pool_clear(struct parsed_object_pool *o)
packfile.c
+10 -9
@@ -757,6 +757,9 @@ void install_packed_git(struct repository *r, struct packed_git *pack)
757
758 pack->next = r->objects->packed_git;
759 r->objects->packed_git = pack;
760 +
761 + hashmap_entry_init(&pack->packmap_ent, strhash(pack->pack_name));
762 + hashmap_add(&r->objects->pack_map, &pack->packmap_ent);
763 }
764
765 void (*report_garbage)(unsigned seen_bits, const char *path);
@@ -856,20 +859,18 @@ static void prepare_pack(const char *full_name, size_t full_name_len,
859
860 if (strip_suffix_mem(full_name, &base_len, ".idx") &&
861 !(data->m && midx_contains_pack(data->m, file_name))) {
859 - /* Don't reopen a pack we already have. */
860 - for (p = data->r->objects->packed_git; p; p = p->next) {
861 - size_t len;
862 - if (strip_suffix(p->pack_name, ".pack", &len) &&
863 - len == base_len &&
864 - !memcmp(p->pack_name, full_name, len))
865 - break;
866 - }
862 + struct hashmap_entry hent;
863 + char *pack_name = xstrfmt("%.*s.pack", (int)base_len, full_name);
864 + unsigned int hash = strhash(pack_name);
865 + hashmap_entry_init(&hent, hash);
866
868 - if (!p) {
867 + /* Don't reopen a pack we already have. */
868 + if (!hashmap_get(&data->r->objects->pack_map, &hent, pack_name)) {
869 p = add_packed_git(full_name, full_name_len, data->local);
870 if (p)
871 install_packed_git(data->r, p);
872 }
873 + free(pack_name);
874 }
875
876 if (!report_garbage)
t/perf/p5303-many-packs.sh
+18
@@ -84,4 +84,22 @@ do
84 '
85 done
86
87 +# Measure pack loading with 10,000 packs.
88 +test_expect_success 'generate lots of packs' '
89 + for i in $(test_seq 10000); do
90 + echo "blob"
91 + echo "data <<EOF"
92 + echo "blob $i"
93 + echo "EOF"
94 + echo "checkpoint"
95 + done |
96 + git -c fastimport.unpackLimit=0 fast-import
97 +'
98 +
99 +# The purpose of this test is to evaluate load time for a large number
100 +# of packs while doing as little other work as possible.
101 +test_perf "load 10,000 packs" '
102 + git rev-parse --verify "HEAD^{commit}"
103 +'
104 +
105 test_done