speed up alt_odb_usable() with many alternates
With many alternates, the duplicate check in alt_odb_usable() wastes many cycles doing repeated fspathcmp() on every existing alternate. Use a khash to speed up lookups by odb->path. Since the kh_put_* API uses the supplied key without duplicating it, we also take advantage of it to replace both xstrdup() and strbuf_release() in link_alt_odb_entry() with strbuf_detach() to avoid the allocation and copy. In a test repository with 50K alternates and each of those 50K alternates having one alternate each (for a total of 100K total alternates); this speeds up lookup of a non-existent blob from over 16 minutes to roughly 2.7 seconds on my busy workstation. Note: all underlying git object directories were small and unpacked with only loose objects and no packs. Having to load packs increases times significantly. Signed-off-by: Eric Wong <e@80x24.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Eric Wong committed
Jul 7, 2021 at 23:10 UTC
cf2dc1c238c6fd5f93c315a3045ccf95459701cd
5 files changed
+42
-12
dir.c
+10
@@ -84,11 +84,21 @@ int fspathcmp(const char *a, const char *b)
84
return ignore_case ? strcasecmp(a, b) : strcmp(a, b);
85
}
86
87
+int fspatheq(const char *a, const char *b)
88
+{
89
+ return !fspathcmp(a, b);
90
+}
91
+
92
int fspathncmp(const char *a, const char *b, size_t count)
93
{
94
return ignore_case ? strncasecmp(a, b, count) : strncmp(a, b, count);
95
}
96
97
+unsigned int fspathhash(const char *str)
98
+{
99
+ return ignore_case ? strihash(str) : strhash(str);
100
+}
101
+
102
int git_fnmatch(const struct pathspec_item *item,
103
const char *pattern, const char *string,
104
int prefix)
dir.h
+2
@@ -489,7 +489,9 @@ int remove_dir_recursively(struct strbuf *path, int flag);
489
int remove_path(const char *path);
490
491
int fspathcmp(const char *a, const char *b);
492
+int fspatheq(const char *a, const char *b);
493
int fspathncmp(const char *a, const char *b, size_t count);
494
+unsigned int fspathhash(const char *str);
495
496
/*
497
* The prefix part of pattern must not contains wildcards.
object-file.c
+21
-12
@@ -517,9 +517,9 @@ const char *loose_object_path(struct repository *r, struct strbuf *buf,
517
*/
518
static int alt_odb_usable(struct raw_object_store *o,
519
struct strbuf *path,
520
- const char *normalized_objdir)
520
+ const char *normalized_objdir, khiter_t *pos)
521
{
522
- struct object_directory *odb;
522
+ int r;
523
524
/* Detect cases where alternate disappeared */
525
if (!is_directory(path->buf)) {
@@ -533,14 +533,20 @@ static int alt_odb_usable(struct raw_object_store *o,
533
* Prevent the common mistake of listing the same
534
* thing twice, or object directory itself.
535
*/
536
- for (odb = o->odb; odb; odb = odb->next) {
537
- if (!fspathcmp(path->buf, odb->path))
538
- return 0;
536
+ if (!o->odb_by_path) {
537
+ khiter_t p;
538
+
539
+ o->odb_by_path = kh_init_odb_path_map();
540
+ assert(!o->odb->next);
541
+ p = kh_put_odb_path_map(o->odb_by_path, o->odb->path, &r);
542
+ assert(r == 1); /* never used */
543
+ kh_value(o->odb_by_path, p) = o->odb;
544
}
540
- if (!fspathcmp(path->buf, normalized_objdir))
545
+ if (fspatheq(path->buf, normalized_objdir))
546
return 0;
542
-
543
- return 1;
547
+ *pos = kh_put_odb_path_map(o->odb_by_path, path->buf, &r);
548
+ /* r: 0 = exists, 1 = never used, 2 = deleted */
549
+ return r == 0 ? 0 : 1;
550
}
551
552
/*
@@ -566,6 +572,7 @@ static int link_alt_odb_entry(struct repository *r, const char *entry,
572
{
573
struct object_directory *ent;
574
struct strbuf pathbuf = STRBUF_INIT;
575
+ khiter_t pos;
576
577
if (!is_absolute_path(entry) && relative_base) {
578
strbuf_realpath(&pathbuf, relative_base, 1);
@@ -587,23 +594,25 @@ static int link_alt_odb_entry(struct repository *r, const char *entry,
594
while (pathbuf.len && pathbuf.buf[pathbuf.len - 1] == '/')
595
strbuf_setlen(&pathbuf, pathbuf.len - 1);
596
590
- if (!alt_odb_usable(r->objects, &pathbuf, normalized_objdir)) {
597
+ if (!alt_odb_usable(r->objects, &pathbuf, normalized_objdir, &pos)) {
598
strbuf_release(&pathbuf);
599
return -1;
600
}
601
602
CALLOC_ARRAY(ent, 1);
596
- ent->path = xstrdup(pathbuf.buf);
603
+ /* pathbuf.buf is already in r->objects->odb_by_path */
604
+ ent->path = strbuf_detach(&pathbuf, NULL);
605
606
/* add the alternate entry */
607
*r->objects->odb_tail = ent;
608
r->objects->odb_tail = &(ent->next);
609
ent->next = NULL;
610
+ assert(r->objects->odb_by_path);
611
+ kh_value(r->objects->odb_by_path, pos) = ent;
612
613
/* recursively add alternates */
604
- read_info_alternates(r, pathbuf.buf, depth + 1);
614
+ read_info_alternates(r, ent->path, depth + 1);
615
606
- strbuf_release(&pathbuf);
616
return 0;
617
}
618
object-store.h
+7
@@ -7,6 +7,8 @@
7
#include "oid-array.h"
8
#include "strbuf.h"
9
#include "thread-utils.h"
10
+#include "khash.h"
11
+#include "dir.h"
12
13
struct object_directory {
14
struct object_directory *next;
@@ -30,6 +32,9 @@ struct object_directory {
32
char *path;
33
};
34
35
+KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
36
+ struct object_directory *, 1, fspathhash, fspatheq);
37
+
38
void prepare_alt_odb(struct repository *r);
39
char *compute_alternate_path(const char *path, struct strbuf *err);
40
typedef int alt_odb_fn(struct object_directory *, void *);
@@ -116,6 +121,8 @@ struct raw_object_store {
121
*/
122
struct object_directory *odb;
123
struct object_directory **odb_tail;
124
+ kh_odb_path_map_t *odb_by_path;
125
+
126
int loaded_alternates;
127
128
/*
object.c
+2
@@ -511,6 +511,8 @@ static void free_object_directories(struct raw_object_store *o)
511
free_object_directory(o->odb);
512
o->odb = next;
513
}
514
+ kh_destroy_odb_path_map(o->odb_by_path);
515
+ o->odb_by_path = NULL;
516
}
517
518
void raw_object_store_clear(struct raw_object_store *o)