sha1_name: cache readdir(3) results in find_short_object_filename()
Read each loose object subdirectory at most once when looking for unique abbreviated hashes. This speeds up commands like "git log --pretty=%h" considerably, which previously caused one readdir(3) call for each candidate, even for subdirectories that were visited before. The new cache is kept until the program ends and never invalidated. The same is already true for pack indexes. The inherent racy nature of finding unique short hashes makes it still fit for this purpose -- a conflicting new object may be added at any time. Tasks with higher consistency requirements should not use it, though. The cached object names are stored in an oid_array, which is quite compact. The bitmap for remembering which subdir was already read is stored as a char array, with one char per directory -- that's not quite as compact, but really simple and incurs only an overhead equivalent to 11 hashes after all. Suggested-by: Jeff King <peff@peff.net> Helped-by: Jeff King <peff@peff.net> Signed-off-by: Rene Scharfe <l.s.r@web.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
René Scharfe committed
Jun 22, 2017 at 20:19 UTC
cc817ca3ef2267c21af9589a7f92190a3659906c
3 files changed
+53
-26
cache.h
+17
@@ -11,6 +11,7 @@
11
#include "string-list.h"
12
#include "pack-revindex.h"
13
#include "hash.h"
14
+#include "sha1-array.h"
15
16
#ifndef platform_SHA_CTX
17
/*
@@ -1579,6 +1580,16 @@ extern struct alternate_object_database {
1580
struct strbuf scratch;
1581
size_t base_len;
1582
1583
+ /*
1584
+ * Used to store the results of readdir(3) calls when searching
1585
+ * for unique abbreviated hashes. This cache is never
1586
+ * invalidated, thus it's racy and not necessarily accurate.
1587
+ * That's fine for its purpose; don't use it for tasks requiring
1588
+ * greater accuracy!
1589
+ */
1590
+ char loose_objects_subdir_seen[256];
1591
+ struct oid_array loose_objects_cache;
1592
+
1593
char path[FLEX_ARRAY];
1594
} *alt_odb_list;
1595
extern void prepare_alt_odb(void);
@@ -1797,6 +1808,12 @@ typedef int each_loose_cruft_fn(const char *basename,
1808
typedef int each_loose_subdir_fn(int nr,
1809
const char *path,
1810
void *data);
1811
+int for_each_file_in_obj_subdir(int subdir_nr,
1812
+ struct strbuf *path,
1813
+ each_loose_object_fn obj_cb,
1814
+ each_loose_cruft_fn cruft_cb,
1815
+ each_loose_subdir_fn subdir_cb,
1816
+ void *data);
1817
int for_each_loose_file_in_objdir(const char *path,
1818
each_loose_object_fn obj_cb,
1819
each_loose_cruft_fn cruft_cb,
sha1_file.c
+6
-6
@@ -3735,12 +3735,12 @@ void assert_sha1_type(const unsigned char *sha1, enum object_type expect)
3735
typename(expect));
3736
}
3737
3738
-static int for_each_file_in_obj_subdir(int subdir_nr,
3739
- struct strbuf *path,
3740
- each_loose_object_fn obj_cb,
3741
- each_loose_cruft_fn cruft_cb,
3742
- each_loose_subdir_fn subdir_cb,
3743
- void *data)
3738
+int for_each_file_in_obj_subdir(int subdir_nr,
3739
+ struct strbuf *path,
3740
+ each_loose_object_fn obj_cb,
3741
+ each_loose_cruft_fn cruft_cb,
3742
+ each_loose_subdir_fn subdir_cb,
3743
+ void *data)
3744
{
3745
size_t baselen = path->len;
3746
DIR *dir = opendir(path->buf);
sha1_name.c
+30
-20
@@ -77,10 +77,19 @@ static void update_candidates(struct disambiguate_state *ds, const struct object
77
/* otherwise, current can be discarded and candidate is still good */
78
}
79
80
+static int append_loose_object(const struct object_id *oid, const char *path,
81
+ void *data)
82
+{
83
+ oid_array_append(data, oid);
84
+ return 0;
85
+}
86
+
87
+static int match_sha(unsigned, const unsigned char *, const unsigned char *);
88
+
89
static void find_short_object_filename(struct disambiguate_state *ds)
90
{
91
+ int subdir_nr = ds->bin_pfx.hash[0];
92
struct alternate_object_database *alt;
83
- char hex[GIT_MAX_HEXSZ];
93
static struct alternate_object_database *fakeent;
94
95
if (!fakeent) {
@@ -95,29 +104,30 @@ static void find_short_object_filename(struct disambiguate_state *ds)
104
}
105
fakeent->next = alt_odb_list;
106
98
- xsnprintf(hex, sizeof(hex), "%.2s", ds->hex_pfx);
107
for (alt = fakeent; alt && !ds->ambiguous; alt = alt->next) {
100
- struct strbuf *buf = alt_scratch_buf(alt);
101
- struct dirent *de;
102
- DIR *dir;
103
-
104
- strbuf_addf(buf, "%.2s/", ds->hex_pfx);
105
- dir = opendir(buf->buf);
106
- if (!dir)
107
- continue;
108
+ int pos;
109
109
- while (!ds->ambiguous && (de = readdir(dir)) != NULL) {
110
- struct object_id oid;
110
+ if (!alt->loose_objects_subdir_seen[subdir_nr]) {
111
+ struct strbuf *buf = alt_scratch_buf(alt);
112
+ strbuf_addf(buf, "%02x/", subdir_nr);
113
+ for_each_file_in_obj_subdir(subdir_nr, buf,
114
+ append_loose_object,
115
+ NULL, NULL,
116
+ &alt->loose_objects_cache);
117
+ alt->loose_objects_subdir_seen[subdir_nr] = 1;
118
+ }
119
112
- if (strlen(de->d_name) != GIT_SHA1_HEXSZ - 2)
113
- continue;
114
- if (memcmp(de->d_name, ds->hex_pfx + 2, ds->len - 2))
115
- continue;
116
- memcpy(hex + 2, de->d_name, GIT_SHA1_HEXSZ - 2);
117
- if (!get_oid_hex(hex, &oid))
118
- update_candidates(ds, &oid);
120
+ pos = oid_array_lookup(&alt->loose_objects_cache, &ds->bin_pfx);
121
+ if (pos < 0)
122
+ pos = -1 - pos;
123
+ while (!ds->ambiguous && pos < alt->loose_objects_cache.nr) {
124
+ const struct object_id *oid;
125
+ oid = alt->loose_objects_cache.oid + pos;
126
+ if (!match_sha(ds->len, ds->bin_pfx.hash, oid->hash))
127
+ break;
128
+ update_candidates(ds, oid);
129
+ pos++;
130
}
120
- closedir(dir);
131
}
132
}
133