use a hashmap to make remotes faster
Remotes are stored as an array, so looking one up or adding one without duplication is an O(n) operation. Reading an entire config file full of remotes is O(n^2) in the number of remotes. For a repository with tens of thousands of remotes, the running time can hit multiple minutes. Hash tables are way faster. So we add a hashmap from remote name to struct remote and use it for all lookups. The time to add a new remote to a repo that already has 50,000 remotes drops from ~2 minutes to < 1 second. We retain the old array of remotes so iterators proceed in config-file order. Signed-off-by: Patrick Reynolds <patrick.reynolds@github.com> Reviewed-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Patrick Reynolds committed
Jul 29, 2014 at 14:43 UTC
d0da003d5b1e65f6e52920e42582f43b357782ee
2 files changed
+49
-17
remote.c
+46
-17
@@ -42,6 +42,7 @@ struct rewrites {
42
static struct remote **remotes;
43
static int remotes_alloc;
44
static int remotes_nr;
45
+static struct hashmap remotes_hash;
46
47
static struct branch **branches;
48
static int branches_alloc;
@@ -136,26 +137,51 @@ static void add_url_alias(struct remote *remote, const char *url)
137
add_pushurl_alias(remote, url);
138
}
139
140
+struct remotes_hash_key {
141
+ const char *str;
142
+ int len;
143
+};
144
+
145
+static int remotes_hash_cmp(const struct remote *a, const struct remote *b, const struct remotes_hash_key *key)
146
+{
147
+ if (key)
148
+ return strncmp(a->name, key->str, key->len) || a->name[key->len];
149
+ else
150
+ return strcmp(a->name, b->name);
151
+}
152
+
153
+static inline void init_remotes_hash(void)
154
+{
155
+ if (!remotes_hash.cmpfn)
156
+ hashmap_init(&remotes_hash, (hashmap_cmp_fn)remotes_hash_cmp, 0);
157
+}
158
+
159
static struct remote *make_remote(const char *name, int len)
160
{
141
- struct remote *ret;
142
- int i;
161
+ struct remote *ret, *replaced;
162
+ struct remotes_hash_key lookup;
163
+ struct hashmap_entry lookup_entry;
164
144
- for (i = 0; i < remotes_nr; i++) {
145
- if (len ? (!strncmp(name, remotes[i]->name, len) &&
146
- !remotes[i]->name[len]) :
147
- !strcmp(name, remotes[i]->name))
148
- return remotes[i];
149
- }
165
+ if (!len)
166
+ len = strlen(name);
167
+
168
+ init_remotes_hash();
169
+ lookup.str = name;
170
+ lookup.len = len;
171
+ hashmap_entry_init(&lookup_entry, memhash(name, len));
172
+
173
+ if ((ret = hashmap_get(&remotes_hash, &lookup_entry, &lookup)) != NULL)
174
+ return ret;
175
176
ret = xcalloc(1, sizeof(struct remote));
177
ret->prune = -1; /* unspecified */
178
ALLOC_GROW(remotes, remotes_nr + 1, remotes_alloc);
179
remotes[remotes_nr++] = ret;
155
- if (len)
156
- ret->name = xstrndup(name, len);
157
- else
158
- ret->name = xstrdup(name);
180
+ ret->name = xstrndup(name, len);
181
+
182
+ hashmap_entry_init(ret, lookup_entry.hash);
183
+ replaced = hashmap_put(&remotes_hash, ret);
184
+ assert(replaced == NULL); /* no previous entry overwritten */
185
return ret;
186
}
187
@@ -717,13 +743,16 @@ struct remote *pushremote_get(const char *name)
743
744
int remote_is_configured(const char *name)
745
{
720
- int i;
746
+ struct remotes_hash_key lookup;
747
+ struct hashmap_entry lookup_entry;
748
read_config();
749
723
- for (i = 0; i < remotes_nr; i++)
724
- if (!strcmp(name, remotes[i]->name))
725
- return 1;
726
- return 0;
750
+ init_remotes_hash();
751
+ lookup.str = name;
752
+ lookup.len = strlen(name);
753
+ hashmap_entry_init(&lookup_entry, memhash(name, lookup.len));
754
+
755
+ return hashmap_get(&remotes_hash, &lookup_entry, &lookup) != NULL;
756
}
757
758
int for_each_remote(each_remote_fn fn, void *priv)
remote.h
+3
@@ -2,6 +2,7 @@
2
#define REMOTE_H
3
4
#include "parse-options.h"
5
+#include "hashmap.h"
6
7
enum {
8
REMOTE_CONFIG,
@@ -10,6 +11,8 @@ enum {
11
};
12
13
struct remote {
14
+ struct hashmap_entry ent; /* must be first */
15
+
16
const char *name;
17
int origin;
18