refs: Use binary search to lookup refs faster

Currently we linearly search through lists of refs when we need to find a specific ref. This can be very slow if we need to lookup a large number of refs. By changing to a binary search we can make this faster. In order to be able to use a binary search we need to change from using linked lists to arrays, which we can manage using ALLOC_GROW. We can now also use the standard library qsort function to sort the refs arrays. Signed-off-by: Julian Phillips <julian@quantumfyre.co.uk> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Julian Phillips committed Sep 29, 2011 at 23:11 UTC e9c4c11165e48b8f3fe1b4fc4db513f8e57202fb
1 file changed +152 -193
refs.c
+152 -193
@@ -8,14 +8,18 @@
8 #define REF_KNOWS_PEELED 04
9 #define REF_BROKEN 010
10
11 -struct ref_list {
12 - struct ref_list *next;
11 +struct ref_entry {
12 unsigned char flag; /* ISSYMREF? ISPACKED? */
13 unsigned char sha1[20];
14 unsigned char peeled[20];
15 char name[FLEX_ARRAY];
16 };
17
18 +struct ref_array {
19 + int nr, alloc;
20 + struct ref_entry **refs;
21 +};
22 +
23 static const char *parse_ref_line(char *line, unsigned char *sha1)
24 {
25 /*
@@ -44,108 +48,80 @@ static const char *parse_ref_line(char *line, unsigned char *sha1)
48 return line;
49 }
50
47 -static struct ref_list *add_ref(const char *name, const unsigned char *sha1,
48 - int flag, struct ref_list *list,
49 - struct ref_list **new_entry)
51 +static void add_ref(const char *name, const unsigned char *sha1,
52 + int flag, struct ref_array *refs,
53 + struct ref_entry **new_entry)
54 {
55 int len;
52 - struct ref_list *entry;
56 + struct ref_entry *entry;
57
58 /* Allocate it and add it in.. */
59 len = strlen(name) + 1;
56 - entry = xmalloc(sizeof(struct ref_list) + len);
60 + entry = xmalloc(sizeof(struct ref_entry) + len);
61 hashcpy(entry->sha1, sha1);
62 hashclr(entry->peeled);
63 memcpy(entry->name, name, len);
64 entry->flag = flag;
61 - entry->next = list;
65 if (new_entry)
66 *new_entry = entry;
64 - return entry;
67 + ALLOC_GROW(refs->refs, refs->nr + 1, refs->alloc);
68 + refs->refs[refs->nr++] = entry;
69 }
70
67 -/* merge sort the ref list */
68 -static struct ref_list *sort_ref_list(struct ref_list *list)
71 +static int ref_entry_cmp(const void *a, const void *b)
72 {
70 - int psize, qsize, last_merge_count, cmp;
71 - struct ref_list *p, *q, *l, *e;
72 - struct ref_list *new_list = list;
73 - int k = 1;
74 - int merge_count = 0;
73 + struct ref_entry *one = *(struct ref_entry **)a;
74 + struct ref_entry *two = *(struct ref_entry **)b;
75 + return strcmp(one->name, two->name);
76 +}
77
76 - if (!list)
77 - return list;
78 +static void sort_ref_array(struct ref_array *array)
79 +{
80 + int i = 0, j = 1;
81
79 - do {
80 - last_merge_count = merge_count;
81 - merge_count = 0;
82 + /* Nothing to sort unless there are at least two entries */
83 + if (array->nr < 2)
84 + return;
85
83 - psize = 0;
86 + qsort(array->refs, array->nr, sizeof(*array->refs), ref_entry_cmp);
87
85 - p = new_list;
86 - q = new_list;
87 - new_list = NULL;
88 - l = NULL;
88 + /* Remove any duplicates from the ref_array */
89 + for (; j < array->nr; j++) {
90 + struct ref_entry *a = array->refs[i];
91 + struct ref_entry *b = array->refs[j];
92 + if (!strcmp(a->name, b->name)) {
93 + if (hashcmp(a->sha1, b->sha1))
94 + die("Duplicated ref, and SHA1s don't match: %s",
95 + a->name);
96 + warning("Duplicated ref: %s", a->name);
97 + continue;
98 + }
99 + i++;
100 + array->refs[i] = array->refs[j];
101 + }
102 + array->nr = i + 1;
103 +}
104
90 - while (p) {
91 - merge_count++;
105 +static struct ref_entry *search_ref_array(struct ref_array *array, const char *name)
106 +{
107 + struct ref_entry *e, **r;
108 + int len;
109
93 - while (psize < k && q->next) {
94 - q = q->next;
95 - psize++;
96 - }
97 - qsize = k;
98 -
99 - while ((psize > 0) || (qsize > 0 && q)) {
100 - if (qsize == 0 || !q) {
101 - e = p;
102 - p = p->next;
103 - psize--;
104 - } else if (psize == 0) {
105 - e = q;
106 - q = q->next;
107 - qsize--;
108 - } else {
109 - cmp = strcmp(q->name, p->name);
110 - if (cmp < 0) {
111 - e = q;
112 - q = q->next;
113 - qsize--;
114 - } else if (cmp > 0) {
115 - e = p;
116 - p = p->next;
117 - psize--;
118 - } else {
119 - if (hashcmp(q->sha1, p->sha1))
120 - die("Duplicated ref, and SHA1s don't match: %s",
121 - q->name);
122 - warning("Duplicated ref: %s", q->name);
123 - e = q;
124 - q = q->next;
125 - qsize--;
126 - free(e);
127 - e = p;
128 - p = p->next;
129 - psize--;
130 - }
131 - }
110 + if (name == NULL)
111 + return NULL;
112
133 - e->next = NULL;
113 + len = strlen(name) + 1;
114 + e = xmalloc(sizeof(struct ref_entry) + len);
115 + memcpy(e->name, name, len);
116
135 - if (l)
136 - l->next = e;
137 - if (!new_list)
138 - new_list = e;
139 - l = e;
140 - }
117 + r = bsearch(&e, array->refs, array->nr, sizeof(*array->refs), ref_entry_cmp);
118
142 - p = q;
143 - };
119 + free(e);
120
145 - k = k * 2;
146 - } while ((last_merge_count != merge_count) || (last_merge_count != 1));
121 + if (r == NULL)
122 + return NULL;
123
148 - return new_list;
124 + return *r;
125 }
126
127 /*
@@ -155,38 +131,37 @@ static struct ref_list *sort_ref_list(struct ref_list *list)
131 static struct cached_refs {
132 char did_loose;
133 char did_packed;
158 - struct ref_list *loose;
159 - struct ref_list *packed;
134 + struct ref_array loose;
135 + struct ref_array packed;
136 } cached_refs, submodule_refs;
161 -static struct ref_list *current_ref;
137 +static struct ref_entry *current_ref;
138
163 -static struct ref_list *extra_refs;
139 +static struct ref_array extra_refs;
140
165 -static void free_ref_list(struct ref_list *list)
141 +static void free_ref_array(struct ref_array *array)
142 {
167 - struct ref_list *next;
168 - for ( ; list; list = next) {
169 - next = list->next;
170 - free(list);
171 - }
143 + int i;
144 + for (i = 0; i < array->nr; i++)
145 + free(array->refs[i]);
146 + free(array->refs);
147 + array->nr = array->alloc = 0;
148 + array->refs = NULL;
149 }
150
151 static void invalidate_cached_refs(void)
152 {
153 struct cached_refs *ca = &cached_refs;
154
178 - if (ca->did_loose && ca->loose)
179 - free_ref_list(ca->loose);
180 - if (ca->did_packed && ca->packed)
181 - free_ref_list(ca->packed);
182 - ca->loose = ca->packed = NULL;
155 + if (ca->did_loose)
156 + free_ref_array(&ca->loose);
157 + if (ca->did_packed)
158 + free_ref_array(&ca->packed);
159 ca->did_loose = ca->did_packed = 0;
160 }
161
162 static void read_packed_refs(FILE *f, struct cached_refs *cached_refs)
163 {
188 - struct ref_list *list = NULL;
189 - struct ref_list *last = NULL;
164 + struct ref_entry *last = NULL;
165 char refline[PATH_MAX];
166 int flag = REF_ISPACKED;
167
@@ -205,7 +180,7 @@ static void read_packed_refs(FILE *f, struct cached_refs *cached_refs)
180
181 name = parse_ref_line(refline, sha1);
182 if (name) {
208 - list = add_ref(name, sha1, flag, list, &last);
183 + add_ref(name, sha1, flag, &cached_refs->packed, &last);
184 continue;
185 }
186 if (last &&
@@ -215,21 +190,20 @@ static void read_packed_refs(FILE *f, struct cached_refs *cached_refs)
190 !get_sha1_hex(refline + 1, sha1))
191 hashcpy(last->peeled, sha1);
192 }
218 - cached_refs->packed = sort_ref_list(list);
193 + sort_ref_array(&cached_refs->packed);
194 }
195
196 void add_extra_ref(const char *name, const unsigned char *sha1, int flag)
197 {
223 - extra_refs = add_ref(name, sha1, flag, extra_refs, NULL);
198 + add_ref(name, sha1, flag, &extra_refs, NULL);
199 }
200
201 void clear_extra_refs(void)
202 {
228 - free_ref_list(extra_refs);
229 - extra_refs = NULL;
203 + free_ref_array(&extra_refs);
204 }
205
232 -static struct ref_list *get_packed_refs(const char *submodule)
206 +static struct ref_array *get_packed_refs(const char *submodule)
207 {
208 const char *packed_refs_file;
209 struct cached_refs *refs;
@@ -237,7 +211,7 @@ static struct ref_list *get_packed_refs(const char *submodule)
211 if (submodule) {
212 packed_refs_file = git_path_submodule(submodule, "packed-refs");
213 refs = &submodule_refs;
240 - free_ref_list(refs->packed);
214 + free_ref_array(&refs->packed);
215 } else {
216 packed_refs_file = git_path("packed-refs");
217 refs = &cached_refs;
@@ -245,18 +219,17 @@ static struct ref_list *get_packed_refs(const char *submodule)
219
220 if (!refs->did_packed || submodule) {
221 FILE *f = fopen(packed_refs_file, "r");
248 - refs->packed = NULL;
222 if (f) {
223 read_packed_refs(f, refs);
224 fclose(f);
225 }
226 refs->did_packed = 1;
227 }
255 - return refs->packed;
228 + return &refs->packed;
229 }
230
258 -static struct ref_list *get_ref_dir(const char *submodule, const char *base,
259 - struct ref_list *list)
231 +static void get_ref_dir(const char *submodule, const char *base,
232 + struct ref_array *array)
233 {
234 DIR *dir;
235 const char *path;
@@ -299,7 +272,7 @@ static struct ref_list *get_ref_dir(const char *submodule, const char *base,
272 if (stat(refdir, &st) < 0)
273 continue;
274 if (S_ISDIR(st.st_mode)) {
302 - list = get_ref_dir(submodule, ref, list);
275 + get_ref_dir(submodule, ref, array);
276 continue;
277 }
278 if (submodule) {
@@ -314,12 +287,11 @@ static struct ref_list *get_ref_dir(const char *submodule, const char *base,
287 hashclr(sha1);
288 flag |= REF_BROKEN;
289 }
317 - list = add_ref(ref, sha1, flag, list, NULL);
290 + add_ref(ref, sha1, flag, array, NULL);
291 }
292 free(ref);
293 closedir(dir);
294 }
322 - return list;
295 }
296
297 struct warn_if_dangling_data {
@@ -356,21 +328,21 @@ void warn_dangling_symref(FILE *fp, const char *msg_fmt, const char *refname)
328 for_each_rawref(warn_if_dangling_symref, &data);
329 }
330
359 -static struct ref_list *get_loose_refs(const char *submodule)
331 +static struct ref_array *get_loose_refs(const char *submodule)
332 {
333 if (submodule) {
362 - free_ref_list(submodule_refs.loose);
363 - submodule_refs.loose = get_ref_dir(submodule, "refs", NULL);
364 - submodule_refs.loose = sort_ref_list(submodule_refs.loose);
365 - return submodule_refs.loose;
334 + free_ref_array(&submodule_refs.loose);
335 + get_ref_dir(submodule, "refs", &submodule_refs.loose);
336 + sort_ref_array(&submodule_refs.loose);
337 + return &submodule_refs.loose;
338 }
339
340 if (!cached_refs.did_loose) {
369 - cached_refs.loose = get_ref_dir(NULL, "refs", NULL);
370 - cached_refs.loose = sort_ref_list(cached_refs.loose);
341 + get_ref_dir(NULL, "refs", &cached_refs.loose);
342 + sort_ref_array(&cached_refs.loose);
343 cached_refs.did_loose = 1;
344 }
373 - return cached_refs.loose;
345 + return &cached_refs.loose;
346 }
347
348 /* We allow "recursive" symbolic refs. Only within reason, though */
@@ -381,8 +353,8 @@ static int resolve_gitlink_packed_ref(char *name, int pathlen, const char *refna
353 {
354 FILE *f;
355 struct cached_refs refs;
384 - struct ref_list *ref;
385 - int retval;
356 + struct ref_entry *ref;
357 + int retval = -1;
358
359 strcpy(name + pathlen, "packed-refs");
360 f = fopen(name, "r");
@@ -390,17 +362,12 @@ static int resolve_gitlink_packed_ref(char *name, int pathlen, const char *refna
362 return -1;
363 read_packed_refs(f, &refs);
364 fclose(f);
393 - ref = refs.packed;
394 - retval = -1;
395 - while (ref) {
396 - if (!strcmp(ref->name, refname)) {
397 - retval = 0;
398 - memcpy(result, ref->sha1, 20);
399 - break;
400 - }
401 - ref = ref->next;
365 + ref = search_ref_array(&refs.packed, refname);
366 + if (ref != NULL) {
367 + memcpy(result, ref->sha1, 20);
368 + retval = 0;
369 }
403 - free_ref_list(refs.packed);
370 + free_ref_array(&refs.packed);
371 return retval;
372 }
373
@@ -501,15 +468,13 @@ const char *resolve_ref(const char *ref, unsigned char *sha1, int reading, int *
468 git_snpath(path, sizeof(path), "%s", ref);
469 /* Special case: non-existing file. */
470 if (lstat(path, &st) < 0) {
504 - struct ref_list *list = get_packed_refs(NULL);
505 - while (list) {
506 - if (!strcmp(ref, list->name)) {
507 - hashcpy(sha1, list->sha1);
508 - if (flag)
509 - *flag |= REF_ISPACKED;
510 - return ref;
511 - }
512 - list = list->next;
471 + struct ref_array *packed = get_packed_refs(NULL);
472 + struct ref_entry *r = search_ref_array(packed, ref);
473 + if (r != NULL) {
474 + hashcpy(sha1, r->sha1);
475 + if (flag)
476 + *flag |= REF_ISPACKED;
477 + return ref;
478 }
479 if (reading || errno != ENOENT)
480 return NULL;
@@ -584,7 +549,7 @@ int read_ref(const char *ref, unsigned char *sha1)
549
550 #define DO_FOR_EACH_INCLUDE_BROKEN 01
551 static int do_one_ref(const char *base, each_ref_fn fn, int trim,
587 - int flags, void *cb_data, struct ref_list *entry)
552 + int flags, void *cb_data, struct ref_entry *entry)
553 {
554 if (strncmp(base, entry->name, trim))
555 return 0;
@@ -630,18 +595,12 @@ int peel_ref(const char *ref, unsigned char *sha1)
595 return -1;
596
597 if ((flag & REF_ISPACKED)) {
633 - struct ref_list *list = get_packed_refs(NULL);
598 + struct ref_array *array = get_packed_refs(NULL);
599 + struct ref_entry *r = search_ref_array(array, ref);
600
635 - while (list) {
636 - if (!strcmp(list->name, ref)) {
637 - if (list->flag & REF_KNOWS_PEELED) {
638 - hashcpy(sha1, list->peeled);
639 - return 0;
640 - }
641 - /* older pack-refs did not leave peeled ones */
642 - break;
643 - }
644 - list = list->next;
601 + if (r != NULL && r->flag & REF_KNOWS_PEELED) {
602 + hashcpy(sha1, r->peeled);
603 + return 0;
604 }
605 }
606
@@ -660,36 +619,39 @@ fallback:
619 static int do_for_each_ref(const char *submodule, const char *base, each_ref_fn fn,
620 int trim, int flags, void *cb_data)
621 {
663 - int retval = 0;
664 - struct ref_list *packed = get_packed_refs(submodule);
665 - struct ref_list *loose = get_loose_refs(submodule);
622 + int retval = 0, i, p = 0, l = 0;
623 + struct ref_array *packed = get_packed_refs(submodule);
624 + struct ref_array *loose = get_loose_refs(submodule);
625
667 - struct ref_list *extra;
626 + struct ref_array *extra = &extra_refs;
627
669 - for (extra = extra_refs; extra; extra = extra->next)
670 - retval = do_one_ref(base, fn, trim, flags, cb_data, extra);
628 + for (i = 0; i < extra->nr; i++)
629 + retval = do_one_ref(base, fn, trim, flags, cb_data, extra->refs[i]);
630
672 - while (packed && loose) {
673 - struct ref_list *entry;
674 - int cmp = strcmp(packed->name, loose->name);
631 + while (p < packed->nr && l < loose->nr) {
632 + struct ref_entry *entry;
633 + int cmp = strcmp(packed->refs[p]->name, loose->refs[l]->name);
634 if (!cmp) {
676 - packed = packed->next;
635 + p++;
636 continue;
637 }
638 if (cmp > 0) {
680 - entry = loose;
681 - loose = loose->next;
639 + entry = loose->refs[l++];
640 } else {
683 - entry = packed;
684 - packed = packed->next;
641 + entry = packed->refs[p++];
642 }
643 retval = do_one_ref(base, fn, trim, flags, cb_data, entry);
644 if (retval)
645 goto end_each;
646 }
647
691 - for (packed = packed ? packed : loose; packed; packed = packed->next) {
692 - retval = do_one_ref(base, fn, trim, flags, cb_data, packed);
648 + if (l < loose->nr) {
649 + p = l;
650 + packed = loose;
651 + }
652 +
653 + for (; p < packed->nr; p++) {
654 + retval = do_one_ref(base, fn, trim, flags, cb_data, packed->refs[p]);
655 if (retval)
656 goto end_each;
657 }
@@ -980,24 +942,24 @@ static int remove_empty_directories(const char *file)
942 }
943
944 static int is_refname_available(const char *ref, const char *oldref,
983 - struct ref_list *list, int quiet)
984 -{
985 - int namlen = strlen(ref); /* e.g. 'foo/bar' */
986 - while (list) {
987 - /* list->name could be 'foo' or 'foo/bar/baz' */
988 - if (!oldref || strcmp(oldref, list->name)) {
989 - int len = strlen(list->name);
945 + struct ref_array *array, int quiet)
946 +{
947 + int i, namlen = strlen(ref); /* e.g. 'foo/bar' */
948 + for (i = 0; i < array->nr; i++ ) {
949 + struct ref_entry *entry = array->refs[i];
950 + /* entry->name could be 'foo' or 'foo/bar/baz' */
951 + if (!oldref || strcmp(oldref, entry->name)) {
952 + int len = strlen(entry->name);
953 int cmplen = (namlen < len) ? namlen : len;
991 - const char *lead = (namlen < len) ? list->name : ref;
992 - if (!strncmp(ref, list->name, cmplen) &&
954 + const char *lead = (namlen < len) ? entry->name : ref;
955 + if (!strncmp(ref, entry->name, cmplen) &&
956 lead[cmplen] == '/') {
957 if (!quiet)
958 error("'%s' exists; cannot create '%s'",
996 - list->name, ref);
959 + entry->name, ref);
960 return 0;
961 }
962 }
1000 - list = list->next;
963 }
964 return 1;
965 }
@@ -1104,18 +1066,13 @@ static struct lock_file packlock;
1066
1067 static int repack_without_ref(const char *refname)
1068 {
1107 - struct ref_list *list, *packed_ref_list;
1108 - int fd;
1109 - int found = 0;
1069 + struct ref_array *packed;
1070 + struct ref_entry *ref;
1071 + int fd, i;
1072
1111 - packed_ref_list = get_packed_refs(NULL);
1112 - for (list = packed_ref_list; list; list = list->next) {
1113 - if (!strcmp(refname, list->name)) {
1114 - found = 1;
1115 - break;
1116 - }
1117 - }
1118 - if (!found)
1073 + packed = get_packed_refs(NULL);
1074 + ref = search_ref_array(packed, refname);
1075 + if (ref == NULL)
1076 return 0;
1077 fd = hold_lock_file_for_update(&packlock, git_path("packed-refs"), 0);
1078 if (fd < 0) {
@@ -1123,17 +1080,19 @@ static int repack_without_ref(const char *refname)
1080 return error("cannot delete '%s' from packed refs", refname);
1081 }
1082
1126 - for (list = packed_ref_list; list; list = list->next) {
1083 + for (i = 0; i < packed->nr; i++) {
1084 char line[PATH_MAX + 100];
1085 int len;
1086
1130 - if (!strcmp(refname, list->name))
1087 + ref = packed->refs[i];
1088 +
1089 + if (!strcmp(refname, ref->name))
1090 continue;
1091 len = snprintf(line, sizeof(line), "%s %s\n",
1133 - sha1_to_hex(list->sha1), list->name);
1092 + sha1_to_hex(ref->sha1), ref->name);
1093 /* this should not happen but just being defensive */
1094 if (len > sizeof(line))
1136 - die("too long a refname '%s'", list->name);
1095 + die("too long a refname '%s'", ref->name);
1096 write_or_die(fd, line, len);
1097 }
1098 return commit_lock_file(&packlock);