refs: Use binary search to lookup refs faster
Currently we linearly search through lists of refs when we need to find a specific ref. This can be very slow if we need to lookup a large number of refs. By changing to a binary search we can make this faster. In order to be able to use a binary search we need to change from using linked lists to arrays, which we can manage using ALLOC_GROW. We can now also use the standard library qsort function to sort the refs arrays. Signed-off-by: Julian Phillips <julian@quantumfyre.co.uk> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Julian Phillips committed
Sep 29, 2011 at 23:11 UTC
e9c4c11165e48b8f3fe1b4fc4db513f8e57202fb
1 file changed
+152
-193
refs.c
+152
-193
@@ -8,14 +8,18 @@
8
#define REF_KNOWS_PEELED 04
9
#define REF_BROKEN 010
10
11
-struct ref_list {
12
- struct ref_list *next;
11
+struct ref_entry {
12
unsigned char flag; /* ISSYMREF? ISPACKED? */
13
unsigned char sha1[20];
14
unsigned char peeled[20];
15
char name[FLEX_ARRAY];
16
};
17
18
+struct ref_array {
19
+ int nr, alloc;
20
+ struct ref_entry **refs;
21
+};
22
+
23
static const char *parse_ref_line(char *line, unsigned char *sha1)
24
{
25
/*
@@ -44,108 +48,80 @@ static const char *parse_ref_line(char *line, unsigned char *sha1)
48
return line;
49
}
50
47
-static struct ref_list *add_ref(const char *name, const unsigned char *sha1,
48
- int flag, struct ref_list *list,
49
- struct ref_list **new_entry)
51
+static void add_ref(const char *name, const unsigned char *sha1,
52
+ int flag, struct ref_array *refs,
53
+ struct ref_entry **new_entry)
54
{
55
int len;
52
- struct ref_list *entry;
56
+ struct ref_entry *entry;
57
58
/* Allocate it and add it in.. */
59
len = strlen(name) + 1;
56
- entry = xmalloc(sizeof(struct ref_list) + len);
60
+ entry = xmalloc(sizeof(struct ref_entry) + len);
61
hashcpy(entry->sha1, sha1);
62
hashclr(entry->peeled);
63
memcpy(entry->name, name, len);
64
entry->flag = flag;
61
- entry->next = list;
65
if (new_entry)
66
*new_entry = entry;
64
- return entry;
67
+ ALLOC_GROW(refs->refs, refs->nr + 1, refs->alloc);
68
+ refs->refs[refs->nr++] = entry;
69
}
70
67
-/* merge sort the ref list */
68
-static struct ref_list *sort_ref_list(struct ref_list *list)
71
+static int ref_entry_cmp(const void *a, const void *b)
72
{
70
- int psize, qsize, last_merge_count, cmp;
71
- struct ref_list *p, *q, *l, *e;
72
- struct ref_list *new_list = list;
73
- int k = 1;
74
- int merge_count = 0;
73
+ struct ref_entry *one = *(struct ref_entry **)a;
74
+ struct ref_entry *two = *(struct ref_entry **)b;
75
+ return strcmp(one->name, two->name);
76
+}
77
76
- if (!list)
77
- return list;
78
+static void sort_ref_array(struct ref_array *array)
79
+{
80
+ int i = 0, j = 1;
81
79
- do {
80
- last_merge_count = merge_count;
81
- merge_count = 0;
82
+ /* Nothing to sort unless there are at least two entries */
83
+ if (array->nr < 2)
84
+ return;
85
83
- psize = 0;
86
+ qsort(array->refs, array->nr, sizeof(*array->refs), ref_entry_cmp);
87
85
- p = new_list;
86
- q = new_list;
87
- new_list = NULL;
88
- l = NULL;
88
+ /* Remove any duplicates from the ref_array */
89
+ for (; j < array->nr; j++) {
90
+ struct ref_entry *a = array->refs[i];
91
+ struct ref_entry *b = array->refs[j];
92
+ if (!strcmp(a->name, b->name)) {
93
+ if (hashcmp(a->sha1, b->sha1))
94
+ die("Duplicated ref, and SHA1s don't match: %s",
95
+ a->name);
96
+ warning("Duplicated ref: %s", a->name);
97
+ continue;
98
+ }
99
+ i++;
100
+ array->refs[i] = array->refs[j];
101
+ }
102
+ array->nr = i + 1;
103
+}
104
90
- while (p) {
91
- merge_count++;
105
+static struct ref_entry *search_ref_array(struct ref_array *array, const char *name)
106
+{
107
+ struct ref_entry *e, **r;
108
+ int len;
109
93
- while (psize < k && q->next) {
94
- q = q->next;
95
- psize++;
96
- }
97
- qsize = k;
98
-
99
- while ((psize > 0) || (qsize > 0 && q)) {
100
- if (qsize == 0 || !q) {
101
- e = p;
102
- p = p->next;
103
- psize--;
104
- } else if (psize == 0) {
105
- e = q;
106
- q = q->next;
107
- qsize--;
108
- } else {
109
- cmp = strcmp(q->name, p->name);
110
- if (cmp < 0) {
111
- e = q;
112
- q = q->next;
113
- qsize--;
114
- } else if (cmp > 0) {
115
- e = p;
116
- p = p->next;
117
- psize--;
118
- } else {
119
- if (hashcmp(q->sha1, p->sha1))
120
- die("Duplicated ref, and SHA1s don't match: %s",
121
- q->name);
122
- warning("Duplicated ref: %s", q->name);
123
- e = q;
124
- q = q->next;
125
- qsize--;
126
- free(e);
127
- e = p;
128
- p = p->next;
129
- psize--;
130
- }
131
- }
110
+ if (name == NULL)
111
+ return NULL;
112
133
- e->next = NULL;
113
+ len = strlen(name) + 1;
114
+ e = xmalloc(sizeof(struct ref_entry) + len);
115
+ memcpy(e->name, name, len);
116
135
- if (l)
136
- l->next = e;
137
- if (!new_list)
138
- new_list = e;
139
- l = e;
140
- }
117
+ r = bsearch(&e, array->refs, array->nr, sizeof(*array->refs), ref_entry_cmp);
118
142
- p = q;
143
- };
119
+ free(e);
120
145
- k = k * 2;
146
- } while ((last_merge_count != merge_count) || (last_merge_count != 1));
121
+ if (r == NULL)
122
+ return NULL;
123
148
- return new_list;
124
+ return *r;
125
}
126
127
/*
@@ -155,38 +131,37 @@ static struct ref_list *sort_ref_list(struct ref_list *list)
131
static struct cached_refs {
132
char did_loose;
133
char did_packed;
158
- struct ref_list *loose;
159
- struct ref_list *packed;
134
+ struct ref_array loose;
135
+ struct ref_array packed;
136
} cached_refs, submodule_refs;
161
-static struct ref_list *current_ref;
137
+static struct ref_entry *current_ref;
138
163
-static struct ref_list *extra_refs;
139
+static struct ref_array extra_refs;
140
165
-static void free_ref_list(struct ref_list *list)
141
+static void free_ref_array(struct ref_array *array)
142
{
167
- struct ref_list *next;
168
- for ( ; list; list = next) {
169
- next = list->next;
170
- free(list);
171
- }
143
+ int i;
144
+ for (i = 0; i < array->nr; i++)
145
+ free(array->refs[i]);
146
+ free(array->refs);
147
+ array->nr = array->alloc = 0;
148
+ array->refs = NULL;
149
}
150
151
static void invalidate_cached_refs(void)
152
{
153
struct cached_refs *ca = &cached_refs;
154
178
- if (ca->did_loose && ca->loose)
179
- free_ref_list(ca->loose);
180
- if (ca->did_packed && ca->packed)
181
- free_ref_list(ca->packed);
182
- ca->loose = ca->packed = NULL;
155
+ if (ca->did_loose)
156
+ free_ref_array(&ca->loose);
157
+ if (ca->did_packed)
158
+ free_ref_array(&ca->packed);
159
ca->did_loose = ca->did_packed = 0;
160
}
161
162
static void read_packed_refs(FILE *f, struct cached_refs *cached_refs)
163
{
188
- struct ref_list *list = NULL;
189
- struct ref_list *last = NULL;
164
+ struct ref_entry *last = NULL;
165
char refline[PATH_MAX];
166
int flag = REF_ISPACKED;
167
@@ -205,7 +180,7 @@ static void read_packed_refs(FILE *f, struct cached_refs *cached_refs)
180
181
name = parse_ref_line(refline, sha1);
182
if (name) {
208
- list = add_ref(name, sha1, flag, list, &last);
183
+ add_ref(name, sha1, flag, &cached_refs->packed, &last);
184
continue;
185
}
186
if (last &&
@@ -215,21 +190,20 @@ static void read_packed_refs(FILE *f, struct cached_refs *cached_refs)
190
!get_sha1_hex(refline + 1, sha1))
191
hashcpy(last->peeled, sha1);
192
}
218
- cached_refs->packed = sort_ref_list(list);
193
+ sort_ref_array(&cached_refs->packed);
194
}
195
196
void add_extra_ref(const char *name, const unsigned char *sha1, int flag)
197
{
223
- extra_refs = add_ref(name, sha1, flag, extra_refs, NULL);
198
+ add_ref(name, sha1, flag, &extra_refs, NULL);
199
}
200
201
void clear_extra_refs(void)
202
{
228
- free_ref_list(extra_refs);
229
- extra_refs = NULL;
203
+ free_ref_array(&extra_refs);
204
}
205
232
-static struct ref_list *get_packed_refs(const char *submodule)
206
+static struct ref_array *get_packed_refs(const char *submodule)
207
{
208
const char *packed_refs_file;
209
struct cached_refs *refs;
@@ -237,7 +211,7 @@ static struct ref_list *get_packed_refs(const char *submodule)
211
if (submodule) {
212
packed_refs_file = git_path_submodule(submodule, "packed-refs");
213
refs = &submodule_refs;
240
- free_ref_list(refs->packed);
214
+ free_ref_array(&refs->packed);
215
} else {
216
packed_refs_file = git_path("packed-refs");
217
refs = &cached_refs;
@@ -245,18 +219,17 @@ static struct ref_list *get_packed_refs(const char *submodule)
219
220
if (!refs->did_packed || submodule) {
221
FILE *f = fopen(packed_refs_file, "r");
248
- refs->packed = NULL;
222
if (f) {
223
read_packed_refs(f, refs);
224
fclose(f);
225
}
226
refs->did_packed = 1;
227
}
255
- return refs->packed;
228
+ return &refs->packed;
229
}
230
258
-static struct ref_list *get_ref_dir(const char *submodule, const char *base,
259
- struct ref_list *list)
231
+static void get_ref_dir(const char *submodule, const char *base,
232
+ struct ref_array *array)
233
{
234
DIR *dir;
235
const char *path;
@@ -299,7 +272,7 @@ static struct ref_list *get_ref_dir(const char *submodule, const char *base,
272
if (stat(refdir, &st) < 0)
273
continue;
274
if (S_ISDIR(st.st_mode)) {
302
- list = get_ref_dir(submodule, ref, list);
275
+ get_ref_dir(submodule, ref, array);
276
continue;
277
}
278
if (submodule) {
@@ -314,12 +287,11 @@ static struct ref_list *get_ref_dir(const char *submodule, const char *base,
287
hashclr(sha1);
288
flag |= REF_BROKEN;
289
}
317
- list = add_ref(ref, sha1, flag, list, NULL);
290
+ add_ref(ref, sha1, flag, array, NULL);
291
}
292
free(ref);
293
closedir(dir);
294
}
322
- return list;
295
}
296
297
struct warn_if_dangling_data {
@@ -356,21 +328,21 @@ void warn_dangling_symref(FILE *fp, const char *msg_fmt, const char *refname)
328
for_each_rawref(warn_if_dangling_symref, &data);
329
}
330
359
-static struct ref_list *get_loose_refs(const char *submodule)
331
+static struct ref_array *get_loose_refs(const char *submodule)
332
{
333
if (submodule) {
362
- free_ref_list(submodule_refs.loose);
363
- submodule_refs.loose = get_ref_dir(submodule, "refs", NULL);
364
- submodule_refs.loose = sort_ref_list(submodule_refs.loose);
365
- return submodule_refs.loose;
334
+ free_ref_array(&submodule_refs.loose);
335
+ get_ref_dir(submodule, "refs", &submodule_refs.loose);
336
+ sort_ref_array(&submodule_refs.loose);
337
+ return &submodule_refs.loose;
338
}
339
340
if (!cached_refs.did_loose) {
369
- cached_refs.loose = get_ref_dir(NULL, "refs", NULL);
370
- cached_refs.loose = sort_ref_list(cached_refs.loose);
341
+ get_ref_dir(NULL, "refs", &cached_refs.loose);
342
+ sort_ref_array(&cached_refs.loose);
343
cached_refs.did_loose = 1;
344
}
373
- return cached_refs.loose;
345
+ return &cached_refs.loose;
346
}
347
348
/* We allow "recursive" symbolic refs. Only within reason, though */
@@ -381,8 +353,8 @@ static int resolve_gitlink_packed_ref(char *name, int pathlen, const char *refna
353
{
354
FILE *f;
355
struct cached_refs refs;
384
- struct ref_list *ref;
385
- int retval;
356
+ struct ref_entry *ref;
357
+ int retval = -1;
358
359
strcpy(name + pathlen, "packed-refs");
360
f = fopen(name, "r");
@@ -390,17 +362,12 @@ static int resolve_gitlink_packed_ref(char *name, int pathlen, const char *refna
362
return -1;
363
read_packed_refs(f, &refs);
364
fclose(f);
393
- ref = refs.packed;
394
- retval = -1;
395
- while (ref) {
396
- if (!strcmp(ref->name, refname)) {
397
- retval = 0;
398
- memcpy(result, ref->sha1, 20);
399
- break;
400
- }
401
- ref = ref->next;
365
+ ref = search_ref_array(&refs.packed, refname);
366
+ if (ref != NULL) {
367
+ memcpy(result, ref->sha1, 20);
368
+ retval = 0;
369
}
403
- free_ref_list(refs.packed);
370
+ free_ref_array(&refs.packed);
371
return retval;
372
}
373
@@ -501,15 +468,13 @@ const char *resolve_ref(const char *ref, unsigned char *sha1, int reading, int *
468
git_snpath(path, sizeof(path), "%s", ref);
469
/* Special case: non-existing file. */
470
if (lstat(path, &st) < 0) {
504
- struct ref_list *list = get_packed_refs(NULL);
505
- while (list) {
506
- if (!strcmp(ref, list->name)) {
507
- hashcpy(sha1, list->sha1);
508
- if (flag)
509
- *flag |= REF_ISPACKED;
510
- return ref;
511
- }
512
- list = list->next;
471
+ struct ref_array *packed = get_packed_refs(NULL);
472
+ struct ref_entry *r = search_ref_array(packed, ref);
473
+ if (r != NULL) {
474
+ hashcpy(sha1, r->sha1);
475
+ if (flag)
476
+ *flag |= REF_ISPACKED;
477
+ return ref;
478
}
479
if (reading || errno != ENOENT)
480
return NULL;
@@ -584,7 +549,7 @@ int read_ref(const char *ref, unsigned char *sha1)
549
550
#define DO_FOR_EACH_INCLUDE_BROKEN 01
551
static int do_one_ref(const char *base, each_ref_fn fn, int trim,
587
- int flags, void *cb_data, struct ref_list *entry)
552
+ int flags, void *cb_data, struct ref_entry *entry)
553
{
554
if (strncmp(base, entry->name, trim))
555
return 0;
@@ -630,18 +595,12 @@ int peel_ref(const char *ref, unsigned char *sha1)
595
return -1;
596
597
if ((flag & REF_ISPACKED)) {
633
- struct ref_list *list = get_packed_refs(NULL);
598
+ struct ref_array *array = get_packed_refs(NULL);
599
+ struct ref_entry *r = search_ref_array(array, ref);
600
635
- while (list) {
636
- if (!strcmp(list->name, ref)) {
637
- if (list->flag & REF_KNOWS_PEELED) {
638
- hashcpy(sha1, list->peeled);
639
- return 0;
640
- }
641
- /* older pack-refs did not leave peeled ones */
642
- break;
643
- }
644
- list = list->next;
601
+ if (r != NULL && r->flag & REF_KNOWS_PEELED) {
602
+ hashcpy(sha1, r->peeled);
603
+ return 0;
604
}
605
}
606
@@ -660,36 +619,39 @@ fallback:
619
static int do_for_each_ref(const char *submodule, const char *base, each_ref_fn fn,
620
int trim, int flags, void *cb_data)
621
{
663
- int retval = 0;
664
- struct ref_list *packed = get_packed_refs(submodule);
665
- struct ref_list *loose = get_loose_refs(submodule);
622
+ int retval = 0, i, p = 0, l = 0;
623
+ struct ref_array *packed = get_packed_refs(submodule);
624
+ struct ref_array *loose = get_loose_refs(submodule);
625
667
- struct ref_list *extra;
626
+ struct ref_array *extra = &extra_refs;
627
669
- for (extra = extra_refs; extra; extra = extra->next)
670
- retval = do_one_ref(base, fn, trim, flags, cb_data, extra);
628
+ for (i = 0; i < extra->nr; i++)
629
+ retval = do_one_ref(base, fn, trim, flags, cb_data, extra->refs[i]);
630
672
- while (packed && loose) {
673
- struct ref_list *entry;
674
- int cmp = strcmp(packed->name, loose->name);
631
+ while (p < packed->nr && l < loose->nr) {
632
+ struct ref_entry *entry;
633
+ int cmp = strcmp(packed->refs[p]->name, loose->refs[l]->name);
634
if (!cmp) {
676
- packed = packed->next;
635
+ p++;
636
continue;
637
}
638
if (cmp > 0) {
680
- entry = loose;
681
- loose = loose->next;
639
+ entry = loose->refs[l++];
640
} else {
683
- entry = packed;
684
- packed = packed->next;
641
+ entry = packed->refs[p++];
642
}
643
retval = do_one_ref(base, fn, trim, flags, cb_data, entry);
644
if (retval)
645
goto end_each;
646
}
647
691
- for (packed = packed ? packed : loose; packed; packed = packed->next) {
692
- retval = do_one_ref(base, fn, trim, flags, cb_data, packed);
648
+ if (l < loose->nr) {
649
+ p = l;
650
+ packed = loose;
651
+ }
652
+
653
+ for (; p < packed->nr; p++) {
654
+ retval = do_one_ref(base, fn, trim, flags, cb_data, packed->refs[p]);
655
if (retval)
656
goto end_each;
657
}
@@ -980,24 +942,24 @@ static int remove_empty_directories(const char *file)
942
}
943
944
static int is_refname_available(const char *ref, const char *oldref,
983
- struct ref_list *list, int quiet)
984
-{
985
- int namlen = strlen(ref); /* e.g. 'foo/bar' */
986
- while (list) {
987
- /* list->name could be 'foo' or 'foo/bar/baz' */
988
- if (!oldref || strcmp(oldref, list->name)) {
989
- int len = strlen(list->name);
945
+ struct ref_array *array, int quiet)
946
+{
947
+ int i, namlen = strlen(ref); /* e.g. 'foo/bar' */
948
+ for (i = 0; i < array->nr; i++ ) {
949
+ struct ref_entry *entry = array->refs[i];
950
+ /* entry->name could be 'foo' or 'foo/bar/baz' */
951
+ if (!oldref || strcmp(oldref, entry->name)) {
952
+ int len = strlen(entry->name);
953
int cmplen = (namlen < len) ? namlen : len;
991
- const char *lead = (namlen < len) ? list->name : ref;
992
- if (!strncmp(ref, list->name, cmplen) &&
954
+ const char *lead = (namlen < len) ? entry->name : ref;
955
+ if (!strncmp(ref, entry->name, cmplen) &&
956
lead[cmplen] == '/') {
957
if (!quiet)
958
error("'%s' exists; cannot create '%s'",
996
- list->name, ref);
959
+ entry->name, ref);
960
return 0;
961
}
962
}
1000
- list = list->next;
963
}
964
return 1;
965
}
@@ -1104,18 +1066,13 @@ static struct lock_file packlock;
1066
1067
static int repack_without_ref(const char *refname)
1068
{
1107
- struct ref_list *list, *packed_ref_list;
1108
- int fd;
1109
- int found = 0;
1069
+ struct ref_array *packed;
1070
+ struct ref_entry *ref;
1071
+ int fd, i;
1072
1111
- packed_ref_list = get_packed_refs(NULL);
1112
- for (list = packed_ref_list; list; list = list->next) {
1113
- if (!strcmp(refname, list->name)) {
1114
- found = 1;
1115
- break;
1116
- }
1117
- }
1118
- if (!found)
1073
+ packed = get_packed_refs(NULL);
1074
+ ref = search_ref_array(packed, refname);
1075
+ if (ref == NULL)
1076
return 0;
1077
fd = hold_lock_file_for_update(&packlock, git_path("packed-refs"), 0);
1078
if (fd < 0) {
@@ -1123,17 +1080,19 @@ static int repack_without_ref(const char *refname)
1080
return error("cannot delete '%s' from packed refs", refname);
1081
}
1082
1126
- for (list = packed_ref_list; list; list = list->next) {
1083
+ for (i = 0; i < packed->nr; i++) {
1084
char line[PATH_MAX + 100];
1085
int len;
1086
1130
- if (!strcmp(refname, list->name))
1087
+ ref = packed->refs[i];
1088
+
1089
+ if (!strcmp(refname, ref->name))
1090
continue;
1091
len = snprintf(line, sizeof(line), "%s %s\n",
1133
- sha1_to_hex(list->sha1), list->name);
1092
+ sha1_to_hex(ref->sha1), ref->name);
1093
/* this should not happen but just being defensive */
1094
if (len > sizeof(line))
1136
- die("too long a refname '%s'", list->name);
1095
+ die("too long a refname '%s'", ref->name);
1096
write_or_die(fd, line, len);
1097
}
1098
return commit_lock_file(&packlock);