Rework object refs tracking to reduce memory usage
Store pointers to referenced objects in a variable sized array instead of linked list. This cuts down memory usage of utilities which use object references; e.g., git-fsck-objects --full on the git.git repository consumes about 2 MB of memory tracked by Massif instead of 7 MB before the change. Object refs are still the biggest consumer of memory (57%), but the malloc overhead for a single block instead of a linked list is substantially smaller. Signed-off-by: Sergey Vlasov <vsu@altlinux.ru> Signed-off-by: Junio C Hamano <junkio@cox.net>
Sergey Vlasov committed
Nov 15, 2005 at 19:08 UTC
4a4e6fd74f7d4564ed43eaaf59b6bd70c1959f1a
7 files changed
+116
-42
commit.c
+16
-3
@@ -204,6 +204,7 @@ int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size)
204
unsigned char parent[20];
205
struct commit_list **pptr;
206
struct commit_graft *graft;
207
+ unsigned n_refs = 0;
208
209
if (item->object.parsed)
210
return 0;
@@ -214,7 +215,7 @@ int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size)
215
return error("bad tree pointer in commit %s\n", sha1_to_hex(item->object.sha1));
216
item->tree = lookup_tree(parent);
217
if (item->tree)
217
- add_ref(&item->object, &item->tree->object);
218
+ n_refs++;
219
bufptr += 46; /* "tree " + "hex sha1" + "\n" */
220
pptr = &item->parents;
221
@@ -230,7 +231,7 @@ int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size)
231
new_parent = lookup_commit(parent);
232
if (new_parent) {
233
pptr = &commit_list_insert(new_parent, pptr)->next;
233
- add_ref(&item->object, &new_parent->object);
234
+ n_refs++;
235
}
236
}
237
if (graft) {
@@ -241,10 +242,22 @@ int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size)
242
if (!new_parent)
243
continue;
244
pptr = &commit_list_insert(new_parent, pptr)->next;
244
- add_ref(&item->object, &new_parent->object);
245
+ n_refs++;
246
}
247
}
248
item->date = parse_commit_date(bufptr);
249
+
250
+ if (track_object_refs) {
251
+ unsigned i = 0;
252
+ struct commit_list *p;
253
+ struct object_refs *refs = alloc_object_refs(n_refs);
254
+ if (item->tree)
255
+ refs->ref[i++] = &item->tree->object;
256
+ for (p = item->parents; p; p = p->next)
257
+ refs->ref[i++] = &p->item->object;
258
+ set_object_refs(&item->object, refs);
259
+ }
260
+
261
return 0;
262
}
263
fsck-objects.c
+13
-9
@@ -56,7 +56,6 @@ static void check_connectivity(void)
56
/* Look up all the requirements, warn about missing objects.. */
57
for (i = 0; i < nr_objs; i++) {
58
struct object *obj = objs[i];
59
- struct object_list *refs;
59
60
if (!obj->parsed) {
61
if (!standalone && has_sha1_file(obj->sha1))
@@ -67,14 +66,19 @@ static void check_connectivity(void)
66
continue;
67
}
68
70
- for (refs = obj->refs; refs; refs = refs->next) {
71
- if (refs->item->parsed ||
72
- (!standalone && has_sha1_file(refs->item->sha1)))
73
- continue;
74
- printf("broken link from %7s %s\n",
75
- obj->type, sha1_to_hex(obj->sha1));
76
- printf(" to %7s %s\n",
77
- refs->item->type, sha1_to_hex(refs->item->sha1));
69
+ if (obj->refs) {
70
+ const struct object_refs *refs = obj->refs;
71
+ unsigned j;
72
+ for (j = 0; j < refs->count; j++) {
73
+ struct object *ref = refs->ref[j];
74
+ if (ref->parsed ||
75
+ (!standalone && has_sha1_file(ref->sha1)))
76
+ continue;
77
+ printf("broken link from %7s %s\n",
78
+ obj->type, sha1_to_hex(obj->sha1));
79
+ printf(" to %7s %s\n",
80
+ ref->type, sha1_to_hex(ref->sha1));
81
+ }
82
}
83
84
if (show_unreachable && !(obj->flags & REACHABLE)) {
object.c
+44
-18
@@ -67,40 +67,66 @@ void created_object(const unsigned char *sha1, struct object *obj)
67
nr_objs++;
68
}
69
70
-void add_ref(struct object *refer, struct object *target)
70
+struct object_refs *alloc_object_refs(unsigned count)
71
{
72
- struct object_list **pp, *p;
72
+ struct object_refs *refs;
73
+ size_t size = sizeof(*refs) + count*sizeof(struct object *);
74
74
- if (!track_object_refs)
75
+ refs = xmalloc(size);
76
+ memset(refs, 0, size);
77
+ refs->count = count;
78
+ return refs;
79
+}
80
+
81
+static int compare_object_pointers(const void *a, const void *b)
82
+{
83
+ const struct object * const *pa = a;
84
+ const struct object * const *pb = b;
85
+ return *pa - *pb;
86
+}
87
+
88
+void set_object_refs(struct object *obj, struct object_refs *refs)
89
+{
90
+ unsigned int i, j;
91
+
92
+ /* Do not install empty list of references */
93
+ if (refs->count < 1) {
94
+ free(refs);
95
return;
96
+ }
97
77
- pp = &refer->refs;
78
- while ((p = *pp) != NULL) {
79
- if (p->item == target)
80
- return;
81
- pp = &p->next;
98
+ /* Sort the list and filter out duplicates */
99
+ qsort(refs->ref, refs->count, sizeof(refs->ref[0]),
100
+ compare_object_pointers);
101
+ for (i = j = 1; i < refs->count; i++) {
102
+ if (refs->ref[i] != refs->ref[i - 1])
103
+ refs->ref[j++] = refs->ref[i];
104
+ }
105
+ if (j < refs->count) {
106
+ /* Duplicates were found - reallocate list */
107
+ size_t size = sizeof(*refs) + j*sizeof(struct object *);
108
+ refs->count = j;
109
+ refs = xrealloc(refs, size);
110
}
111
84
- target->used = 1;
85
- p = xmalloc(sizeof(*p));
86
- p->item = target;
87
- p->next = NULL;
88
- *pp = p;
112
+ for (i = 0; i < refs->count; i++)
113
+ refs->ref[i]->used = 1;
114
+ obj->refs = refs;
115
}
116
117
void mark_reachable(struct object *obj, unsigned int mask)
118
{
93
- struct object_list *p = obj->refs;
94
-
119
if (!track_object_refs)
120
die("cannot do reachability with object refs turned off");
121
/* If we've been here already, don't bother */
122
if (obj->flags & mask)
123
return;
124
obj->flags |= mask;
101
- while (p) {
102
- mark_reachable(p->item, mask);
103
- p = p->next;
125
+ if (obj->refs) {
126
+ const struct object_refs *refs = obj->refs;
127
+ unsigned i;
128
+ for (i = 0; i < refs->count; i++)
129
+ mark_reachable(refs->ref[i], mask);
130
}
131
}
132
object.h
+8
-2
@@ -7,13 +7,18 @@ struct object_list {
7
const char *name;
8
};
9
10
+struct object_refs {
11
+ unsigned count;
12
+ struct object *ref[0];
13
+};
14
+
15
struct object {
16
unsigned parsed : 1;
17
unsigned used : 1;
18
unsigned int flags;
19
unsigned char sha1[20];
20
const char *type;
16
- struct object_list *refs;
21
+ struct object_refs *refs;
22
void *util;
23
};
24
@@ -35,7 +40,8 @@ struct object *parse_object(const unsigned char *sha1);
40
/** Returns the object, with potentially excess memory allocated. **/
41
struct object *lookup_unknown_object(const unsigned char *sha1);
42
38
-void add_ref(struct object *refer, struct object *target);
43
+struct object_refs *alloc_object_refs(unsigned count);
44
+void set_object_refs(struct object *obj, struct object_refs *refs);
45
46
void mark_reachable(struct object *obj, unsigned int mask);
47
server-info.c
+18
-7
@@ -424,7 +424,6 @@ static void find_pack_info_one(int pack_ix)
424
{
425
unsigned char sha1[20];
426
struct object *o;
427
- struct object_list *ref;
427
int i;
428
struct packed_git *p = info[pack_ix]->p;
429
int num = num_packed_objects(p);
@@ -437,8 +436,12 @@ static void find_pack_info_one(int pack_ix)
436
die("corrupt pack file %s?", p->pack_name);
437
if ((o = lookup_object(sha1)) == NULL)
438
die("cannot parse %s", sha1_to_hex(sha1));
440
- for (ref = o->refs; ref; ref = ref->next)
441
- ref->item->flags = 0;
439
+ if (o->refs) {
440
+ struct object_refs *refs = o->refs;
441
+ int j;
442
+ for (j = 0; j < refs->count; j++)
443
+ refs->ref[j]->flags = 0;
444
+ }
445
o->flags = 0;
446
}
447
@@ -448,8 +451,12 @@ static void find_pack_info_one(int pack_ix)
451
die("corrupt pack file %s?", p->pack_name);
452
if ((o = lookup_object(sha1)) == NULL)
453
die("cannot find %s", sha1_to_hex(sha1));
451
- for (ref = o->refs; ref; ref = ref->next)
452
- ref->item->flags |= REFERENCED;
454
+ if (o->refs) {
455
+ struct object_refs *refs = o->refs;
456
+ int j;
457
+ for (j = 0; j < refs->count; j++)
458
+ refs->ref[j]->flags |= REFERENCED;
459
+ }
460
o->flags |= INTERNAL;
461
}
462
@@ -460,8 +467,12 @@ static void find_pack_info_one(int pack_ix)
467
die("cannot find %s", sha1_to_hex(sha1));
468
469
show(o, pack_ix);
463
- for (ref = o->refs; ref; ref = ref->next)
464
- show(ref->item, pack_ix);
470
+ if (o->refs) {
471
+ struct object_refs *refs = o->refs;
472
+ int j;
473
+ for (j = 0; j < refs->count; j++)
474
+ show(refs->ref[j], pack_ix);
475
+ }
476
}
477
478
}
tag.c
+5
-2
@@ -75,8 +75,11 @@ int parse_tag_buffer(struct tag *item, void *data, unsigned long size)
75
item->tag[taglen] = '\0';
76
77
item->tagged = lookup_object_type(object, type);
78
- if (item->tagged)
79
- add_ref(&item->object, item->tagged);
78
+ if (item->tagged && track_object_refs) {
79
+ struct object_refs *refs = alloc_object_refs(1);
80
+ refs->ref[0] = item->tagged;
81
+ set_object_refs(&item->object, refs);
82
+ }
83
84
return 0;
85
}
tree.c
+12
-1
@@ -148,6 +148,7 @@ int parse_tree_buffer(struct tree *item, void *buffer, unsigned long size)
148
{
149
void *bufptr = buffer;
150
struct tree_entry_list **list_p;
151
+ int n_refs = 0;
152
153
if (item->object.parsed)
154
return 0;
@@ -184,11 +185,21 @@ int parse_tree_buffer(struct tree *item, void *buffer, unsigned long size)
185
obj = &entry->item.blob->object;
186
}
187
if (obj)
187
- add_ref(&item->object, obj);
188
+ n_refs++;
189
entry->parent = NULL; /* needs to be filled by the user */
190
*list_p = entry;
191
list_p = &entry->next;
192
}
193
+
194
+ if (track_object_refs) {
195
+ struct tree_entry_list *entry;
196
+ unsigned i = 0;
197
+ struct object_refs *refs = alloc_object_refs(n_refs);
198
+ for (entry = item->entries; entry; entry = entry->next)
199
+ refs->ref[i++] = entry->item.any;
200
+ set_object_refs(&item->object, refs);
201
+ }
202
+
203
return 0;
204
}
205