@matheusbsbs / matheus-bagliano / commits / 4a4e6fd74f
Fork 0

Rework object refs tracking to reduce memory usage

Store pointers to referenced objects in a variable sized array instead of linked list. This cuts down memory usage of utilities which use object references; e.g., git-fsck-objects --full on the git.git repository consumes about 2 MB of memory tracked by Massif instead of 7 MB before the change. Object refs are still the biggest consumer of memory (57%), but the malloc overhead for a single block instead of a linked list is substantially smaller. Signed-off-by: Sergey Vlasov <vsu@altlinux.ru> Signed-off-by: Junio C Hamano <junkio@cox.net>

Sergey Vlasov committed Nov 15, 2005 at 19:08 UTC 4a4e6fd74f7d4564ed43eaaf59b6bd70c1959f1a
7 files changed +116 -42
commit.c
+16 -3
@@ -204,6 +204,7 @@ int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size)
204 unsigned char parent[20];
205 struct commit_list **pptr;
206 struct commit_graft *graft;
207 + unsigned n_refs = 0;
208
209 if (item->object.parsed)
210 return 0;
@@ -214,7 +215,7 @@ int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size)
215 return error("bad tree pointer in commit %s\n", sha1_to_hex(item->object.sha1));
216 item->tree = lookup_tree(parent);
217 if (item->tree)
217 - add_ref(&item->object, &item->tree->object);
218 + n_refs++;
219 bufptr += 46; /* "tree " + "hex sha1" + "\n" */
220 pptr = &item->parents;
221
@@ -230,7 +231,7 @@ int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size)
231 new_parent = lookup_commit(parent);
232 if (new_parent) {
233 pptr = &commit_list_insert(new_parent, pptr)->next;
233 - add_ref(&item->object, &new_parent->object);
234 + n_refs++;
235 }
236 }
237 if (graft) {
@@ -241,10 +242,22 @@ int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size)
242 if (!new_parent)
243 continue;
244 pptr = &commit_list_insert(new_parent, pptr)->next;
244 - add_ref(&item->object, &new_parent->object);
245 + n_refs++;
246 }
247 }
248 item->date = parse_commit_date(bufptr);
249 +
250 + if (track_object_refs) {
251 + unsigned i = 0;
252 + struct commit_list *p;
253 + struct object_refs *refs = alloc_object_refs(n_refs);
254 + if (item->tree)
255 + refs->ref[i++] = &item->tree->object;
256 + for (p = item->parents; p; p = p->next)
257 + refs->ref[i++] = &p->item->object;
258 + set_object_refs(&item->object, refs);
259 + }
260 +
261 return 0;
262 }
263
fsck-objects.c
+13 -9
@@ -56,7 +56,6 @@ static void check_connectivity(void)
56 /* Look up all the requirements, warn about missing objects.. */
57 for (i = 0; i < nr_objs; i++) {
58 struct object *obj = objs[i];
59 - struct object_list *refs;
59
60 if (!obj->parsed) {
61 if (!standalone && has_sha1_file(obj->sha1))
@@ -67,14 +66,19 @@ static void check_connectivity(void)
66 continue;
67 }
68
70 - for (refs = obj->refs; refs; refs = refs->next) {
71 - if (refs->item->parsed ||
72 - (!standalone && has_sha1_file(refs->item->sha1)))
73 - continue;
74 - printf("broken link from %7s %s\n",
75 - obj->type, sha1_to_hex(obj->sha1));
76 - printf(" to %7s %s\n",
77 - refs->item->type, sha1_to_hex(refs->item->sha1));
69 + if (obj->refs) {
70 + const struct object_refs *refs = obj->refs;
71 + unsigned j;
72 + for (j = 0; j < refs->count; j++) {
73 + struct object *ref = refs->ref[j];
74 + if (ref->parsed ||
75 + (!standalone && has_sha1_file(ref->sha1)))
76 + continue;
77 + printf("broken link from %7s %s\n",
78 + obj->type, sha1_to_hex(obj->sha1));
79 + printf(" to %7s %s\n",
80 + ref->type, sha1_to_hex(ref->sha1));
81 + }
82 }
83
84 if (show_unreachable && !(obj->flags & REACHABLE)) {
object.c
+44 -18
@@ -67,40 +67,66 @@ void created_object(const unsigned char *sha1, struct object *obj)
67 nr_objs++;
68 }
69
70 -void add_ref(struct object *refer, struct object *target)
70 +struct object_refs *alloc_object_refs(unsigned count)
71 {
72 - struct object_list **pp, *p;
72 + struct object_refs *refs;
73 + size_t size = sizeof(*refs) + count*sizeof(struct object *);
74
74 - if (!track_object_refs)
75 + refs = xmalloc(size);
76 + memset(refs, 0, size);
77 + refs->count = count;
78 + return refs;
79 +}
80 +
81 +static int compare_object_pointers(const void *a, const void *b)
82 +{
83 + const struct object * const *pa = a;
84 + const struct object * const *pb = b;
85 + return *pa - *pb;
86 +}
87 +
88 +void set_object_refs(struct object *obj, struct object_refs *refs)
89 +{
90 + unsigned int i, j;
91 +
92 + /* Do not install empty list of references */
93 + if (refs->count < 1) {
94 + free(refs);
95 return;
96 + }
97
77 - pp = &refer->refs;
78 - while ((p = *pp) != NULL) {
79 - if (p->item == target)
80 - return;
81 - pp = &p->next;
98 + /* Sort the list and filter out duplicates */
99 + qsort(refs->ref, refs->count, sizeof(refs->ref[0]),
100 + compare_object_pointers);
101 + for (i = j = 1; i < refs->count; i++) {
102 + if (refs->ref[i] != refs->ref[i - 1])
103 + refs->ref[j++] = refs->ref[i];
104 + }
105 + if (j < refs->count) {
106 + /* Duplicates were found - reallocate list */
107 + size_t size = sizeof(*refs) + j*sizeof(struct object *);
108 + refs->count = j;
109 + refs = xrealloc(refs, size);
110 }
111
84 - target->used = 1;
85 - p = xmalloc(sizeof(*p));
86 - p->item = target;
87 - p->next = NULL;
88 - *pp = p;
112 + for (i = 0; i < refs->count; i++)
113 + refs->ref[i]->used = 1;
114 + obj->refs = refs;
115 }
116
117 void mark_reachable(struct object *obj, unsigned int mask)
118 {
93 - struct object_list *p = obj->refs;
94 -
119 if (!track_object_refs)
120 die("cannot do reachability with object refs turned off");
121 /* If we've been here already, don't bother */
122 if (obj->flags & mask)
123 return;
124 obj->flags |= mask;
101 - while (p) {
102 - mark_reachable(p->item, mask);
103 - p = p->next;
125 + if (obj->refs) {
126 + const struct object_refs *refs = obj->refs;
127 + unsigned i;
128 + for (i = 0; i < refs->count; i++)
129 + mark_reachable(refs->ref[i], mask);
130 }
131 }
132
object.h
+8 -2
@@ -7,13 +7,18 @@ struct object_list {
7 const char *name;
8 };
9
10 +struct object_refs {
11 + unsigned count;
12 + struct object *ref[0];
13 +};
14 +
15 struct object {
16 unsigned parsed : 1;
17 unsigned used : 1;
18 unsigned int flags;
19 unsigned char sha1[20];
20 const char *type;
16 - struct object_list *refs;
21 + struct object_refs *refs;
22 void *util;
23 };
24
@@ -35,7 +40,8 @@ struct object *parse_object(const unsigned char *sha1);
40 /** Returns the object, with potentially excess memory allocated. **/
41 struct object *lookup_unknown_object(const unsigned char *sha1);
42
38 -void add_ref(struct object *refer, struct object *target);
43 +struct object_refs *alloc_object_refs(unsigned count);
44 +void set_object_refs(struct object *obj, struct object_refs *refs);
45
46 void mark_reachable(struct object *obj, unsigned int mask);
47
server-info.c
+18 -7
@@ -424,7 +424,6 @@ static void find_pack_info_one(int pack_ix)
424 {
425 unsigned char sha1[20];
426 struct object *o;
427 - struct object_list *ref;
427 int i;
428 struct packed_git *p = info[pack_ix]->p;
429 int num = num_packed_objects(p);
@@ -437,8 +436,12 @@ static void find_pack_info_one(int pack_ix)
436 die("corrupt pack file %s?", p->pack_name);
437 if ((o = lookup_object(sha1)) == NULL)
438 die("cannot parse %s", sha1_to_hex(sha1));
440 - for (ref = o->refs; ref; ref = ref->next)
441 - ref->item->flags = 0;
439 + if (o->refs) {
440 + struct object_refs *refs = o->refs;
441 + int j;
442 + for (j = 0; j < refs->count; j++)
443 + refs->ref[j]->flags = 0;
444 + }
445 o->flags = 0;
446 }
447
@@ -448,8 +451,12 @@ static void find_pack_info_one(int pack_ix)
451 die("corrupt pack file %s?", p->pack_name);
452 if ((o = lookup_object(sha1)) == NULL)
453 die("cannot find %s", sha1_to_hex(sha1));
451 - for (ref = o->refs; ref; ref = ref->next)
452 - ref->item->flags |= REFERENCED;
454 + if (o->refs) {
455 + struct object_refs *refs = o->refs;
456 + int j;
457 + for (j = 0; j < refs->count; j++)
458 + refs->ref[j]->flags |= REFERENCED;
459 + }
460 o->flags |= INTERNAL;
461 }
462
@@ -460,8 +467,12 @@ static void find_pack_info_one(int pack_ix)
467 die("cannot find %s", sha1_to_hex(sha1));
468
469 show(o, pack_ix);
463 - for (ref = o->refs; ref; ref = ref->next)
464 - show(ref->item, pack_ix);
470 + if (o->refs) {
471 + struct object_refs *refs = o->refs;
472 + int j;
473 + for (j = 0; j < refs->count; j++)
474 + show(refs->ref[j], pack_ix);
475 + }
476 }
477
478 }
tag.c
+5 -2
@@ -75,8 +75,11 @@ int parse_tag_buffer(struct tag *item, void *data, unsigned long size)
75 item->tag[taglen] = '\0';
76
77 item->tagged = lookup_object_type(object, type);
78 - if (item->tagged)
79 - add_ref(&item->object, item->tagged);
78 + if (item->tagged && track_object_refs) {
79 + struct object_refs *refs = alloc_object_refs(1);
80 + refs->ref[0] = item->tagged;
81 + set_object_refs(&item->object, refs);
82 + }
83
84 return 0;
85 }
tree.c
+12 -1
@@ -148,6 +148,7 @@ int parse_tree_buffer(struct tree *item, void *buffer, unsigned long size)
148 {
149 void *bufptr = buffer;
150 struct tree_entry_list **list_p;
151 + int n_refs = 0;
152
153 if (item->object.parsed)
154 return 0;
@@ -184,11 +185,21 @@ int parse_tree_buffer(struct tree *item, void *buffer, unsigned long size)
185 obj = &entry->item.blob->object;
186 }
187 if (obj)
187 - add_ref(&item->object, obj);
188 + n_refs++;
189 entry->parent = NULL; /* needs to be filled by the user */
190 *list_p = entry;
191 list_p = &entry->next;
192 }
193 +
194 + if (track_object_refs) {
195 + struct tree_entry_list *entry;
196 + unsigned i = 0;
197 + struct object_refs *refs = alloc_object_refs(n_refs);
198 + for (entry = item->entries; entry; entry = entry->next)
199 + refs->ref[i++] = entry->item.any;
200 + set_object_refs(&item->object, refs);
201 + }
202 +
203 return 0;
204 }
205