@matheusbsbs / matheus-bagliano / commits / 85d106c267
Fork 0

Improve the safety check used in fetch.c

The recent safety check to trust only the commits we have made things impossibly slow and turn out to waste a lot of memory. This commit fixes it with the following improvements: - mark already scanned objects and avoid rescanning the same object again; - free the tree entries when we have scanned the tree entries; this is the same as b0d8923ec01fd91b75ab079034f89ced91500157 which reduced memory usage by rev-list; - plug memory leak from the object_list dequeuing code; - use the process_queue not just for fetching but for scanning, to make things tail recursive to avoid deep recursion; the deep recursion was especially prominent when we cloned a big pack. - avoid has_sha1_file() call when we already know we do not have that object. Signed-off-by: Junio C Hamano <junkio@cox.net>

Junio C Hamano committed Sep 18, 2005 at 01:01 UTC 85d106c267ec26f398e0aaf352d8011f661c459a
1 file changed +44 -32
fetch.c
+44 -32
@@ -33,36 +33,33 @@ static void report_missing(const char *what, const unsigned char *missing)
33 what, missing_hex, sha1_to_hex(current_commit_sha1));
34 }
35
36 -static int make_sure_we_have_it(const char *what, unsigned char *sha1)
37 -{
38 - int status = 0;
39 -
40 - if (!has_sha1_file(sha1)) {
41 - status = fetch(sha1);
42 - if (status && what)
43 - report_missing(what, sha1);
44 - }
45 - return status;
46 -}
47 -
36 static int process(unsigned char *sha1, const char *type);
37
38 static int process_tree(struct tree *tree)
39 {
52 - struct tree_entry_list *entries;
40 + struct tree_entry_list *entry;
41
42 if (parse_tree(tree))
43 return -1;
44
57 - for (entries = tree->entries; entries; entries = entries->next) {
58 - if (process(entries->item.any->sha1,
59 - entries->directory ? tree_type : blob_type))
45 + entry = tree->entries;
46 + tree->entries = NULL;
47 + while (entry) {
48 + struct tree_entry_list *next = entry->next;
49 + if (process(entry->item.any->sha1,
50 + entry->directory ? tree_type : blob_type))
51 return -1;
52 + free(entry);
53 + entry = next;
54 }
55 return 0;
56 }
57
58 #define COMPLETE 1U
59 +#define TO_FETCH 2U
60 +#define TO_SCAN 4U
61 +#define SCANNED 8U
62 +
63 static struct commit_list *complete = NULL;
64
65 static int process_commit(struct commit *commit)
@@ -73,13 +70,14 @@ static int process_commit(struct commit *commit)
70 while (complete && complete->item->date >= commit->date) {
71 pop_most_recent_commit(&complete, COMPLETE);
72 }
76 -
73
74 if (commit->object.flags & COMPLETE)
75 return 0;
76
77 memcpy(current_commit_sha1, commit->object.sha1, 20);
78
79 + pull_say("walk %s\n", sha1_to_hex(commit->object.sha1));
80 +
81 if (get_tree) {
82 if (process(commit->tree->object.sha1, tree_type))
83 return -1;
@@ -89,8 +87,7 @@ static int process_commit(struct commit *commit)
87 if (get_history) {
88 struct commit_list *parents = commit->parents;
89 for (; parents; parents = parents->next) {
92 - if (process(parents->item->object.sha1,
93 - commit_type))
90 + if (process(parents->item->object.sha1, commit_type))
91 return -1;
92 }
93 }
@@ -109,6 +106,10 @@ static struct object_list **process_queue_end = &process_queue;
106
107 static int process_object(struct object *obj)
108 {
109 + if (obj->flags & SCANNED)
110 + return 0;
111 + obj->flags |= SCANNED;
112 +
113 if (obj->type == commit_type) {
114 if (process_commit((struct commit *)obj))
115 return -1;
@@ -139,14 +140,19 @@ static int process(unsigned char *sha1, const char *type)
140 if (has_sha1_file(sha1)) {
141 parse_object(sha1);
142 /* We already have it, so we should scan it now. */
142 - return process_object(obj);
143 + if (obj->flags & (SCANNED | TO_SCAN))
144 + return 0;
145 + object_list_insert(obj, process_queue_end);
146 + process_queue_end = &(*process_queue_end)->next;
147 + obj->flags |= TO_SCAN;
148 + return 0;
149 }
144 - if (object_list_contains(process_queue, obj))
150 + if (obj->flags & (COMPLETE | TO_FETCH))
151 return 0;
152 object_list_insert(obj, process_queue_end);
153 process_queue_end = &(*process_queue_end)->next;
154 + obj->flags |= TO_FETCH;
155
149 - //fprintf(stderr, "prefetch %s\n", sha1_to_hex(sha1));
156 prefetch(sha1);
157
158 return 0;
@@ -154,21 +160,27 @@ static int process(unsigned char *sha1, const char *type)
160
161 static int loop(void)
162 {
163 + struct object_list *elem;
164 +
165 while (process_queue) {
166 struct object *obj = process_queue->item;
159 - /*
160 - fprintf(stderr, "%d objects to pull\n",
161 - object_list_length(process_queue));
162 - */
163 - process_queue = process_queue->next;
167 + elem = process_queue;
168 + process_queue = elem->next;
169 + free(elem);
170 if (!process_queue)
171 process_queue_end = &process_queue;
172
167 - //fprintf(stderr, "fetch %s\n", sha1_to_hex(obj->sha1));
168 -
169 - if (make_sure_we_have_it(obj->type ? obj->type : "object",
170 - obj->sha1))
171 - return -1;
173 + /* If we are not scanning this object, we placed it in
174 + * the queue because we needed to fetch it first.
175 + */
176 + if (! (obj->flags & TO_SCAN)) {
177 + if (fetch(obj->sha1)) {
178 + report_missing(obj->type
179 + ? obj->type
180 + : "object", obj->sha1);
181 + return -1;
182 + }
183 + }
184 if (!obj->type)
185 parse_object(obj->sha1);
186 if (process_object(obj))