@matheusbsbs / matheus-bagliano / commits / e388ab74db
Fork 0

Make http-fetch request types more clear

Rename object request functions and data to make it more clear which type of request is being processed - this is a response to the introduction of slot callbacks and the definition of different types of requests such as alternates_request. Signed-off-by: Nick Hengeveld <nickh@reactrix.com> Signed-off-by: Junio C Hamano <junkio@cox.net>

Nick Hengeveld committed Nov 18, 2005 at 11:03 UTC e388ab74dbaea5cdbf07b175be8d6980579392f4
1 file changed +139 -136
http-fetch.c
+139 -136
@@ -21,14 +21,14 @@ struct alt_base
21
22 static struct alt_base *alt = NULL;
23
24 -enum transfer_state {
24 +enum object_request_state {
25 WAITING,
26 ABORTED,
27 ACTIVE,
28 COMPLETE,
29 };
30
31 -struct transfer_request
31 +struct object_request
32 {
33 unsigned char sha1[20];
34 struct alt_base *repo;
@@ -36,7 +36,7 @@ struct transfer_request
36 char filename[PATH_MAX];
37 char tmpfile[PATH_MAX];
38 int local;
39 - enum transfer_state state;
39 + enum object_request_state state;
40 CURLcode curl_result;
41 char errorstr[CURL_ERROR_SIZE];
42 long http_code;
@@ -46,10 +46,10 @@ struct transfer_request
46 int zret;
47 int rename;
48 struct active_request_slot *slot;
49 - struct transfer_request *next;
49 + struct object_request *next;
50 };
51
52 -struct alt_request {
52 +struct alternates_request {
53 char *base;
54 char *url;
55 struct buffer *buffer;
@@ -57,7 +57,7 @@ struct alt_request {
57 int http_specific;
58 };
59
60 -static struct transfer_request *request_queue_head = NULL;
60 +static struct object_request *object_queue_head = NULL;
61
62 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
63 void *data)
@@ -65,24 +65,24 @@ static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
65 unsigned char expn[4096];
66 size_t size = eltsize * nmemb;
67 int posn = 0;
68 - struct transfer_request *request = (struct transfer_request *)data;
68 + struct object_request *obj_req = (struct object_request *)data;
69 do {
70 - ssize_t retval = write(request->local,
70 + ssize_t retval = write(obj_req->local,
71 ptr + posn, size - posn);
72 if (retval < 0)
73 return posn;
74 posn += retval;
75 } while (posn < size);
76
77 - request->stream.avail_in = size;
78 - request->stream.next_in = ptr;
77 + obj_req->stream.avail_in = size;
78 + obj_req->stream.next_in = ptr;
79 do {
80 - request->stream.next_out = expn;
81 - request->stream.avail_out = sizeof(expn);
82 - request->zret = inflate(&request->stream, Z_SYNC_FLUSH);
83 - SHA1_Update(&request->c, expn,
84 - sizeof(expn) - request->stream.avail_out);
85 - } while (request->stream.avail_in && request->zret == Z_OK);
80 + obj_req->stream.next_out = expn;
81 + obj_req->stream.avail_out = sizeof(expn);
82 + obj_req->zret = inflate(&obj_req->stream, Z_SYNC_FLUSH);
83 + SHA1_Update(&obj_req->c, expn,
84 + sizeof(expn) - obj_req->stream.avail_out);
85 + } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
86 data_received++;
87 return size;
88 }
@@ -91,9 +91,9 @@ static void fetch_alternates(char *base);
91
92 static void process_object_response(void *callback_data);
93
94 -static void start_request(struct transfer_request *request)
94 +static void start_object_request(struct object_request *obj_req)
95 {
96 - char *hex = sha1_to_hex(request->sha1);
96 + char *hex = sha1_to_hex(obj_req->sha1);
97 char prevfile[PATH_MAX];
98 char *url;
99 char *posn;
@@ -105,53 +105,53 @@ static void start_request(struct transfer_request *request)
105 struct curl_slist *range_header = NULL;
106 struct active_request_slot *slot;
107
108 - snprintf(prevfile, sizeof(prevfile), "%s.prev", request->filename);
108 + snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
109 unlink(prevfile);
110 - rename(request->tmpfile, prevfile);
111 - unlink(request->tmpfile);
110 + rename(obj_req->tmpfile, prevfile);
111 + unlink(obj_req->tmpfile);
112
113 - if (request->local != -1)
114 - error("fd leakage in start: %d", request->local);
115 - request->local = open(request->tmpfile,
113 + if (obj_req->local != -1)
114 + error("fd leakage in start: %d", obj_req->local);
115 + obj_req->local = open(obj_req->tmpfile,
116 O_WRONLY | O_CREAT | O_EXCL, 0666);
117 /* This could have failed due to the "lazy directory creation";
118 * try to mkdir the last path component.
119 */
120 - if (request->local < 0 && errno == ENOENT) {
121 - char *dir = strrchr(request->tmpfile, '/');
120 + if (obj_req->local < 0 && errno == ENOENT) {
121 + char *dir = strrchr(obj_req->tmpfile, '/');
122 if (dir) {
123 *dir = 0;
124 - mkdir(request->tmpfile, 0777);
124 + mkdir(obj_req->tmpfile, 0777);
125 *dir = '/';
126 }
127 - request->local = open(request->tmpfile,
127 + obj_req->local = open(obj_req->tmpfile,
128 O_WRONLY | O_CREAT | O_EXCL, 0666);
129 }
130
131 - if (request->local < 0) {
132 - request->state = ABORTED;
131 + if (obj_req->local < 0) {
132 + obj_req->state = ABORTED;
133 error("Couldn't create temporary file %s for %s: %s\n",
134 - request->tmpfile, request->filename, strerror(errno));
134 + obj_req->tmpfile, obj_req->filename, strerror(errno));
135 return;
136 }
137
138 - memset(&request->stream, 0, sizeof(request->stream));
138 + memset(&obj_req->stream, 0, sizeof(obj_req->stream));
139
140 - inflateInit(&request->stream);
140 + inflateInit(&obj_req->stream);
141
142 - SHA1_Init(&request->c);
142 + SHA1_Init(&obj_req->c);
143
144 - url = xmalloc(strlen(request->repo->base) + 50);
145 - request->url = xmalloc(strlen(request->repo->base) + 50);
146 - strcpy(url, request->repo->base);
147 - posn = url + strlen(request->repo->base);
144 + url = xmalloc(strlen(obj_req->repo->base) + 50);
145 + obj_req->url = xmalloc(strlen(obj_req->repo->base) + 50);
146 + strcpy(url, obj_req->repo->base);
147 + posn = url + strlen(obj_req->repo->base);
148 strcpy(posn, "objects/");
149 posn += 8;
150 memcpy(posn, hex, 2);
151 posn += 2;
152 *(posn++) = '/';
153 strcpy(posn, hex + 2);
154 - strcpy(request->url, url);
154 + strcpy(obj_req->url, url);
155
156 /* If a previous temp file is present, process what was already
157 fetched. */
@@ -163,7 +163,7 @@ static void start_request(struct transfer_request *request)
163 if (fwrite_sha1_file(prev_buf,
164 1,
165 prev_read,
166 - request) == prev_read) {
166 + obj_req) == prev_read) {
167 prev_posn += prev_read;
168 } else {
169 prev_read = -1;
@@ -177,24 +177,24 @@ static void start_request(struct transfer_request *request)
177 /* Reset inflate/SHA1 if there was an error reading the previous temp
178 file; also rewind to the beginning of the local file. */
179 if (prev_read == -1) {
180 - memset(&request->stream, 0, sizeof(request->stream));
181 - inflateInit(&request->stream);
182 - SHA1_Init(&request->c);
180 + memset(&obj_req->stream, 0, sizeof(obj_req->stream));
181 + inflateInit(&obj_req->stream);
182 + SHA1_Init(&obj_req->c);
183 if (prev_posn>0) {
184 prev_posn = 0;
185 - lseek(request->local, SEEK_SET, 0);
186 - ftruncate(request->local, 0);
185 + lseek(obj_req->local, SEEK_SET, 0);
186 + ftruncate(obj_req->local, 0);
187 }
188 }
189
190 slot = get_active_slot();
191 slot->callback_func = process_object_response;
192 - slot->callback_data = request;
193 - request->slot = slot;
192 + slot->callback_data = obj_req;
193 + obj_req->slot = slot;
194
195 - curl_easy_setopt(slot->curl, CURLOPT_FILE, request);
195 + curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
196 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
197 - curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr);
197 + curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
198 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
199 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
200
@@ -212,109 +212,111 @@ static void start_request(struct transfer_request *request)
212 }
213
214 /* Try to get the request started, abort the request on error */
215 - request->state = ACTIVE;
215 + obj_req->state = ACTIVE;
216 if (!start_active_slot(slot)) {
217 - request->state = ABORTED;
218 - request->slot = NULL;
219 - close(request->local); request->local = -1;
220 - free(request->url);
217 + obj_req->state = ABORTED;
218 + obj_req->slot = NULL;
219 + close(obj_req->local); obj_req->local = -1;
220 + free(obj_req->url);
221 + return;
222 }
223 +
224 }
225
224 -static void finish_request(struct transfer_request *request)
226 +static void finish_object_request(struct object_request *obj_req)
227 {
228 struct stat st;
229
228 - fchmod(request->local, 0444);
229 - close(request->local); request->local = -1;
230 + fchmod(obj_req->local, 0444);
231 + close(obj_req->local); obj_req->local = -1;
232
231 - if (request->http_code == 416) {
233 + if (obj_req->http_code == 416) {
234 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
233 - } else if (request->curl_result != CURLE_OK) {
234 - if (stat(request->tmpfile, &st) == 0)
235 + } else if (obj_req->curl_result != CURLE_OK) {
236 + if (stat(obj_req->tmpfile, &st) == 0)
237 if (st.st_size == 0)
236 - unlink(request->tmpfile);
238 + unlink(obj_req->tmpfile);
239 return;
240 }
241
240 - inflateEnd(&request->stream);
241 - SHA1_Final(request->real_sha1, &request->c);
242 - if (request->zret != Z_STREAM_END) {
243 - unlink(request->tmpfile);
242 + inflateEnd(&obj_req->stream);
243 + SHA1_Final(obj_req->real_sha1, &obj_req->c);
244 + if (obj_req->zret != Z_STREAM_END) {
245 + unlink(obj_req->tmpfile);
246 return;
247 }
246 - if (memcmp(request->sha1, request->real_sha1, 20)) {
247 - unlink(request->tmpfile);
248 + if (memcmp(obj_req->sha1, obj_req->real_sha1, 20)) {
249 + unlink(obj_req->tmpfile);
250 return;
251 }
250 - request->rename =
251 - move_temp_to_file(request->tmpfile, request->filename);
252 + obj_req->rename =
253 + move_temp_to_file(obj_req->tmpfile, obj_req->filename);
254
253 - if (request->rename == 0)
254 - pull_say("got %s\n", sha1_to_hex(request->sha1));
255 + if (obj_req->rename == 0)
256 + pull_say("got %s\n", sha1_to_hex(obj_req->sha1));
257 }
258
259 static void process_object_response(void *callback_data)
260 {
259 - struct transfer_request *request =
260 - (struct transfer_request *)callback_data;
261 + struct object_request *obj_req =
262 + (struct object_request *)callback_data;
263
262 - request->curl_result = request->slot->curl_result;
263 - request->http_code = request->slot->http_code;
264 - request->slot = NULL;
265 - request->state = COMPLETE;
264 + obj_req->curl_result = obj_req->slot->curl_result;
265 + obj_req->http_code = obj_req->slot->http_code;
266 + obj_req->slot = NULL;
267 + obj_req->state = COMPLETE;
268
269 /* Use alternates if necessary */
268 - if (request->http_code == 404) {
270 + if (obj_req->http_code == 404) {
271 fetch_alternates(alt->base);
270 - if (request->repo->next != NULL) {
271 - request->repo =
272 - request->repo->next;
273 - close(request->local);
274 - request->local = -1;
275 - start_request(request);
272 + if (obj_req->repo->next != NULL) {
273 + obj_req->repo =
274 + obj_req->repo->next;
275 + close(obj_req->local);
276 + obj_req->local = -1;
277 + start_object_request(obj_req);
278 return;
279 }
280 }
281
280 - finish_request(request);
282 + finish_object_request(obj_req);
283 }
284
283 -static void release_request(struct transfer_request *request)
285 +static void release_object_request(struct object_request *obj_req)
286 {
285 - struct transfer_request *entry = request_queue_head;
287 + struct object_request *entry = object_queue_head;
288
287 - if (request->local != -1)
288 - error("fd leakage in release: %d", request->local);
289 - if (request == request_queue_head) {
290 - request_queue_head = request->next;
289 + if (obj_req->local != -1)
290 + error("fd leakage in release: %d", obj_req->local);
291 + if (obj_req == object_queue_head) {
292 + object_queue_head = obj_req->next;
293 } else {
292 - while (entry->next != NULL && entry->next != request)
294 + while (entry->next != NULL && entry->next != obj_req)
295 entry = entry->next;
294 - if (entry->next == request)
296 + if (entry->next == obj_req)
297 entry->next = entry->next->next;
298 }
299
298 - free(request->url);
299 - free(request);
300 + free(obj_req->url);
301 + free(obj_req);
302 }
303
304 #ifdef USE_CURL_MULTI
305 void fill_active_slots(void)
306 {
305 - struct transfer_request *request = request_queue_head;
307 + struct object_request *obj_req = object_queue_head;
308 struct active_request_slot *slot = active_queue_head;
309 int num_transfers;
310
309 - while (active_requests < max_requests && request != NULL) {
310 - if (request->state == WAITING) {
311 - if (has_sha1_file(request->sha1))
312 - release_request(request);
311 + while (active_requests < max_requests && obj_req != NULL) {
312 + if (obj_req->state == WAITING) {
313 + if (has_sha1_file(obj_req->sha1))
314 + release_object_request(obj_req);
315 else
314 - start_request(request);
316 + start_object_request(obj_req);
317 curl_multi_perform(curlm, &num_transfers);
318 }
317 - request = request->next;
319 + obj_req = obj_req->next;
320 }
321
322 while (slot != NULL) {
@@ -329,8 +331,8 @@ void fill_active_slots(void)
331
332 void prefetch(unsigned char *sha1)
333 {
332 - struct transfer_request *newreq;
333 - struct transfer_request *tail;
334 + struct object_request *newreq;
335 + struct object_request *tail;
336 char *filename = sha1_file_name(sha1);
337
338 newreq = xmalloc(sizeof(*newreq));
@@ -344,10 +346,10 @@ void prefetch(unsigned char *sha1)
346 "%s.temp", filename);
347 newreq->next = NULL;
348
347 - if (request_queue_head == NULL) {
348 - request_queue_head = newreq;
349 + if (object_queue_head == NULL) {
350 + object_queue_head = newreq;
351 } else {
350 - tail = request_queue_head;
352 + tail = object_queue_head;
353 while (tail->next != NULL) {
354 tail = tail->next;
355 }
@@ -441,9 +443,10 @@ static int setup_index(struct alt_base *repo, unsigned char *sha1)
443 return 0;
444 }
445
444 -static void process_alternates(void *callback_data)
446 +static void process_alternates_response(void *callback_data)
447 {
446 - struct alt_request *alt_req = (struct alt_request *)callback_data;
448 + struct alternates_request *alt_req =
449 + (struct alternates_request *)callback_data;
450 struct active_request_slot *slot = alt_req->slot;
451 struct alt_base *tail = alt;
452 char *base = alt_req->base;
@@ -548,7 +551,7 @@ static void fetch_alternates(char *base)
551 char *url;
552 char *data;
553 struct active_request_slot *slot;
551 - static struct alt_request alt_req;
554 + static struct alternates_request alt_req;
555
556 /* If another request has already started fetching alternates,
557 wait for them to arrive and return to processing this request's
@@ -580,7 +583,7 @@ static void fetch_alternates(char *base)
583 /* Use a callback to process the result, since another request
584 may fail and need to have alternates loaded before continuing */
585 slot = get_active_slot();
583 - slot->callback_func = process_alternates;
586 + slot->callback_func = process_alternates_response;
587 slot->callback_data = &alt_req;
588
589 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
@@ -760,54 +763,54 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1)
763 {
764 char *hex = sha1_to_hex(sha1);
765 int ret = 0;
763 - struct transfer_request *request = request_queue_head;
766 + struct object_request *obj_req = object_queue_head;
767
765 - while (request != NULL && memcmp(request->sha1, sha1, 20))
766 - request = request->next;
767 - if (request == NULL)
768 + while (obj_req != NULL && memcmp(obj_req->sha1, sha1, 20))
769 + obj_req = obj_req->next;
770 + if (obj_req == NULL)
771 return error("Couldn't find request for %s in the queue", hex);
772
770 - if (has_sha1_file(request->sha1)) {
771 - release_request(request);
773 + if (has_sha1_file(obj_req->sha1)) {
774 + release_object_request(obj_req);
775 return 0;
776 }
777
778 #ifdef USE_CURL_MULTI
776 - while (request->state == WAITING) {
779 + while (obj_req->state == WAITING) {
780 step_active_slots();
781 }
782 #else
780 - start_request(request);
783 + start_object_request(obj_req);
784 #endif
785
783 - while (request->state == ACTIVE) {
784 - run_active_slot(request->slot);
786 + while (obj_req->state == ACTIVE) {
787 + run_active_slot(obj_req->slot);
788 }
786 - if (request->local != -1) {
787 - close(request->local); request->local = -1;
789 + if (obj_req->local != -1) {
790 + close(obj_req->local); obj_req->local = -1;
791 }
792
790 - if (request->state == ABORTED) {
793 + if (obj_req->state == ABORTED) {
794 ret = error("Request for %s aborted", hex);
792 - } else if (request->curl_result != CURLE_OK &&
793 - request->http_code != 416) {
794 - if (request->http_code == 404)
795 + } else if (obj_req->curl_result != CURLE_OK &&
796 + obj_req->http_code != 416) {
797 + if (obj_req->http_code == 404)
798 ret = -1; /* Be silent, it is probably in a pack. */
799 else
800 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
798 - request->errorstr, request->curl_result,
799 - request->http_code, hex);
800 - } else if (request->zret != Z_STREAM_END) {
801 - ret = error("File %s (%s) corrupt\n", hex, request->url);
802 - } else if (memcmp(request->sha1, request->real_sha1, 20)) {
801 + obj_req->errorstr, obj_req->curl_result,
802 + obj_req->http_code, hex);
803 + } else if (obj_req->zret != Z_STREAM_END) {
804 + ret = error("File %s (%s) corrupt\n", hex, obj_req->url);
805 + } else if (memcmp(obj_req->sha1, obj_req->real_sha1, 20)) {
806 ret = error("File %s has bad hash\n", hex);
804 - } else if (request->rename < 0) {
807 + } else if (obj_req->rename < 0) {
808 ret = error("unable to write sha1 filename %s: %s",
806 - request->filename,
807 - strerror(request->rename));
809 + obj_req->filename,
810 + strerror(obj_req->rename));
811 }
812
810 - release_request(request);
813 + release_object_request(obj_req);
814 return ret;
815 }
816