Fix for multiple alternates requests in http-fetch
Stop additional alternates requests from starting if one is already in progress. This adds an optional callback which is processed after a slot has finished running. Signed-off-by: Nick Hengeveld <nickh@reactrix.com> Signed-off-by: Junio C Hamano <junkio@cox.net>
Nick Hengeveld committed
Nov 12, 2005 at 09:11 UTC
acc075a8ad5ee798c170fc2276e1c840a03b5fa4
1 file changed
+117
-61
http-fetch.c
+117
-61
@@ -25,7 +25,7 @@
25
#define PREV_BUF_SIZE 4096
26
#define RANGE_HEADER_SIZE 30
27
28
-static int got_alternates = 0;
28
+static int got_alternates = -1;
29
static int active_requests = 0;
30
static int data_received;
31
@@ -87,9 +87,19 @@ struct active_request_slot
87
int done;
88
CURLcode curl_result;
89
long http_code;
90
+ void *callback_data;
91
+ void (*callback_func)(void *data);
92
struct active_request_slot *next;
93
};
94
95
+struct alt_request {
96
+ char *base;
97
+ char *url;
98
+ struct buffer *buffer;
99
+ struct active_request_slot *slot;
100
+ int http_specific;
101
+};
102
+
103
static struct transfer_request *request_queue_head = NULL;
104
static struct active_request_slot *active_queue_head = NULL;
105
@@ -237,7 +247,7 @@ static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
247
static void process_curl_messages(void);
248
static void process_request_queue(void);
249
#endif
240
-static int fetch_alternates(char *base);
250
+static void fetch_alternates(char *base);
251
252
static CURL* get_curl_handle(void)
253
{
@@ -324,6 +334,8 @@ static struct active_request_slot *get_active_slot(void)
334
slot->in_use = 1;
335
slot->done = 0;
336
slot->local = NULL;
337
+ slot->callback_data = NULL;
338
+ slot->callback_func = NULL;
339
curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
340
curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header);
341
curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
@@ -601,6 +613,12 @@ static void process_curl_messages(void)
613
} else {
614
fprintf(stderr, "Received DONE message for unknown request!\n");
615
}
616
+
617
+ /* Process slot callback if appropriate */
618
+ if (slot->callback_func != NULL) {
619
+ slot->callback_func(slot->callback_data);
620
+ }
621
+
622
if (request != NULL) {
623
request->curl_result = curl_result;
624
request->http_code = slot->http_code;
@@ -766,72 +784,51 @@ static int setup_index(struct alt_base *repo, unsigned char *sha1)
784
return 0;
785
}
786
769
-static int fetch_alternates(char *base)
787
+static void process_alternates(void *callback_data)
788
{
771
- int ret = 0;
772
- struct buffer buffer;
773
- char *url;
774
- char *data;
775
- int i = 0;
776
- int http_specific = 1;
789
+ struct alt_request *alt_req = (struct alt_request *)callback_data;
790
+ struct active_request_slot *slot = alt_req->slot;
791
struct alt_base *tail = alt;
792
+ char *base = alt_req->base;
793
static const char null_byte = '\0';
794
+ char *data;
795
+ int i = 0;
796
780
- struct active_request_slot *slot;
781
-
782
- if (got_alternates)
783
- return 0;
784
-
785
- data = xmalloc(4096);
786
- buffer.size = 4096;
787
- buffer.posn = 0;
788
- buffer.buffer = data;
789
-
790
- if (get_verbosely)
791
- fprintf(stderr, "Getting alternates list for %s\n", base);
792
-
793
- url = xmalloc(strlen(base) + 31);
794
- sprintf(url, "%s/objects/info/http-alternates", base);
795
-
796
- slot = get_active_slot();
797
- curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
798
- curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
799
- fwrite_buffer_dynamic);
800
- curl_easy_setopt(slot->curl, CURLOPT_URL, url);
801
- if (start_active_slot(slot)) {
802
- run_active_slot(slot);
803
- if (slot->curl_result != CURLE_OK || !buffer.posn) {
804
- http_specific = 0;
805
-
806
- sprintf(url, "%s/objects/info/alternates", base);
807
-
808
- slot = get_active_slot();
809
- curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
810
- curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
811
- fwrite_buffer_dynamic);
812
- curl_easy_setopt(slot->curl, CURLOPT_URL, url);
797
+ if (alt_req->http_specific) {
798
+ if (slot->curl_result != CURLE_OK ||
799
+ !alt_req->buffer->posn) {
800
+
801
+ /* Try reusing the slot to get non-http alternates */
802
+ alt_req->http_specific = 0;
803
+ sprintf(alt_req->url, "%s/objects/info/alternates",
804
+ base);
805
+ curl_easy_setopt(slot->curl, CURLOPT_URL,
806
+ alt_req->url);
807
+ active_requests++;
808
+ slot->in_use = 1;
809
+ slot->done = 0;
810
if (start_active_slot(slot)) {
814
- run_active_slot(slot);
815
- if (slot->curl_result != CURLE_OK) {
816
- free(buffer.buffer);
817
- if (slot->http_code == 404)
818
- got_alternates = 1;
819
- return 0;
820
- }
811
+ return;
812
+ } else {
813
+ got_alternates = -1;
814
+ slot->done = 1;
815
+ return;
816
}
817
}
823
- } else {
824
- free(buffer.buffer);
825
- return 0;
818
+ } else if (slot->curl_result != CURLE_OK) {
819
+ if (slot->http_code != 404) {
820
+ got_alternates = -1;
821
+ return;
822
+ }
823
}
824
828
- fwrite_buffer_dynamic(&null_byte, 1, 1, &buffer);
829
- buffer.posn--;
830
- data = buffer.buffer;
825
+ fwrite_buffer_dynamic(&null_byte, 1, 1, alt_req->buffer);
826
+ alt_req->buffer->posn--;
827
+ data = alt_req->buffer->buffer;
828
832
- while (i < buffer.posn) {
829
+ while (i < alt_req->buffer->posn) {
830
int posn = i;
834
- while (posn < buffer.posn && data[posn] != '\n')
831
+ while (posn < alt_req->buffer->posn && data[posn] != '\n')
832
posn++;
833
if (data[posn] == '\n') {
834
int okay = 0;
@@ -855,7 +852,7 @@ static int fetch_alternates(char *base)
852
// If the server got removed, give up.
853
okay = strchr(base, ':') - base + 3 <
854
serverlen;
858
- } else if (http_specific) {
855
+ } else if (alt_req->http_specific) {
856
char *colon = strchr(data + i, ':');
857
char *slash = strchr(data + i, '/');
858
if (colon && slash && colon < data + posn &&
@@ -881,15 +878,74 @@ static int fetch_alternates(char *base)
878
while (tail->next != NULL)
879
tail = tail->next;
880
tail->next = newalt;
884
- ret++;
881
}
882
}
883
i = posn + 1;
884
}
885
886
got_alternates = 1;
891
- free(buffer.buffer);
892
- return ret;
887
+}
888
+
889
+static void fetch_alternates(char *base)
890
+{
891
+ struct buffer buffer;
892
+ char *url;
893
+ char *data;
894
+ struct active_request_slot *slot;
895
+ static struct alt_request alt_req;
896
+ int num_transfers;
897
+
898
+ /* If another request has already started fetching alternates,
899
+ wait for them to arrive and return to processing this request's
900
+ curl message */
901
+ while (got_alternates == 0) {
902
+ curl_multi_perform(curlm, &num_transfers);
903
+ process_curl_messages();
904
+ process_request_queue();
905
+ }
906
+
907
+ /* Nothing to do if they've already been fetched */
908
+ if (got_alternates == 1)
909
+ return;
910
+
911
+ /* Start the fetch */
912
+ got_alternates = 0;
913
+
914
+ data = xmalloc(4096);
915
+ buffer.size = 4096;
916
+ buffer.posn = 0;
917
+ buffer.buffer = data;
918
+
919
+ if (get_verbosely)
920
+ fprintf(stderr, "Getting alternates list for %s\n", base);
921
+
922
+ url = xmalloc(strlen(base) + 31);
923
+ sprintf(url, "%s/objects/info/http-alternates", base);
924
+
925
+ /* Use a callback to process the result, since another request
926
+ may fail and need to have alternates loaded before continuing */
927
+ slot = get_active_slot();
928
+ slot->callback_func = process_alternates;
929
+ slot->callback_data = &alt_req;
930
+
931
+ curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
932
+ curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
933
+ fwrite_buffer_dynamic);
934
+ curl_easy_setopt(slot->curl, CURLOPT_URL, url);
935
+
936
+ alt_req.base = base;
937
+ alt_req.url = url;
938
+ alt_req.buffer = &buffer;
939
+ alt_req.http_specific = 1;
940
+ alt_req.slot = slot;
941
+
942
+ if (start_active_slot(slot))
943
+ run_active_slot(slot);
944
+ else
945
+ got_alternates = -1;
946
+
947
+ free(data);
948
+ free(url);
949
}
950
951
static int fetch_indices(struct alt_base *repo)