@matheusbsbs / matheus-bagliano / commits / 2264dfa5c4
Fork 0

http*: add helper methods for fetching packs

The code handling the fetching of packs in http-push.c and http-walker.c have been refactored into new methods and a new struct (http_pack_request) in http.c. They are not meant to be invoked elsewhere. The new methods in http.c are - new_http_pack_request - finish_http_pack_request - release_http_pack_request and the new struct is http_pack_request. Add a function, new_http_pack_request(), that deals with the details of coming up with the filename to store the retrieved packfile, resuming a previously aborted request, and making a new curl request. Update http-push.c::start_fetch_packed() and http-walker.c::fetch_pack() to use this. Add a function, finish_http_pack_request(), that deals with renaming the pack, advancing the pack list, and installing the pack. Update http-push.c::finish_request() and http-walker.c::fetch_pack to use this. Update release_request() in http-push.c and http-walker.c to invoke release_http_pack_request() to clean up pack request helper data. The local_stream member of the transfer_request struct in http-push.c has been removed, as the packfile pointer will be managed in the struct http_pack_request. Signed-off-by: Tay Ray Chuan <rctay89@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Tay Ray Chuan committed Jun 6, 2009 at 16:44 UTC 2264dfa5c4f11e2b0e2740072208186bee361afd
4 files changed +167 -151
http-push.c
+26 -84
@@ -1,6 +1,5 @@
1 #include "cache.h"
2 #include "commit.h"
3 -#include "pack.h"
3 #include "tag.h"
4 #include "blob.h"
5 #include "http.h"
@@ -119,7 +118,6 @@ struct transfer_request
118 char filename[PATH_MAX];
119 char tmpfile[PATH_MAX];
120 int local_fileno;
122 - FILE *local_stream;
121 enum transfer_state state;
122 CURLcode curl_result;
123 char errorstr[CURL_ERROR_SIZE];
@@ -452,16 +450,10 @@ static void start_mkcol(struct transfer_request *request)
450
451 static void start_fetch_packed(struct transfer_request *request)
452 {
455 - char *url;
453 struct packed_git *target;
457 - FILE *packfile;
458 - char *filename;
459 - long prev_posn = 0;
460 - char range[RANGE_HEADER_SIZE];
461 - struct curl_slist *range_header = NULL;
454
455 struct transfer_request *check_request = request_queue_head;
464 - struct active_request_slot *slot;
456 + struct http_pack_request *preq;
457
458 target = find_sha1_pack(request->obj->sha1, repo->packs);
459 if (!target) {
@@ -474,68 +466,35 @@ static void start_fetch_packed(struct transfer_request *request)
466 fprintf(stderr, "Fetching pack %s\n", sha1_to_hex(target->sha1));
467 fprintf(stderr, " which contains %s\n", sha1_to_hex(request->obj->sha1));
468
477 - filename = sha1_pack_name(target->sha1);
478 - snprintf(request->filename, sizeof(request->filename), "%s", filename);
479 - snprintf(request->tmpfile, sizeof(request->tmpfile),
480 - "%s.temp", filename);
481 -
482 - url = xmalloc(strlen(repo->url) + 64);
483 - sprintf(url, "%sobjects/pack/pack-%s.pack",
484 - repo->url, sha1_to_hex(target->sha1));
469 + preq = new_http_pack_request(target, repo->url);
470 + if (preq == NULL) {
471 + release_http_pack_request(preq);
472 + repo->can_update_info_refs = 0;
473 + return;
474 + }
475 + preq->lst = &repo->packs;
476
477 /* Make sure there isn't another open request for this pack */
478 while (check_request) {
479 if (check_request->state == RUN_FETCH_PACKED &&
489 - !strcmp(check_request->url, url)) {
490 - free(url);
480 + !strcmp(check_request->url, preq->url)) {
481 + release_http_pack_request(preq);
482 release_request(request);
483 return;
484 }
485 check_request = check_request->next;
486 }
487
497 - packfile = fopen(request->tmpfile, "a");
498 - if (!packfile) {
499 - fprintf(stderr, "Unable to open local file %s for pack",
500 - request->tmpfile);
501 - repo->can_update_info_refs = 0;
502 - free(url);
503 - return;
504 - }
505 -
506 - slot = get_active_slot();
507 - slot->callback_func = process_response;
508 - slot->callback_data = request;
509 - request->slot = slot;
510 - request->local_stream = packfile;
511 - request->userData = target;
512 -
513 - request->url = url;
514 - curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
515 - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
516 - curl_easy_setopt(slot->curl, CURLOPT_URL, url);
517 - curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
518 - slot->local = packfile;
519 -
520 - /*
521 - * If there is data present from a previous transfer attempt,
522 - * resume where it left off
523 - */
524 - prev_posn = ftell(packfile);
525 - if (prev_posn>0) {
526 - if (push_verbosely)
527 - fprintf(stderr,
528 - "Resuming fetch of pack %s at byte %ld\n",
529 - sha1_to_hex(target->sha1), prev_posn);
530 - sprintf(range, "Range: bytes=%ld-", prev_posn);
531 - range_header = curl_slist_append(range_header, range);
532 - curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
533 - }
488 + preq->slot->callback_func = process_response;
489 + preq->slot->callback_data = request;
490 + request->slot = preq->slot;
491 + request->userData = preq;
492
493 /* Try to get the request started, abort the request on error */
494 request->state = RUN_FETCH_PACKED;
537 - if (!start_active_slot(slot)) {
495 + if (!start_active_slot(preq->slot)) {
496 fprintf(stderr, "Unable to start GET request\n");
497 + release_http_pack_request(preq);
498 repo->can_update_info_refs = 0;
499 release_request(request);
500 }
@@ -718,8 +677,6 @@ static void release_request(struct transfer_request *request)
677
678 if (request->local_fileno != -1)
679 close(request->local_fileno);
721 - if (request->local_stream)
722 - fclose(request->local_stream);
680 free(request->url);
681 free(request);
682 }
@@ -727,13 +684,10 @@ static void release_request(struct transfer_request *request)
684 static void finish_request(struct transfer_request *request)
685 {
686 struct stat st;
730 - struct packed_git *target;
731 - struct packed_git **lst;
732 - struct active_request_slot *slot;
687 + struct http_pack_request *preq;
688
689 request->curl_result = request->slot->curl_result;
690 request->http_code = request->slot->http_code;
736 - slot = request->slot;
691 request->slot = NULL;
692
693 /* Keep locks active */
@@ -821,31 +775,21 @@ static void finish_request(struct transfer_request *request)
775 start_fetch_packed(request);
776
777 } else if (request->state == RUN_FETCH_PACKED) {
778 + int fail = 1;
779 if (request->curl_result != CURLE_OK) {
780 fprintf(stderr, "Unable to get pack file %s\n%s",
781 request->url, curl_errorstr);
827 - repo->can_update_info_refs = 0;
782 } else {
829 - off_t pack_size = ftell(request->local_stream);
830 -
831 - fclose(request->local_stream);
832 - request->local_stream = NULL;
833 - slot->local = NULL;
834 - if (!move_temp_to_file(request->tmpfile,
835 - request->filename)) {
836 - target = (struct packed_git *)request->userData;
837 - target->pack_size = pack_size;
838 - lst = &repo->packs;
839 - while (*lst != target)
840 - lst = &((*lst)->next);
841 - *lst = (*lst)->next;
842 -
843 - if (!verify_pack(target))
844 - install_packed_git(target);
845 - else
846 - repo->can_update_info_refs = 0;
783 + preq = (struct http_pack_request *)request->userData;
784 +
785 + if (preq) {
786 + if (finish_http_pack_request(preq) > 0)
787 + fail = 0;
788 + release_http_pack_request(preq);
789 }
790 }
791 + if (fail)
792 + repo->can_update_info_refs = 0;
793 release_request(request);
794 }
795 }
@@ -900,7 +844,6 @@ static void add_fetch_request(struct object *obj)
844 request->lock = NULL;
845 request->headers = NULL;
846 request->local_fileno = -1;
903 - request->local_stream = NULL;
847 request->state = NEED_FETCH;
848 request->next = request_queue_head;
849 request_queue_head = request;
@@ -940,7 +883,6 @@ static int add_send_request(struct object *obj, struct remote_lock *lock)
883 request->lock = lock;
884 request->headers = NULL;
885 request->local_fileno = -1;
943 - request->local_stream = NULL;
886 request->state = NEED_PUSH;
887 request->next = request_queue_head;
888 request_queue_head = request;
http-walker.c
+18 -67
@@ -1,6 +1,5 @@
1 #include "cache.h"
2 #include "commit.h"
3 -#include "pack.h"
3 #include "walker.h"
4 #include "http.h"
5
@@ -590,19 +589,10 @@ static int fetch_indices(struct walker *walker, struct alt_base *repo)
589
590 static int fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
591 {
593 - char *url;
592 struct packed_git *target;
595 - struct packed_git **lst;
596 - FILE *packfile;
597 - char *filename;
598 - char tmpfile[PATH_MAX];
593 int ret;
600 - long prev_posn = 0;
601 - char range[RANGE_HEADER_SIZE];
602 - struct curl_slist *range_header = NULL;
603 -
604 - struct active_request_slot *slot;
594 struct slot_results results;
595 + struct http_pack_request *preq;
596
597 if (fetch_indices(walker, repo))
598 return -1;
@@ -617,72 +607,33 @@ static int fetch_pack(struct walker *walker, struct alt_base *repo, unsigned cha
607 sha1_to_hex(sha1));
608 }
609
620 - url = xmalloc(strlen(repo->base) + 65);
621 - sprintf(url, "%s/objects/pack/pack-%s.pack",
622 - repo->base, sha1_to_hex(target->sha1));
610 + preq = new_http_pack_request(target, repo->base);
611 + if (preq == NULL)
612 + goto abort;
613 + preq->lst = &repo->packs;
614 + preq->slot->results = &results;
615
624 - filename = sha1_pack_name(target->sha1);
625 - snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
626 - packfile = fopen(tmpfile, "a");
627 - if (!packfile)
628 - return error("Unable to open local file %s for pack",
629 - tmpfile);
630 -
631 - slot = get_active_slot();
632 - slot->results = &results;
633 - curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
634 - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
635 - curl_easy_setopt(slot->curl, CURLOPT_URL, url);
636 - curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
637 - slot->local = packfile;
638 -
639 - /*
640 - * If there is data present from a previous transfer attempt,
641 - * resume where it left off
642 - */
643 - prev_posn = ftell(packfile);
644 - if (prev_posn>0) {
645 - if (walker->get_verbosely)
646 - fprintf(stderr,
647 - "Resuming fetch of pack %s at byte %ld\n",
648 - sha1_to_hex(target->sha1), prev_posn);
649 - sprintf(range, "Range: bytes=%ld-", prev_posn);
650 - range_header = curl_slist_append(range_header, range);
651 - curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
652 - }
653 -
654 - if (start_active_slot(slot)) {
655 - run_active_slot(slot);
616 + if (start_active_slot(preq->slot)) {
617 + run_active_slot(preq->slot);
618 if (results.curl_result != CURLE_OK) {
657 - fclose(packfile);
658 - slot->local = NULL;
659 - return error("Unable to get pack file %s\n%s", url,
660 - curl_errorstr);
619 + error("Unable to get pack file %s\n%s", preq->url,
620 + curl_errorstr);
621 + goto abort;
622 }
623 } else {
663 - fclose(packfile);
664 - slot->local = NULL;
665 - return error("Unable to start request");
624 + error("Unable to start request");
625 + goto abort;
626 }
627
668 - target->pack_size = ftell(packfile);
669 - fclose(packfile);
670 - slot->local = NULL;
671 -
672 - ret = move_temp_to_file(tmpfile, filename);
628 + ret = finish_http_pack_request(preq);
629 + release_http_pack_request(preq);
630 if (ret)
631 return ret;
632
676 - lst = &repo->packs;
677 - while (*lst != target)
678 - lst = &((*lst)->next);
679 - *lst = (*lst)->next;
680 -
681 - if (verify_pack(target))
682 - return -1;
683 - install_packed_git(target);
684 -
633 return 0;
634 +
635 +abort:
636 + return -1;
637 }
638
639 static void abort_object_request(struct object_request *obj_req)
http.c
+106
@@ -1,4 +1,5 @@
1 #include "http.h"
2 +#include "pack.h"
3
4 int data_received;
5 int active_requests;
@@ -889,3 +890,108 @@ cleanup:
890 free(url);
891 return ret;
892 }
893 +
894 +void release_http_pack_request(struct http_pack_request *preq)
895 +{
896 + if (preq->packfile != NULL) {
897 + fclose(preq->packfile);
898 + preq->packfile = NULL;
899 + preq->slot->local = NULL;
900 + }
901 + if (preq->range_header != NULL) {
902 + curl_slist_free_all(preq->range_header);
903 + preq->range_header = NULL;
904 + }
905 + preq->slot = NULL;
906 + free(preq->url);
907 +}
908 +
909 +int finish_http_pack_request(struct http_pack_request *preq)
910 +{
911 + int ret;
912 + struct packed_git **lst;
913 +
914 + preq->target->pack_size = ftell(preq->packfile);
915 +
916 + if (preq->packfile != NULL) {
917 + fclose(preq->packfile);
918 + preq->packfile = NULL;
919 + preq->slot->local = NULL;
920 + }
921 +
922 + ret = move_temp_to_file(preq->tmpfile, preq->filename);
923 + if (ret)
924 + return ret;
925 +
926 + lst = preq->lst;
927 + while (*lst != preq->target)
928 + lst = &((*lst)->next);
929 + *lst = (*lst)->next;
930 +
931 + if (verify_pack(preq->target))
932 + return -1;
933 + install_packed_git(preq->target);
934 +
935 + return 0;
936 +}
937 +
938 +struct http_pack_request *new_http_pack_request(
939 + struct packed_git *target, const char *base_url)
940 +{
941 + char *url;
942 + char *filename;
943 + long prev_posn = 0;
944 + char range[RANGE_HEADER_SIZE];
945 + struct strbuf buf = STRBUF_INIT;
946 + struct http_pack_request *preq;
947 +
948 + preq = xmalloc(sizeof(*preq));
949 + preq->target = target;
950 + preq->range_header = NULL;
951 +
952 + end_url_with_slash(&buf, base_url);
953 + strbuf_addf(&buf, "objects/pack/pack-%s.pack",
954 + sha1_to_hex(target->sha1));
955 + url = strbuf_detach(&buf, NULL);
956 + preq->url = xstrdup(url);
957 +
958 + filename = sha1_pack_name(target->sha1);
959 + snprintf(preq->filename, sizeof(preq->filename), "%s", filename);
960 + snprintf(preq->tmpfile, sizeof(preq->tmpfile), "%s.temp", filename);
961 + preq->packfile = fopen(preq->tmpfile, "a");
962 + if (!preq->packfile) {
963 + error("Unable to open local file %s for pack",
964 + preq->tmpfile);
965 + goto abort;
966 + }
967 +
968 + preq->slot = get_active_slot();
969 + preq->slot->local = preq->packfile;
970 + curl_easy_setopt(preq->slot->curl, CURLOPT_FILE, preq->packfile);
971 + curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
972 + curl_easy_setopt(preq->slot->curl, CURLOPT_URL, url);
973 + curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER,
974 + no_pragma_header);
975 +
976 + /*
977 + * If there is data present from a previous transfer attempt,
978 + * resume where it left off
979 + */
980 + prev_posn = ftell(preq->packfile);
981 + if (prev_posn>0) {
982 + if (http_is_verbose)
983 + fprintf(stderr,
984 + "Resuming fetch of pack %s at byte %ld\n",
985 + sha1_to_hex(target->sha1), prev_posn);
986 + sprintf(range, "Range: bytes=%ld-", prev_posn);
987 + preq->range_header = curl_slist_append(NULL, range);
988 + curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER,
989 + preq->range_header);
990 + }
991 +
992 + return preq;
993 +
994 +abort:
995 + free(filename);
996 + return NULL;
997 +}
http.h
+17
@@ -150,4 +150,21 @@ extern int http_fetch_ref(const char *base, struct ref *ref);
150 extern int http_get_info_packs(const char *base_url,
151 struct packed_git **packs_head);
152
153 +struct http_pack_request
154 +{
155 + char *url;
156 + struct packed_git *target;
157 + struct packed_git **lst;
158 + FILE *packfile;
159 + char filename[PATH_MAX];
160 + char tmpfile[PATH_MAX];
161 + struct curl_slist *range_header;
162 + struct active_request_slot *slot;
163 +};
164 +
165 +extern struct http_pack_request *new_http_pack_request(
166 + struct packed_git *target, const char *base_url);
167 +extern int finish_http_pack_request(struct http_pack_request *preq);
168 +extern void release_http_pack_request(struct http_pack_request *preq);
169 +
170 #endif /* HTTP_H */