@matheusbsbs / matheus-bagliano / commits / 5424bc557f

Fork 0

http*: add helper methods for fetching objects (loose)

The code handling the fetching of loose objects in http-push.c and http-walker.c have been refactored into new methods and a new struct (object_http_request) in http.c. They are not meant to be invoked elsewhere. The new methods in http.c are - new_http_object_request - process_http_object_request - finish_http_object_request - abort_http_object_request - release_http_object_request and the new struct is http_object_request. RANGER_HEADER_SIZE and no_pragma_header is no longer made available outside of http.c, since after the above changes, there are no other instances of usage outside of http.c. Remove members of the transfer_request struct in http-push.c and http-walker.c, including filename, real_sha1 and zret, as they are used no longer used. Move the methods append_remote_object_url() and get_remote_object_url() from http-push.c to http.c. Additionally, get_remote_object_url() is no longer defined only when USE_CURL_MULTI is defined, since non-USE_CURL_MULTI code in http.c uses it (namely, in new_http_object_request()). Refactor code from http-push.c::start_fetch_loose() and http-walker.c::start_object_fetch_request() that deals with the details of coming up with the filename to store the retrieved object, resuming a previously aborted request, and making a new curl request, into a new function, new_http_object_request(). Refactor code from http-walker.c::process_object_request() into the function, process_http_object_request(). Refactor code from http-push.c::finish_request() and http-walker.c::finish_object_request() into a new function, finish_http_object_request(). It returns the result of the move_temp_to_file() invocation. Add a function, release_http_object_request(), which cleans up object request data. http-push.c and http-walker.c invoke this function separately; http-push.c::release_request() and http-walker.c::release_object_request() do not invoke this function. Add a function, abort_http_object_request(), which unlink()s the object file and invokes release_http_object_request(). Update http-walker.c::abort_object_request() to use this. Signed-off-by: Tay Ray Chuan <rctay89@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Tay Ray Chuan committed Jun 6, 2009 at 16:44 UTC 5424bc557fc6414660830b470dd45774b8f5f281

4 files changed +336 -431

http-push.c

+15 -198

@@ -115,18 +115,10 @@ struct transfer_request

115 struct remote_lock *lock;

116 struct curl_slist *headers;

117 struct buffer buffer;

118 - char filename[PATH_MAX];

119 - char tmpfile[PATH_MAX];

120 - int local_fileno;

118 enum transfer_state state;

119 CURLcode curl_result;

120 char errorstr[CURL_ERROR_SIZE];

121 long http_code;

125 - unsigned char real_sha1[20];

126 - git_SHA_CTX c;

127 - z_stream stream;

128 - int zret;

129 - int rename;

122 void *userData;

123 struct active_request_slot *slot;

124 struct transfer_request *next;

@@ -232,15 +224,6 @@ static struct curl_slist *get_dav_token_headers(struct remote_lock *lock, enum d

224 return dav_headers;

225 }

226

235 -static void append_remote_object_url(struct strbuf *buf, const char *url,

236 - const char *hex,

237 - int only_two_digit_prefix)

238 -{

239 - strbuf_addf(buf, "%sobjects/%.*s/", url, 2, hex);

240 - if (!only_two_digit_prefix)

241 - strbuf_addf(buf, "%s", hex+2);

242 -}

243 -

227 static void finish_request(struct transfer_request *request);

228 static void release_request(struct transfer_request *request);

229

@@ -254,169 +237,29 @@ static void process_response(void *callback_data)

237

238 #ifdef USE_CURL_MULTI

239

257 -static char *get_remote_object_url(const char *url, const char *hex,

258 - int only_two_digit_prefix)

259 -{

260 - struct strbuf buf = STRBUF_INIT;

261 - append_remote_object_url(&buf, url, hex, only_two_digit_prefix);

262 - return strbuf_detach(&buf, NULL);

263 -}

264 -

265 -static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,

266 - void *data)

267 -{

268 - unsigned char expn[4096];

269 - size_t size = eltsize * nmemb;

270 - int posn = 0;

271 - struct transfer_request *request = (struct transfer_request *)data;

272 - do {

273 - ssize_t retval = xwrite(request->local_fileno,

274 - (char *) ptr + posn, size - posn);

275 - if (retval < 0)

276 - return posn;

277 - posn += retval;

278 - } while (posn < size);

279 -

280 - request->stream.avail_in = size;

281 - request->stream.next_in = ptr;

282 - do {

283 - request->stream.next_out = expn;

284 - request->stream.avail_out = sizeof(expn);

285 - request->zret = git_inflate(&request->stream, Z_SYNC_FLUSH);

286 - git_SHA1_Update(&request->c, expn,

287 - sizeof(expn) - request->stream.avail_out);

288 - } while (request->stream.avail_in && request->zret == Z_OK);

289 - data_received++;

290 - return size;

291 -}

292 -

240 static void start_fetch_loose(struct transfer_request *request)

241 {

295 - char *hex = sha1_to_hex(request->obj->sha1);

296 - char *filename;

297 - char prevfile[PATH_MAX];

298 - char *url;

299 - int prevlocal;

300 - unsigned char prev_buf[PREV_BUF_SIZE];

301 - ssize_t prev_read = 0;

302 - long prev_posn = 0;

303 - char range[RANGE_HEADER_SIZE];

304 - struct curl_slist *range_header = NULL;

242 struct active_request_slot *slot;

243 + struct http_object_request *obj_req;

244

307 - filename = sha1_file_name(request->obj->sha1);

308 - snprintf(request->filename, sizeof(request->filename), "%s", filename);

309 - snprintf(request->tmpfile, sizeof(request->tmpfile),

310 - "%s.temp", filename);

311 -

312 - snprintf(prevfile, sizeof(prevfile), "%s.prev", request->filename);

313 - unlink_or_warn(prevfile);

314 - rename(request->tmpfile, prevfile);

315 - unlink_or_warn(request->tmpfile);

316 -

317 - if (request->local_fileno != -1)

318 - error("fd leakage in start: %d", request->local_fileno);

319 - request->local_fileno = open(request->tmpfile,

320 - O_WRONLY | O_CREAT | O_EXCL, 0666);

321 - /*

322 - * This could have failed due to the "lazy directory creation";

323 - * try to mkdir the last path component.

324 - */

325 - if (request->local_fileno < 0 && errno == ENOENT) {

326 - char *dir = strrchr(request->tmpfile, '/');

327 - if (dir) {

328 - *dir = 0;

329 - mkdir(request->tmpfile, 0777);

330 - *dir = '/';

331 - }

332 - request->local_fileno = open(request->tmpfile,

333 - O_WRONLY | O_CREAT | O_EXCL, 0666);

334 - }

335 -

336 - if (request->local_fileno < 0) {

245 + obj_req = new_http_object_request(repo->url, request->obj->sha1);

246 + if (obj_req == NULL) {

247 request->state = ABORTED;

338 - error("Couldn't create temporary file %s for %s: %s",

339 - request->tmpfile, request->filename, strerror(errno));

248 return;

249 }

250

343 - memset(&request->stream, 0, sizeof(request->stream));

344 -

345 - git_inflate_init(&request->stream);

346 -

347 - git_SHA1_Init(&request->c);

348 -

349 - url = get_remote_object_url(repo->url, hex, 0);

350 - request->url = xstrdup(url);

351 -

352 - /*

353 - * If a previous temp file is present, process what was already

354 - * fetched.

355 - */

356 - prevlocal = open(prevfile, O_RDONLY);

357 - if (prevlocal != -1) {

358 - do {

359 - prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);

360 - if (prev_read>0) {

361 - if (fwrite_sha1_file(prev_buf,

362 - 1,

363 - prev_read,

364 - request) == prev_read)

365 - prev_posn += prev_read;

366 - else

367 - prev_read = -1;

368 - }

369 - } while (prev_read > 0);

370 - close(prevlocal);

371 - }

372 - unlink_or_warn(prevfile);

373 -

374 - /*

375 - * Reset inflate/SHA1 if there was an error reading the previous temp

376 - * file; also rewind to the beginning of the local file.

377 - */

378 - if (prev_read == -1) {

379 - memset(&request->stream, 0, sizeof(request->stream));

380 - git_inflate_init(&request->stream);

381 - git_SHA1_Init(&request->c);

382 - if (prev_posn>0) {

383 - prev_posn = 0;

384 - lseek(request->local_fileno, 0, SEEK_SET);

385 - ftruncate(request->local_fileno, 0);

386 - }

387 - }

388 -

389 - slot = get_active_slot();

251 + slot = obj_req->slot;

252 slot->callback_func = process_response;

253 slot->callback_data = request;

254 request->slot = slot;

393 -

394 - curl_easy_setopt(slot->curl, CURLOPT_FILE, request);

395 - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);

396 - curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr);

397 - curl_easy_setopt(slot->curl, CURLOPT_URL, url);

398 - curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);

399 -

400 - /*

401 - * If we have successfully processed data from a previous fetch

402 - * attempt, only fetch the data we don't already have.

403 - */

404 - if (prev_posn>0) {

405 - if (push_verbosely)

406 - fprintf(stderr,

407 - "Resuming fetch of object %s at byte %ld\n",

408 - hex, prev_posn);

409 - sprintf(range, "Range: bytes=%ld-", prev_posn);

410 - range_header = curl_slist_append(range_header, range);

411 - curl_easy_setopt(slot->curl,

412 - CURLOPT_HTTPHEADER, range_header);

413 - }

255 + request->userData = obj_req;

256

257 /* Try to get the request started, abort the request on error */

258 request->state = RUN_FETCH_LOOSE;

259 if (!start_active_slot(slot)) {

260 fprintf(stderr, "Unable to start GET request\n");

261 repo->can_update_info_refs = 0;

262 + release_http_object_request(obj_req);

263 release_request(request);

264 }

265 }

@@ -675,16 +518,14 @@ static void release_request(struct transfer_request *request)

518 entry->next = entry->next->next;

519 }

520

678 - if (request->local_fileno != -1)

679 - close(request->local_fileno);

521 free(request->url);

522 free(request);

523 }

524

525 static void finish_request(struct transfer_request *request)

526 {

686 - struct stat st;

527 struct http_pack_request *preq;

528 + struct http_object_request *obj_req;

529

530 request->curl_result = request->slot->curl_result;

531 request->http_code = request->slot->http_code;

@@ -739,39 +580,17 @@ static void finish_request(struct transfer_request *request)

580 aborted = 1;

581 }

582 } else if (request->state == RUN_FETCH_LOOSE) {

742 - close(request->local_fileno);

743 - request->local_fileno = -1;

744 -

745 - if (request->curl_result != CURLE_OK &&

746 - request->http_code != 416) {

747 - if (stat(request->tmpfile, &st) == 0) {

748 - if (st.st_size == 0)

749 - unlink_or_warn(request->tmpfile);

750 - }

751 - } else {

752 - if (request->http_code == 416)

753 - warning("requested range invalid; we may already have all the data.");

754 -

755 - git_inflate_end(&request->stream);

756 - git_SHA1_Final(request->real_sha1, &request->c);

757 - if (request->zret != Z_STREAM_END) {

758 - unlink_or_warn(request->tmpfile);

759 - } else if (hashcmp(request->obj->sha1, request->real_sha1)) {

760 - unlink_or_warn(request->tmpfile);

761 - } else {

762 - request->rename =

763 - move_temp_to_file(

764 - request->tmpfile,

765 - request->filename);

766 - if (request->rename == 0)

767 - request->obj->flags |= (LOCAL | REMOTE);

768 - }

769 - }

583 + obj_req = (struct http_object_request *)request->userData;

584 +

585 + if (finish_http_object_request(obj_req) == 0)

586 + if (obj_req->rename == 0)

587 + request->obj->flags |= (LOCAL | REMOTE);

588

589 /* Try fetching packed if necessary */

772 - if (request->obj->flags & LOCAL)

590 + if (request->obj->flags & LOCAL) {

591 + release_http_object_request(obj_req);

592 release_request(request);

774 - else

593 + } else

594 start_fetch_packed(request);

595

596 } else if (request->state == RUN_FETCH_PACKED) {

@@ -843,7 +662,6 @@ static void add_fetch_request(struct object *obj)

662 request->url = NULL;

663 request->lock = NULL;

664 request->headers = NULL;

846 - request->local_fileno = -1;

665 request->state = NEED_FETCH;

666 request->next = request_queue_head;

667 request_queue_head = request;

@@ -882,7 +700,6 @@ static int add_send_request(struct object *obj, struct remote_lock *lock)

700 request->url = NULL;

701 request->lock = lock;

702 request->headers = NULL;

885 - request->local_fileno = -1;

703 request->state = NEED_PUSH;

704 request->next = request_queue_head;

705 request_queue_head = request;

http-walker.c

+40 -227

@@ -3,8 +3,6 @@

3 #include "walker.h"

4 #include "http.h"

6 -#define PREV_BUF_SIZE 4096

7 -

6 struct alt_base

7 {

8 char *base;

@@ -25,20 +23,8 @@ struct object_request

23 struct walker *walker;

24 unsigned char sha1[20];

25 struct alt_base *repo;

28 - char *url;

29 - char filename[PATH_MAX];

30 - char tmpfile[PATH_MAX];

31 - int local;

26 enum object_request_state state;

33 - CURLcode curl_result;

34 - char errorstr[CURL_ERROR_SIZE];

35 - long http_code;

36 - unsigned char real_sha1[20];

37 - git_SHA_CTX c;

38 - z_stream stream;

39 - int zret;

40 - int rename;

41 - struct active_request_slot *slot;

27 + struct http_object_request *req;

28 struct object_request *next;

29 };

@@ -59,34 +45,6 @@ struct walker_data {

46 static struct object_request *object_queue_head;

62 -static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,

63 - void *data)

64 -{

65 - unsigned char expn[4096];

66 - size_t size = eltsize * nmemb;

67 - int posn = 0;

68 - struct object_request *obj_req = (struct object_request *)data;

69 - do {

70 - ssize_t retval = xwrite(obj_req->local,

71 - (char *) ptr + posn, size - posn);

72 - if (retval < 0)

73 - return posn;

74 - posn += retval;

75 - } while (posn < size);

76 -

77 - obj_req->stream.avail_in = size;

78 - obj_req->stream.next_in = ptr;

79 - do {

80 - obj_req->stream.next_out = expn;

81 - obj_req->stream.avail_out = sizeof(expn);

82 - obj_req->zret = git_inflate(&obj_req->stream, Z_SYNC_FLUSH);

83 - git_SHA1_Update(&obj_req->c, expn,

84 - sizeof(expn) - obj_req->stream.avail_out);

85 - } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);

86 - data_received++;

87 - return size;

88 -}

89 -

48 static void fetch_alternates(struct walker *walker, const char *base);

50 static void process_object_response(void *callback_data);

@@ -94,172 +52,35 @@ static void process_object_response(void *callback_data);

52 static void start_object_request(struct walker *walker,

53 struct object_request *obj_req)

54 {

97 - char *hex = sha1_to_hex(obj_req->sha1);

98 - char prevfile[PATH_MAX];

99 - char *url;

100 - char *posn;

101 - int prevlocal;

102 - unsigned char prev_buf[PREV_BUF_SIZE];

103 - ssize_t prev_read = 0;

104 - long prev_posn = 0;

105 - char range[RANGE_HEADER_SIZE];

106 - struct curl_slist *range_header = NULL;

55 struct active_request_slot *slot;

56 + struct http_object_request *req;

109 - snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);

110 - unlink_or_warn(prevfile);

111 - rename(obj_req->tmpfile, prevfile);

112 - unlink_or_warn(obj_req->tmpfile);

113 -

114 - if (obj_req->local != -1)

115 - error("fd leakage in start: %d", obj_req->local);

116 - obj_req->local = open(obj_req->tmpfile,

117 - O_WRONLY | O_CREAT | O_EXCL, 0666);

118 - /*

119 - * This could have failed due to the "lazy directory creation";

120 - * try to mkdir the last path component.

121 - */

122 - if (obj_req->local < 0 && errno == ENOENT) {

123 - char *dir = strrchr(obj_req->tmpfile, '/');

124 - if (dir) {

125 - *dir = 0;

126 - mkdir(obj_req->tmpfile, 0777);

127 - *dir = '/';

128 - }

129 - obj_req->local = open(obj_req->tmpfile,

130 - O_WRONLY | O_CREAT | O_EXCL, 0666);

131 - }

132 -

133 - if (obj_req->local < 0) {

58 + req = new_http_object_request(obj_req->repo->base, obj_req->sha1);

59 + if (req == NULL) {

60 obj_req->state = ABORTED;

135 - error("Couldn't create temporary file %s for %s: %s",

136 - obj_req->tmpfile, obj_req->filename, strerror(errno));

61 return;

62 }

63 + obj_req->req = req;

140 - memset(&obj_req->stream, 0, sizeof(obj_req->stream));

141 -

142 - git_inflate_init(&obj_req->stream);

143 -

144 - git_SHA1_Init(&obj_req->c);

145 -

146 - url = xmalloc(strlen(obj_req->repo->base) + 51);

147 - obj_req->url = xmalloc(strlen(obj_req->repo->base) + 51);

148 - strcpy(url, obj_req->repo->base);

149 - posn = url + strlen(obj_req->repo->base);

150 - strcpy(posn, "/objects/");

151 - posn += 9;

152 - memcpy(posn, hex, 2);

153 - posn += 2;

154 - *(posn++) = '/';

155 - strcpy(posn, hex + 2);

156 - strcpy(obj_req->url, url);

157 -

158 - /*

159 - * If a previous temp file is present, process what was already

160 - * fetched.

161 - */

162 - prevlocal = open(prevfile, O_RDONLY);

163 - if (prevlocal != -1) {

164 - do {

165 - prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);

166 - if (prev_read>0) {

167 - if (fwrite_sha1_file(prev_buf,

168 - 1,

169 - prev_read,

170 - obj_req) == prev_read)

171 - prev_posn += prev_read;

172 - else

173 - prev_read = -1;

174 - }

175 - } while (prev_read > 0);

176 - close(prevlocal);

177 - }

178 - unlink_or_warn(prevfile);

179 -

180 - /*

181 - * Reset inflate/SHA1 if there was an error reading the previous temp

182 - * file; also rewind to the beginning of the local file.

183 - */

184 - if (prev_read == -1) {

185 - memset(&obj_req->stream, 0, sizeof(obj_req->stream));

186 - git_inflate_init(&obj_req->stream);

187 - git_SHA1_Init(&obj_req->c);

188 - if (prev_posn>0) {

189 - prev_posn = 0;

190 - lseek(obj_req->local, 0, SEEK_SET);

191 - ftruncate(obj_req->local, 0);

192 - }

193 - }

194 -

195 - slot = get_active_slot();

65 + slot = req->slot;

66 slot->callback_func = process_object_response;

67 slot->callback_data = obj_req;

198 - obj_req->slot = slot;

199 -

200 - curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);

201 - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);

202 - curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);

203 - curl_easy_setopt(slot->curl, CURLOPT_URL, url);

204 - curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);

205 -

206 - /*

207 - * If we have successfully processed data from a previous fetch

208 - * attempt, only fetch the data we don't already have.

209 - */

210 - if (prev_posn>0) {

211 - if (walker->get_verbosely)

212 - fprintf(stderr,

213 - "Resuming fetch of object %s at byte %ld\n",

214 - hex, prev_posn);

215 - sprintf(range, "Range: bytes=%ld-", prev_posn);

216 - range_header = curl_slist_append(range_header, range);

217 - curl_easy_setopt(slot->curl,

218 - CURLOPT_HTTPHEADER, range_header);

219 - }

69 /* Try to get the request started, abort the request on error */

70 obj_req->state = ACTIVE;

71 if (!start_active_slot(slot)) {

72 obj_req->state = ABORTED;

225 - obj_req->slot = NULL;

226 - close(obj_req->local);

227 - obj_req->local = -1;

228 - free(obj_req->url);

73 + release_http_object_request(req);

74 return;

75 }

76 }

78 static void finish_object_request(struct object_request *obj_req)

79 {

235 - struct stat st;

236 -

237 - close(obj_req->local);

238 - obj_req->local = -1;

239 -

240 - if (obj_req->http_code == 416) {

241 - fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");

242 - } else if (obj_req->curl_result != CURLE_OK) {

243 - if (stat(obj_req->tmpfile, &st) == 0)

244 - if (st.st_size == 0)

245 - unlink_or_warn(obj_req->tmpfile);

80 + if (finish_http_object_request(obj_req->req))

81 return;

247 - }

248 -

249 - git_inflate_end(&obj_req->stream);

250 - git_SHA1_Final(obj_req->real_sha1, &obj_req->c);

251 - if (obj_req->zret != Z_STREAM_END) {

252 - unlink_or_warn(obj_req->tmpfile);

253 - return;

254 - }

255 - if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {

256 - unlink_or_warn(obj_req->tmpfile);

257 - return;

258 - }

259 - obj_req->rename =

260 - move_temp_to_file(obj_req->tmpfile, obj_req->filename);

262 - if (obj_req->rename == 0)

83 + if (obj_req->req->rename == 0)

84 walker_say(obj_req->walker, "got %s\n", sha1_to_hex(obj_req->sha1));

85 }

@@ -271,19 +92,16 @@ static void process_object_response(void *callback_data)

92 struct walker_data *data = walker->data;

93 struct alt_base *alt = data->alt;

274 - obj_req->curl_result = obj_req->slot->curl_result;

275 - obj_req->http_code = obj_req->slot->http_code;

276 - obj_req->slot = NULL;

95 + process_http_object_request(obj_req->req);

96 obj_req->state = COMPLETE;

98 /* Use alternates if necessary */

280 - if (missing_target(obj_req)) {

99 + if (missing_target(obj_req->req)) {

100 fetch_alternates(walker, alt->base);

101 if (obj_req->repo->next != NULL) {

102 obj_req->repo =

103 obj_req->repo->next;

285 - close(obj_req->local);

286 - obj_req->local = -1;

104 + release_http_object_request(obj_req->req);

105 start_object_request(walker, obj_req);

106 return;

107 }

@@ -296,8 +114,8 @@ static void release_object_request(struct object_request *obj_req)

114 {

115 struct object_request *entry = object_queue_head;

116

299 - if (obj_req->local != -1)

300 - error("fd leakage in release: %d", obj_req->local);

117 + if (obj_req->req !=NULL && obj_req->req->localfile != -1)

118 + error("fd leakage in release: %d", obj_req->req->localfile);

119 if (obj_req == object_queue_head) {

120 object_queue_head = obj_req->next;

121 } else {

@@ -307,7 +125,6 @@ static void release_object_request(struct object_request *obj_req)

125 entry->next = entry->next->next;

126 }

127

310 - free(obj_req->url);

128 free(obj_req);

129 }

130

@@ -335,19 +152,13 @@ static void prefetch(struct walker *walker, unsigned char *sha1)

152 struct object_request *newreq;

153 struct object_request *tail;

154 struct walker_data *data = walker->data;

338 - char *filename = sha1_file_name(sha1);

155

156 newreq = xmalloc(sizeof(*newreq));

157 newreq->walker = walker;

158 hashcpy(newreq->sha1, sha1);

159 newreq->repo = data->alt;

344 - newreq->url = NULL;

345 - newreq->local = -1;

160 newreq->state = WAITING;

347 - snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);

348 - snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),

349 - "%s.temp", filename);

350 - newreq->slot = NULL;

161 + newreq->req = NULL;

162 newreq->next = NULL;

163

164 http_is_verbose = walker->get_verbosely;

@@ -638,15 +449,6 @@ abort:

449

450 static void abort_object_request(struct object_request *obj_req)

451 {

641 - if (obj_req->local >= 0) {

642 - close(obj_req->local);

643 - obj_req->local = -1;

644 - }

645 - unlink_or_warn(obj_req->tmpfile);

646 - if (obj_req->slot) {

647 - release_active_slot(obj_req->slot);

648 - obj_req->slot = NULL;

649 - }

452 release_object_request(obj_req);

453 }

454

@@ -655,6 +457,7 @@ static int fetch_object(struct walker *walker, struct alt_base *repo, unsigned c

457 char *hex = sha1_to_hex(sha1);

458 int ret = 0;

459 struct object_request *obj_req = object_queue_head;

460 + struct http_object_request *req;

461

462 while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))

463 obj_req = obj_req->next;

@@ -662,6 +465,8 @@ static int fetch_object(struct walker *walker, struct alt_base *repo, unsigned c

465 return error("Couldn't find request for %s in the queue", hex);

466

467 if (has_sha1_file(obj_req->sha1)) {

468 + if (obj_req->req != NULL)

469 + abort_http_object_request(obj_req->req);

470 abort_object_request(obj_req);

471 return 0;

472 }

@@ -673,34 +478,42 @@ static int fetch_object(struct walker *walker, struct alt_base *repo, unsigned c

478 start_object_request(walker, obj_req);

479 #endif

480

481 + /*

482 + * obj_req->req might change when fetching alternates in the callback

483 + * process_object_response; therefore, the "shortcut" variable, req,

484 + * is used only after we're done with slots.

485 + */

486 while (obj_req->state == ACTIVE)

677 - run_active_slot(obj_req->slot);

487 + run_active_slot(obj_req->req->slot);

488 +

489 + req = obj_req->req;

490

679 - if (obj_req->local != -1) {

680 - close(obj_req->local);

681 - obj_req->local = -1;

491 + if (req->localfile != -1) {

492 + close(req->localfile);

493 + req->localfile = -1;

494 }

495

496 if (obj_req->state == ABORTED) {

497 ret = error("Request for %s aborted", hex);

686 - } else if (obj_req->curl_result != CURLE_OK &&

687 - obj_req->http_code != 416) {

688 - if (missing_target(obj_req))

498 + } else if (req->curl_result != CURLE_OK &&

499 + req->http_code != 416) {

500 + if (missing_target(req))

501 ret = -1; /* Be silent, it is probably in a pack. */

502 else

503 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",

692 - obj_req->errorstr, obj_req->curl_result,

693 - obj_req->http_code, hex);

694 - } else if (obj_req->zret != Z_STREAM_END) {

504 + req->errorstr, req->curl_result,

505 + req->http_code, hex);

506 + } else if (req->zret != Z_STREAM_END) {

507 walker->corrupt_object_found++;

696 - ret = error("File %s (%s) corrupt", hex, obj_req->url);

697 - } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {

508 + ret = error("File %s (%s) corrupt", hex, req->url);

509 + } else if (hashcmp(obj_req->sha1, req->real_sha1)) {

510 ret = error("File %s has bad hash", hex);

699 - } else if (obj_req->rename < 0) {

511 + } else if (req->rename < 0) {

512 ret = error("unable to write sha1 filename %s",

701 - obj_req->filename);

513 + req->filename);

514 }

515

516 + release_http_object_request(req);

517 release_object_request(obj_req);

518 return ret;

519 }

http.c

+248 -2

@@ -12,6 +12,10 @@ static CURLM *curlm;

12 #ifndef NO_CURL_EASY_DUPHANDLE

13 static CURL *curl_default;

14 #endif

15 +

16 +#define PREV_BUF_SIZE 4096

17 +#define RANGE_HEADER_SIZE 30

18 +

19 char curl_errorstr[CURL_ERROR_SIZE];

21 static int curl_ssl_verify = -1;

@@ -30,8 +34,7 @@ static const char *curl_http_proxy;

34 static char *user_name, *user_pass;

36 static struct curl_slist *pragma_header;

33 -

34 -struct curl_slist *no_pragma_header;

37 +static struct curl_slist *no_pragma_header;

39 static struct active_request_slot *active_queue_head;

@@ -666,6 +669,23 @@ static char *quote_ref_url(const char *base, const char *ref)

669 return strbuf_detach(&buf, NULL);

670 }

671

672 +void append_remote_object_url(struct strbuf *buf, const char *url,

673 + const char *hex,

674 + int only_two_digit_prefix)

675 +{

676 + strbuf_addf(buf, "%s/objects/%.*s/", url, 2, hex);

677 + if (!only_two_digit_prefix)

678 + strbuf_addf(buf, "%s", hex+2);

679 +}

680 +

681 +char *get_remote_object_url(const char *url, const char *hex,

682 + int only_two_digit_prefix)

683 +{

684 + struct strbuf buf = STRBUF_INIT;

685 + append_remote_object_url(&buf, url, hex, only_two_digit_prefix);

686 + return strbuf_detach(&buf, NULL);

687 +}

688 +

689 /* http_request() targets */

690 #define HTTP_REQUEST_STRBUF 0

691 #define HTTP_REQUEST_FILE 1

@@ -995,3 +1015,229 @@ abort:

1015 free(filename);

1016 return NULL;

1017 }

1018 +

1019 +/* Helpers for fetching objects (loose) */

1020 +static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,

1021 + void *data)

1022 +{

1023 + unsigned char expn[4096];

1024 + size_t size = eltsize * nmemb;

1025 + int posn = 0;

1026 + struct http_object_request *freq =

1027 + (struct http_object_request *)data;

1028 + do {

1029 + ssize_t retval = xwrite(freq->localfile,

1030 + (char *) ptr + posn, size - posn);

1031 + if (retval < 0)

1032 + return posn;

1033 + posn += retval;

1034 + } while (posn < size);

1035 +

1036 + freq->stream.avail_in = size;

1037 + freq->stream.next_in = ptr;

1038 + do {

1039 + freq->stream.next_out = expn;

1040 + freq->stream.avail_out = sizeof(expn);

1041 + freq->zret = git_inflate(&freq->stream, Z_SYNC_FLUSH);

1042 + git_SHA1_Update(&freq->c, expn,

1043 + sizeof(expn) - freq->stream.avail_out);

1044 + } while (freq->stream.avail_in && freq->zret == Z_OK);

1045 + data_received++;

1046 + return size;

1047 +}

1048 +

1049 +struct http_object_request *new_http_object_request(const char *base_url,

1050 + unsigned char *sha1)

1051 +{

1052 + char *hex = sha1_to_hex(sha1);

1053 + char *filename;

1054 + char prevfile[PATH_MAX];

1055 + char *url;

1056 + int prevlocal;

1057 + unsigned char prev_buf[PREV_BUF_SIZE];

1058 + ssize_t prev_read = 0;

1059 + long prev_posn = 0;

1060 + char range[RANGE_HEADER_SIZE];

1061 + struct curl_slist *range_header = NULL;

1062 + struct http_object_request *freq;

1063 +

1064 + freq = xmalloc(sizeof(*freq));

1065 + hashcpy(freq->sha1, sha1);

1066 + freq->localfile = -1;

1067 +

1068 + filename = sha1_file_name(sha1);

1069 + snprintf(freq->filename, sizeof(freq->filename), "%s", filename);

1070 + snprintf(freq->tmpfile, sizeof(freq->tmpfile),

1071 + "%s.temp", filename);

1072 +

1073 + snprintf(prevfile, sizeof(prevfile), "%s.prev", filename);

1074 + unlink_or_warn(prevfile);

1075 + rename(freq->tmpfile, prevfile);

1076 + unlink_or_warn(freq->tmpfile);

1077 +

1078 + if (freq->localfile != -1)

1079 + error("fd leakage in start: %d", freq->localfile);

1080 + freq->localfile = open(freq->tmpfile,

1081 + O_WRONLY | O_CREAT | O_EXCL, 0666);

1082 + /*

1083 + * This could have failed due to the "lazy directory creation";

1084 + * try to mkdir the last path component.

1085 + */

1086 + if (freq->localfile < 0 && errno == ENOENT) {

1087 + char *dir = strrchr(freq->tmpfile, '/');

1088 + if (dir) {

1089 + *dir = 0;

1090 + mkdir(freq->tmpfile, 0777);

1091 + *dir = '/';

1092 + }

1093 + freq->localfile = open(freq->tmpfile,

1094 + O_WRONLY | O_CREAT | O_EXCL, 0666);

1095 + }

1096 +

1097 + if (freq->localfile < 0) {

1098 + error("Couldn't create temporary file %s for %s: %s",

1099 + freq->tmpfile, freq->filename, strerror(errno));

1100 + goto abort;

1101 + }

1102 +

1103 + memset(&freq->stream, 0, sizeof(freq->stream));

1104 +

1105 + git_inflate_init(&freq->stream);

1106 +

1107 + git_SHA1_Init(&freq->c);

1108 +

1109 + url = get_remote_object_url(base_url, hex, 0);

1110 + freq->url = xstrdup(url);

1111 +

1112 + /*

1113 + * If a previous temp file is present, process what was already

1114 + * fetched.

1115 + */

1116 + prevlocal = open(prevfile, O_RDONLY);

1117 + if (prevlocal != -1) {

1118 + do {

1119 + prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);

1120 + if (prev_read>0) {

1121 + if (fwrite_sha1_file(prev_buf,

1122 + 1,

1123 + prev_read,

1124 + freq) == prev_read) {

1125 + prev_posn += prev_read;

1126 + } else {

1127 + prev_read = -1;

1128 + }

1129 + }

1130 + } while (prev_read > 0);

1131 + close(prevlocal);

1132 + }

1133 + unlink_or_warn(prevfile);

1134 +

1135 + /*

1136 + * Reset inflate/SHA1 if there was an error reading the previous temp

1137 + * file; also rewind to the beginning of the local file.

1138 + */

1139 + if (prev_read == -1) {

1140 + memset(&freq->stream, 0, sizeof(freq->stream));

1141 + git_inflate_init(&freq->stream);

1142 + git_SHA1_Init(&freq->c);

1143 + if (prev_posn>0) {

1144 + prev_posn = 0;

1145 + lseek(freq->localfile, 0, SEEK_SET);

1146 + ftruncate(freq->localfile, 0);

1147 + }

1148 + }

1149 +

1150 + freq->slot = get_active_slot();

1151 +

1152 + curl_easy_setopt(freq->slot->curl, CURLOPT_FILE, freq);

1153 + curl_easy_setopt(freq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);

1154 + curl_easy_setopt(freq->slot->curl, CURLOPT_ERRORBUFFER, freq->errorstr);

1155 + curl_easy_setopt(freq->slot->curl, CURLOPT_URL, url);

1156 + curl_easy_setopt(freq->slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);

1157 +

1158 + /*

1159 + * If we have successfully processed data from a previous fetch

1160 + * attempt, only fetch the data we don't already have.

1161 + */

1162 + if (prev_posn>0) {

1163 + if (http_is_verbose)

1164 + fprintf(stderr,

1165 + "Resuming fetch of object %s at byte %ld\n",

1166 + hex, prev_posn);

1167 + sprintf(range, "Range: bytes=%ld-", prev_posn);

1168 + range_header = curl_slist_append(range_header, range);

1169 + curl_easy_setopt(freq->slot->curl,

1170 + CURLOPT_HTTPHEADER, range_header);

1171 + }

1172 +

1173 + return freq;

1174 +

1175 + free(url);

1176 +abort:

1177 + free(filename);

1178 + free(freq);

1179 + return NULL;

1180 +}

1181 +

1182 +void process_http_object_request(struct http_object_request *freq)

1183 +{

1184 + if (freq->slot == NULL)

1185 + return;

1186 + freq->curl_result = freq->slot->curl_result;

1187 + freq->http_code = freq->slot->http_code;

1188 + freq->slot = NULL;

1189 +}

1190 +

1191 +int finish_http_object_request(struct http_object_request *freq)

1192 +{

1193 + struct stat st;

1194 +

1195 + close(freq->localfile);

1196 + freq->localfile = -1;

1197 +

1198 + process_http_object_request(freq);

1199 +

1200 + if (freq->http_code == 416) {

1201 + fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");

1202 + } else if (freq->curl_result != CURLE_OK) {

1203 + if (stat(freq->tmpfile, &st) == 0)

1204 + if (st.st_size == 0)

1205 + unlink_or_warn(freq->tmpfile);

1206 + return -1;

1207 + }

1208 +

1209 + git_inflate_end(&freq->stream);

1210 + git_SHA1_Final(freq->real_sha1, &freq->c);

1211 + if (freq->zret != Z_STREAM_END) {

1212 + unlink_or_warn(freq->tmpfile);

1213 + return -1;

1214 + }

1215 + if (hashcmp(freq->sha1, freq->real_sha1)) {

1216 + unlink_or_warn(freq->tmpfile);

1217 + return -1;

1218 + }

1219 + freq->rename =

1220 + move_temp_to_file(freq->tmpfile, freq->filename);

1221 +

1222 + return freq->rename;

1223 +}

1224 +

1225 +void abort_http_object_request(struct http_object_request *freq)

1226 +{

1227 + unlink_or_warn(freq->tmpfile);

1228 +

1229 + release_http_object_request(freq);

1230 +}

1231 +

1232 +void release_http_object_request(struct http_object_request *freq)

1233 +{

1234 + if (freq->localfile != -1) {

1235 + close(freq->localfile);

1236 + freq->localfile = -1;

1237 + }

1238 + if (freq->url != NULL) {

1239 + free(freq->url);

1240 + freq->url = NULL;

1241 + }

1242 + freq->slot = NULL;

1243 +}

http.h

+33 -4

@@ -88,10 +88,6 @@ extern void add_fill_function(void *data, int (*fill)(void *));

88 extern void step_active_slots(void);

89 #endif

91 -extern struct curl_slist *no_pragma_header;

92 -

93 -#define RANGE_HEADER_SIZE 30

94 -

91 extern void http_init(struct remote *remote);

92 extern void http_cleanup(void);

@@ -114,6 +110,13 @@ static inline int missing__target(int code, int result)

110

111 #define missing_target(a) missing__target((a)->http_code, (a)->curl_result)

112

113 +/* Helpers for modifying and creating URLs */

114 +extern void append_remote_object_url(struct strbuf *buf, const char *url,

115 + const char *hex,

116 + int only_two_digit_prefix);

117 +extern char *get_remote_object_url(const char *url, const char *hex,

118 + int only_two_digit_prefix);

119 +

120 /* Options for http_request_*() */

121 #define HTTP_NO_CACHE 1

122

@@ -167,4 +170,30 @@ extern struct http_pack_request *new_http_pack_request(

170 extern int finish_http_pack_request(struct http_pack_request *preq);

171 extern void release_http_pack_request(struct http_pack_request *preq);

172

173 +/* Helpers for fetching object */

174 +struct http_object_request

175 +{

176 + char *url;

177 + char filename[PATH_MAX];

178 + char tmpfile[PATH_MAX];

179 + int localfile;

180 + CURLcode curl_result;

181 + char errorstr[CURL_ERROR_SIZE];

182 + long http_code;

183 + unsigned char sha1[20];

184 + unsigned char real_sha1[20];

185 + git_SHA_CTX c;

186 + z_stream stream;

187 + int zret;

188 + int rename;

189 + struct active_request_slot *slot;

190 +};

191 +

192 +extern struct http_object_request *new_http_object_request(

193 + const char *base_url, unsigned char *sha1);

194 +extern void process_http_object_request(struct http_object_request *freq);

195 +extern int finish_http_object_request(struct http_object_request *freq);

196 +extern void abort_http_object_request(struct http_object_request *freq);

197 +extern void release_http_object_request(struct http_object_request *freq);

198 +

199 #endif /* HTTP_H */