http.c · leroysheep/MovieWebApp · siGit Code & Deploy

test

bisect jch maint master next seen test todo

c 3,053 lines 81.9 KB

Raw

1	#define USE_THE_REPOSITORY_VARIABLE
2	#define DISABLE_SIGN_COMPARE_WARNINGS
3
4	#include "git-compat-util.h"
5	#include "git-curl-compat.h"
6	#include "environment.h"
7	#include "hex.h"
8	#include "http.h"
9	#include "config.h"
10	#include "pack.h"
11	#include "run-command.h"
12	#include "url.h"
13	#include "urlmatch.h"
14	#include "credential.h"
15	#include "version.h"
16	#include "pkt-line.h"
17	#include "gettext.h"
18	#include "trace.h"
19	#include "transport.h"
20	#include "packfile.h"
21	#include "string-list.h"
22	#include "object-file.h"
23	#include "odb.h"
24	#include "tempfile.h"
25	#include "date.h"
26	#include "trace2.h"
27
28	static struct trace_key trace_curl = TRACE_KEY_INIT(CURL);
29	static int trace_curl_data = 1;
30	static int trace_curl_redact = 1;
31	long int git_curl_ipresolve = CURL_IPRESOLVE_WHATEVER;
32	int active_requests;
33	int http_is_verbose;
34	ssize_t http_post_buffer = 16 * LARGE_PACKET_MAX;
35
36	static int min_curl_sessions = 1;
37	static int curl_session_count;
38	static int max_requests = -1;
39	static CURLM *curlm;
40	static CURL *curl_default;
41
42	#define PREV_BUF_SIZE 4096
43
44	char curl_errorstr[CURL_ERROR_SIZE];
45
46	static int curl_ssl_verify = -1;
47	static int curl_ssl_try;
48	static char *curl_http_version;
49	static char *ssl_cert;
50	static char *ssl_cert_type;
51	static char *ssl_cipherlist;
52	static char *ssl_version;
53	static struct {
54	const char *name;
55	long ssl_version;
56	} sslversions[] = {
57	{ "sslv2", CURL_SSLVERSION_SSLv2 },
58	{ "sslv3", CURL_SSLVERSION_SSLv3 },
59	{ "tlsv1", CURL_SSLVERSION_TLSv1 },
60	{ "tlsv1.0", CURL_SSLVERSION_TLSv1_0 },
61	{ "tlsv1.1", CURL_SSLVERSION_TLSv1_1 },
62	{ "tlsv1.2", CURL_SSLVERSION_TLSv1_2 },
63	{ "tlsv1.3", CURL_SSLVERSION_TLSv1_3 },
64	};
65	static char *ssl_key;
66	static char *ssl_key_type;
67	static char *ssl_capath;
68	static char *curl_no_proxy;
69	static char *ssl_pinnedkey;
70	static char *ssl_cainfo;
71	static long curl_low_speed_limit = -1;
72	static long curl_low_speed_time = -1;
73	static int curl_ftp_no_epsv;
74	static char *curl_http_proxy;
75	static char *http_proxy_authmethod;
76
77	static char *http_proxy_ssl_cert;
78	static char *http_proxy_ssl_key;
79	static char *http_proxy_ssl_ca_info;
80	static struct credential proxy_cert_auth = CREDENTIAL_INIT;
81	static int proxy_ssl_cert_password_required;
82
83	static struct {
84	const char *name;
85	long curlauth_param;
86	} proxy_authmethods[] = {
87	{ "basic", CURLAUTH_BASIC },
88	{ "digest", CURLAUTH_DIGEST },
89	{ "negotiate", CURLAUTH_GSSNEGOTIATE },
90	{ "ntlm", CURLAUTH_NTLM },
91	{ "anyauth", CURLAUTH_ANY },
92	/*
93	* CURLAUTH_DIGEST_IE has no corresponding command-line option in
94	* curl(1) and is not included in CURLAUTH_ANY, so we leave it out
95	* here, too
96	*/
97	};
98	#ifdef CURLGSSAPI_DELEGATION_FLAG
99	static char *curl_deleg;
100	static struct {
101	const char *name;
102	long curl_deleg_param;
103	} curl_deleg_levels[] = {
104	{ "none", CURLGSSAPI_DELEGATION_NONE },
105	{ "policy", CURLGSSAPI_DELEGATION_POLICY_FLAG },
106	{ "always", CURLGSSAPI_DELEGATION_FLAG },
107	};
108	#endif
109
110	static long curl_tcp_keepidle = -1;
111	static long curl_tcp_keepintvl = -1;
112	static long curl_tcp_keepcnt = -1;
113
114	enum proactive_auth {
115	PROACTIVE_AUTH_NONE = 0,
116	PROACTIVE_AUTH_IF_CREDENTIALS,
117	PROACTIVE_AUTH_AUTO,
118	PROACTIVE_AUTH_BASIC,
119	};
120
121	static struct credential proxy_auth = CREDENTIAL_INIT;
122	static const char *curl_proxyuserpwd;
123	static char *curl_cookie_file;
124	static int curl_save_cookies;
125	struct credential http_auth = CREDENTIAL_INIT;
126	static enum proactive_auth http_proactive_auth;
127	static char *user_agent;
128	static int curl_empty_auth = -1;
129
130	enum http_follow_config http_follow_config = HTTP_FOLLOW_INITIAL;
131
132	static struct credential cert_auth = CREDENTIAL_INIT;
133	static int ssl_cert_password_required;
134	static unsigned long http_auth_methods = CURLAUTH_ANY;
135	static int http_auth_methods_restricted;
136	/* Modes for which empty_auth cannot actually help us. */
137	static unsigned long empty_auth_useless =
138	CURLAUTH_BASIC
139	\| CURLAUTH_DIGEST_IE
140	\| CURLAUTH_DIGEST;
141	static int empty_auth_try_negotiate;
142
143	static struct curl_slist *pragma_header;
144	static struct string_list extra_http_headers = STRING_LIST_INIT_DUP;
145
146	static struct curl_slist *host_resolutions;
147
148	static struct active_request_slot *active_queue_head;
149
150	static char *cached_accept_language;
151
152	static char *http_ssl_backend;
153
154	static int http_schannel_check_revoke = 1;
155
156	static long http_retry_after = 0;
157	static long http_max_retries = 0;
158	static long http_max_retry_time = 300;
159
160	/*
161	* With the backend being set to `schannel`, setting sslCAinfo would override
162	* the Certificate Store in cURL v7.60.0 and later, which is not what we want
163	* by default.
164	*/
165	static int http_schannel_use_ssl_cainfo;
166
167	static int always_auth_proactively(void)
168	{
169	return http_proactive_auth != PROACTIVE_AUTH_NONE &&
170	http_proactive_auth != PROACTIVE_AUTH_IF_CREDENTIALS;
171	}
172
173	size_t fread_buffer(char ptr, size_t eltsize, size_t nmemb, void buffer_)
174	{
175	size_t size = eltsize * nmemb;
176	struct buffer *buffer = buffer_;
177
178	if (size > buffer->buf.len - buffer->posn)
179	size = buffer->buf.len - buffer->posn;
180	memcpy(ptr, buffer->buf.buf + buffer->posn, size);
181	buffer->posn += size;
182
183	return size / eltsize;
184	}
185
186	int seek_buffer(void *clientp, curl_off_t offset, int origin)
187	{
188	struct buffer *buffer = clientp;
189
190	if (origin != SEEK_SET)
191	BUG("seek_buffer only handles SEEK_SET");
192	if (offset < 0 \|\| offset >= buffer->buf.len) {
193	error("curl seek would be outside of buffer");
194	return CURL_SEEKFUNC_FAIL;
195	}
196
197	buffer->posn = offset;
198	return CURL_SEEKFUNC_OK;
199	}
200
201	size_t fwrite_buffer(char ptr, size_t eltsize, size_t nmemb, void buffer_)
202	{
203	size_t size = eltsize * nmemb;
204	struct strbuf *buffer = buffer_;
205
206	strbuf_add(buffer, ptr, size);
207	return nmemb;
208	}
209
210	/*
211	* A folded header continuation line starts with any number of spaces or
212	* horizontal tab characters (SP or HTAB) as per RFC 7230 section 3.2.
213	* It is not a continuation line if the line starts with any other character.
214	*/
215	static inline int is_hdr_continuation(const char *ptr, const size_t size)
216	{
217	return size && (ptr == ' ' \|\| ptr == '\t');
218	}
219
220	static size_t fwrite_wwwauth(char ptr, size_t eltsize, size_t nmemb, void p MAYBE_UNUSED)
221	{
222	size_t size = eltsize * nmemb;
223	struct strvec *values = &http_auth.wwwauth_headers;
224	struct strbuf buf = STRBUF_INIT;
225	const char *val;
226	size_t val_len;
227
228	/*
229	* Header lines may not come NULL-terminated from libcurl so we must
230	* limit all scans to the maximum length of the header line, or leverage
231	* strbufs for all operations.
232	*
233	* In addition, it is possible that header values can be split over
234	* multiple lines as per RFC 7230. 'Line folding' has been deprecated
235	* but older servers may still emit them. A continuation header field
236	* value is identified as starting with a space or horizontal tab.
237	*
238	* The formal definition of a header field as given in RFC 7230 is:
239	*
240	* header-field = field-name ":" OWS field-value OWS
241	*
242	* field-name = token
243	* field-value = *( field-content / obs-fold )
244	* field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ]
245	* field-vchar = VCHAR / obs-text
246	*
247	* obs-fold = CRLF 1*( SP / HTAB )
248	* ; obsolete line folding
249	* ; see Section 3.2.4
250	*/
251
252	/* Start of a new WWW-Authenticate header */
253	if (skip_iprefix_mem(ptr, size, "www-authenticate:", &val, &val_len)) {
254	strbuf_add(&buf, val, val_len);
255
256	/*
257	* Strip the CRLF that should be present at the end of each
258	* field as well as any trailing or leading whitespace from the
259	* value.
260	*/
261	strbuf_trim(&buf);
262
263	strvec_push(values, buf.buf);
264	http_auth.header_is_last_match = 1;
265	goto exit;
266	}
267
268	/*
269	* This line could be a continuation of the previously matched header
270	* field. If this is the case then we should append this value to the
271	* end of the previously consumed value.
272	*/
273	if (http_auth.header_is_last_match && is_hdr_continuation(ptr, size)) {
274	/*
275	* Trim the CRLF and any leading or trailing from this line.
276	*/
277	strbuf_add(&buf, ptr, size);
278	strbuf_trim(&buf);
279
280	/*
281	* At this point we should always have at least one existing
282	* value, even if it is empty. Do not bother appending the new
283	* value if this continuation header is itself empty.
284	*/
285	if (!values->nr) {
286	BUG("should have at least one existing header value");
287	} else if (buf.len) {
288	char *prev = xstrdup(values->v[values->nr - 1]);
289
290	/* Join two non-empty values with a single space. */
291	const char const sp = prev ? " " : "";
292
293	strvec_pop(values);
294	strvec_pushf(values, "%s%s%s", prev, sp, buf.buf);
295	free(prev);
296	}
297
298	goto exit;
299	}
300
301	/* Not a continuation of a previously matched auth header line. */
302	http_auth.header_is_last_match = 0;
303
304	/*
305	* If this is a HTTP status line and not a header field, this signals
306	* a different HTTP response. libcurl writes all the output of all
307	* response headers of all responses, including redirects.
308	* We only care about the last HTTP request response's headers so clear
309	* the existing array.
310	*/
311	if (skip_iprefix_mem(ptr, size, "http/", &val, &val_len))
312	strvec_clear(values);
313
314	exit:
315	strbuf_release(&buf);
316	return size;
317	}
318
319	size_t fwrite_null(char *ptr UNUSED, size_t eltsize UNUSED, size_t nmemb,
320	void *data UNUSED)
321	{
322	return nmemb;
323	}
324
325	static struct curl_slist *object_request_headers(void)
326	{
327	return curl_slist_append(http_copy_default_headers(), "Pragma:");
328	}
329
330	static void closedown_active_slot(struct active_request_slot *slot)
331	{
332	active_requests--;
333	slot->in_use = 0;
334	}
335
336	static void finish_active_slot(struct active_request_slot *slot)
337	{
338	closedown_active_slot(slot);
339	curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code);
340
341	if (slot->finished)
342	(*slot->finished) = 1;
343
344	/* Store slot results so they can be read after the slot is reused */
345	if (slot->results) {
346	slot->results->curl_result = slot->curl_result;
347	slot->results->http_code = slot->http_code;
348	curl_easy_getinfo(slot->curl, CURLINFO_HTTPAUTH_AVAIL,
349	&slot->results->auth_avail);
350
351	curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CONNECTCODE,
352	&slot->results->http_connectcode);
353	}
354
355	/* Run callback if appropriate */
356	if (slot->callback_func)
357	slot->callback_func(slot->callback_data);
358	}
359
360	static void xmulti_remove_handle(struct active_request_slot *slot)
361	{
362	curl_multi_remove_handle(curlm, slot->curl);
363	}
364
365	static void process_curl_messages(void)
366	{
367	int num_messages;
368	struct active_request_slot *slot;
369	CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
370
371	while (curl_message != NULL) {
372	if (curl_message->msg == CURLMSG_DONE) {
373	int curl_result = curl_message->data.result;
374	slot = active_queue_head;
375	while (slot != NULL &&
376	slot->curl != curl_message->easy_handle)
377	slot = slot->next;
378	if (slot) {
379	xmulti_remove_handle(slot);
380	slot->curl_result = curl_result;
381	finish_active_slot(slot);
382	} else {
383	fprintf(stderr, "Received DONE message for unknown request!\n");
384	}
385	} else {
386	fprintf(stderr, "Unknown CURL message received: %d\n",
387	(int)curl_message->msg);
388	}
389	curl_message = curl_multi_info_read(curlm, &num_messages);
390	}
391	}
392
393	static int http_options(const char var, const char value,
394	const struct config_context ctx, void data)
395	{
396	if (!strcmp("http.version", var)) {
397	return git_config_string(&curl_http_version, var, value);
398	}
399	if (!strcmp("http.sslverify", var)) {
400	curl_ssl_verify = git_config_bool(var, value);
401	return 0;
402	}
403	if (!strcmp("http.sslcipherlist", var))
404	return git_config_string(&ssl_cipherlist, var, value);
405	if (!strcmp("http.sslversion", var))
406	return git_config_string(&ssl_version, var, value);
407	if (!strcmp("http.sslcert", var))
408	return git_config_pathname(&ssl_cert, var, value);
409	if (!strcmp("http.sslcerttype", var))
410	return git_config_string(&ssl_cert_type, var, value);
411	if (!strcmp("http.sslkey", var))
412	return git_config_pathname(&ssl_key, var, value);
413	if (!strcmp("http.sslkeytype", var))
414	return git_config_string(&ssl_key_type, var, value);
415	if (!strcmp("http.sslcapath", var))
416	return git_config_pathname(&ssl_capath, var, value);
417	if (!strcmp("http.sslcainfo", var))
418	return git_config_pathname(&ssl_cainfo, var, value);
419	if (!strcmp("http.sslcertpasswordprotected", var)) {
420	ssl_cert_password_required = git_config_bool(var, value);
421	return 0;
422	}
423	if (!strcmp("http.ssltry", var)) {
424	curl_ssl_try = git_config_bool(var, value);
425	return 0;
426	}
427	if (!strcmp("http.sslbackend", var)) {
428	free(http_ssl_backend);
429	http_ssl_backend = xstrdup_or_null(value);
430	return 0;
431	}
432
433	if (!strcmp("http.schannelcheckrevoke", var)) {
434	http_schannel_check_revoke = git_config_bool(var, value);
435	return 0;
436	}
437
438	if (!strcmp("http.schannelusesslcainfo", var)) {
439	http_schannel_use_ssl_cainfo = git_config_bool(var, value);
440	return 0;
441	}
442
443	if (!strcmp("http.minsessions", var)) {
444	min_curl_sessions = git_config_int(var, value, ctx->kvi);
445	if (min_curl_sessions > 1)
446	min_curl_sessions = 1;
447	return 0;
448	}
449	if (!strcmp("http.maxrequests", var)) {
450	max_requests = git_config_int(var, value, ctx->kvi);
451	return 0;
452	}
453	if (!strcmp("http.lowspeedlimit", var)) {
454	curl_low_speed_limit = git_config_int(var, value, ctx->kvi);
455	return 0;
456	}
457	if (!strcmp("http.lowspeedtime", var)) {
458	curl_low_speed_time = git_config_int(var, value, ctx->kvi);
459	return 0;
460	}
461
462	if (!strcmp("http.noepsv", var)) {
463	curl_ftp_no_epsv = git_config_bool(var, value);
464	return 0;
465	}
466	if (!strcmp("http.proxy", var))
467	return git_config_string(&curl_http_proxy, var, value);
468
469	if (!strcmp("http.proxyauthmethod", var))
470	return git_config_string(&http_proxy_authmethod, var, value);
471
472	if (!strcmp("http.proxysslcert", var))
473	return git_config_string(&http_proxy_ssl_cert, var, value);
474
475	if (!strcmp("http.proxysslkey", var))
476	return git_config_string(&http_proxy_ssl_key, var, value);
477
478	if (!strcmp("http.proxysslcainfo", var))
479	return git_config_string(&http_proxy_ssl_ca_info, var, value);
480
481	if (!strcmp("http.proxysslcertpasswordprotected", var)) {
482	proxy_ssl_cert_password_required = git_config_bool(var, value);
483	return 0;
484	}
485
486	if (!strcmp("http.cookiefile", var))
487	return git_config_pathname(&curl_cookie_file, var, value);
488	if (!strcmp("http.savecookies", var)) {
489	curl_save_cookies = git_config_bool(var, value);
490	return 0;
491	}
492
493	if (!strcmp("http.postbuffer", var)) {
494	http_post_buffer = git_config_ssize_t(var, value, ctx->kvi);
495	if (http_post_buffer < 0)
496	warning(_("negative value for http.postBuffer; defaulting to %d"), LARGE_PACKET_MAX);
497	if (http_post_buffer < LARGE_PACKET_MAX)
498	http_post_buffer = LARGE_PACKET_MAX;
499	return 0;
500	}
501
502	if (!strcmp("http.useragent", var))
503	return git_config_string(&user_agent, var, value);
504
505	if (!strcmp("http.emptyauth", var)) {
506	if (value && !strcmp("auto", value))
507	curl_empty_auth = -1;
508	else
509	curl_empty_auth = git_config_bool(var, value);
510	return 0;
511	}
512
513	if (!strcmp("http.delegation", var)) {
514	#ifdef CURLGSSAPI_DELEGATION_FLAG
515	return git_config_string(&curl_deleg, var, value);
516	#else
517	warning(_("Delegation control is not supported with cURL < 7.22.0"));
518	return 0;
519	#endif
520	}
521
522	if (!strcmp("http.pinnedpubkey", var)) {
523	return git_config_pathname(&ssl_pinnedkey, var, value);
524	}
525
526	if (!strcmp("http.extraheader", var)) {
527	if (!value) {
528	return config_error_nonbool(var);
529	} else if (!*value) {
530	string_list_clear(&extra_http_headers, 0);
531	} else {
532	string_list_append(&extra_http_headers, value);
533	}
534	return 0;
535	}
536
537	if (!strcmp("http.curloptresolve", var)) {
538	if (!value) {
539	return config_error_nonbool(var);
540	} else if (!*value) {
541	curl_slist_free_all(host_resolutions);
542	host_resolutions = NULL;
543	} else {
544	host_resolutions = curl_slist_append(host_resolutions, value);
545	}
546	return 0;
547	}
548
549	if (!strcmp("http.followredirects", var)) {
550	if (value && !strcmp(value, "initial"))
551	http_follow_config = HTTP_FOLLOW_INITIAL;
552	else if (git_config_bool(var, value))
553	http_follow_config = HTTP_FOLLOW_ALWAYS;
554	else
555	http_follow_config = HTTP_FOLLOW_NONE;
556	return 0;
557	}
558
559	if (!strcmp("http.proactiveauth", var)) {
560	if (!value)
561	return config_error_nonbool(var);
562	if (!strcmp(value, "auto"))
563	http_proactive_auth = PROACTIVE_AUTH_AUTO;
564	else if (!strcmp(value, "basic"))
565	http_proactive_auth = PROACTIVE_AUTH_BASIC;
566	else if (!strcmp(value, "none"))
567	http_proactive_auth = PROACTIVE_AUTH_NONE;
568	else
569	warning(_("Unknown value for http.proactiveauth"));
570	return 0;
571	}
572
573	if (!strcmp("http.keepaliveidle", var)) {
574	curl_tcp_keepidle = git_config_int(var, value, ctx->kvi);
575	return 0;
576	}
577	if (!strcmp("http.keepaliveinterval", var)) {
578	curl_tcp_keepintvl = git_config_int(var, value, ctx->kvi);
579	return 0;
580	}
581	if (!strcmp("http.keepalivecount", var)) {
582	curl_tcp_keepcnt = git_config_int(var, value, ctx->kvi);
583	return 0;
584	}
585
586	if (!strcmp("http.retryafter", var)) {
587	http_retry_after = git_config_int(var, value, ctx->kvi);
588	return 0;
589	}
590
591	if (!strcmp("http.maxretries", var)) {
592	http_max_retries = git_config_int(var, value, ctx->kvi);
593	return 0;
594	}
595
596	if (!strcmp("http.maxretrytime", var)) {
597	http_max_retry_time = git_config_int(var, value, ctx->kvi);
598	return 0;
599	}
600
601	/* Fall back on the default ones */
602	return git_default_config(var, value, ctx, data);
603	}
604
605	static int curl_empty_auth_enabled(void)
606	{
607	if (curl_empty_auth >= 0)
608	return curl_empty_auth;
609
610	/*
611	* In the automatic case, kick in the empty-auth
612	* hack as long as we would potentially try some
613	* method more exotic than "Basic" or "Digest".
614	*
615	* But only do this when this is our second or
616	* subsequent request, as by then we know what
617	* methods are available.
618	*/
619	if (http_auth_methods_restricted &&
620	(http_auth_methods & ~empty_auth_useless))
621	return 1;
622	return 0;
623	}
624
625	struct curl_slist http_append_auth_header(const struct credential c,
626	struct curl_slist *headers)
627	{
628	if (c->authtype && c->credential) {
629	struct strbuf auth = STRBUF_INIT;
630	strbuf_addf(&auth, "Authorization: %s %s",
631	c->authtype, c->credential);
632	headers = curl_slist_append(headers, auth.buf);
633	strbuf_release(&auth);
634	}
635	return headers;
636	}
637
638	static void init_curl_http_auth(CURL *result)
639	{
640	if ((!http_auth.username \|\| !*http_auth.username) &&
641	(!http_auth.credential \|\| !*http_auth.credential)) {
642	if (!always_auth_proactively() && curl_empty_auth_enabled()) {
643	curl_easy_setopt(result, CURLOPT_USERPWD, ":");
644	return;
645	} else if (!always_auth_proactively()) {
646	return;
647	} else if (http_proactive_auth == PROACTIVE_AUTH_BASIC) {
648	strvec_push(&http_auth.wwwauth_headers, "Basic");
649	}
650	}
651
652	credential_fill(the_repository, &http_auth, 1);
653
654	if (http_auth.password) {
655	if (always_auth_proactively()) {
656	/*
657	* We got a credential without an authtype and we don't
658	* know what's available. Since our only two options at
659	* the moment are auto (which defaults to basic) and
660	* basic, use basic for now.
661	*/
662	curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
663	}
664	curl_easy_setopt(result, CURLOPT_USERNAME, http_auth.username);
665	curl_easy_setopt(result, CURLOPT_PASSWORD, http_auth.password);
666	}
667	}
668
669	void http_reauth_prepare(int all_capabilities)
670	{
671	/*
672	* If we deferred stripping Negotiate to give empty auth a
673	* chance (auto mode), skip credential_fill on this retry so
674	* that init_curl_http_auth() sends empty credentials and
675	* libcurl can attempt Negotiate with the system ticket cache.
676	*/
677	if (empty_auth_try_negotiate &&
678	!http_auth.password && !http_auth.credential &&
679	(http_auth_methods & CURLAUTH_GSSNEGOTIATE))
680	return;
681
682	credential_fill(the_repository, &http_auth, all_capabilities);
683	}
684
685	/* var must be free-able /
686	static void var_override(char *var, char value)
687	{
688	if (value) {
689	free(*var);
690	*var = xstrdup(value);
691	}
692	}
693
694	static void set_proxyauth_name_password(CURL *result)
695	{
696	if (proxy_auth.password) {
697	curl_easy_setopt(result, CURLOPT_PROXYUSERNAME,
698	proxy_auth.username);
699	curl_easy_setopt(result, CURLOPT_PROXYPASSWORD,
700	proxy_auth.password);
701	} else if (proxy_auth.authtype && proxy_auth.credential) {
702	curl_easy_setopt(result, CURLOPT_PROXYHEADER,
703	http_append_auth_header(&proxy_auth, NULL));
704	}
705	}
706
707	static void init_curl_proxy_auth(CURL *result)
708	{
709	if (proxy_auth.username) {
710	if (!proxy_auth.password && !proxy_auth.credential)
711	credential_fill(the_repository, &proxy_auth, 1);
712	set_proxyauth_name_password(result);
713	}
714
715	var_override(&http_proxy_authmethod, getenv("GIT_HTTP_PROXY_AUTHMETHOD"));
716
717	if (http_proxy_authmethod) {
718	int i;
719	for (i = 0; i < ARRAY_SIZE(proxy_authmethods); i++) {
720	if (!strcmp(http_proxy_authmethod, proxy_authmethods[i].name)) {
721	curl_easy_setopt(result, CURLOPT_PROXYAUTH,
722	proxy_authmethods[i].curlauth_param);
723	break;
724	}
725	}
726	if (i == ARRAY_SIZE(proxy_authmethods)) {
727	warning("unsupported proxy authentication method %s: using anyauth",
728	http_proxy_authmethod);
729	curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
730	}
731	}
732	else
733	curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
734	}
735
736	static int has_cert_password(void)
737	{
738	if (ssl_cert == NULL \|\| ssl_cert_password_required != 1)
739	return 0;
740	if (!cert_auth.password) {
741	cert_auth.protocol = xstrdup("cert");
742	cert_auth.host = xstrdup("");
743	cert_auth.username = xstrdup("");
744	cert_auth.path = xstrdup(ssl_cert);
745	credential_fill(the_repository, &cert_auth, 0);
746	}
747	return 1;
748	}
749
750	static int has_proxy_cert_password(void)
751	{
752	if (http_proxy_ssl_cert == NULL \|\| proxy_ssl_cert_password_required != 1)
753	return 0;
754	if (!proxy_cert_auth.password) {
755	proxy_cert_auth.protocol = xstrdup("cert");
756	proxy_cert_auth.host = xstrdup("");
757	proxy_cert_auth.username = xstrdup("");
758	proxy_cert_auth.path = xstrdup(http_proxy_ssl_cert);
759	credential_fill(the_repository, &proxy_cert_auth, 0);
760	}
761	return 1;
762	}
763
764	static const struct socks_proxy_type {
765	const char *name;
766	long curlsym;
767	} socks_proxy_types[] = {
768	{ "socks", CURLPROXY_SOCKS4 },
769	{ "socks4", CURLPROXY_SOCKS4 },
770	{ "socks4a", CURLPROXY_SOCKS4A },
771	{ "socks5", CURLPROXY_SOCKS5 },
772	{ "socks5h", CURLPROXY_SOCKS5_HOSTNAME },
773	};
774
775	static const struct socks_proxy_type find_socks_proxy_type(const char protocol)
776	{
777	int i;
778
779	if (!protocol)
780	return NULL;
781
782	for (i = 0; i < ARRAY_SIZE(socks_proxy_types); i++) {
783	if (!strcmp(socks_proxy_types[i].name, protocol))
784	return &socks_proxy_types[i];
785	}
786
787	return NULL;
788	}
789
790	static int is_socks_proxy_protocol(const char *protocol)
791	{
792	return !!find_socks_proxy_type(protocol);
793	}
794
795	static int set_curl_proxy_type(CURL result, const char protocol)
796	{
797	const struct socks_proxy_type *socks_proxy_type;
798
799	if (!protocol \|\| !strcmp(protocol, "http"))
800	return 0;
801
802	socks_proxy_type = find_socks_proxy_type(protocol);
803	if (socks_proxy_type) {
804	curl_easy_setopt(result, CURLOPT_PROXYTYPE, socks_proxy_type->curlsym);
805	return 0;
806	}
807
808	if (!strcmp(protocol, "https")) {
809	curl_easy_setopt(result, CURLOPT_PROXYTYPE, (long)CURLPROXY_HTTPS);
810
811	if (http_proxy_ssl_cert)
812	curl_easy_setopt(result, CURLOPT_PROXY_SSLCERT,
813	http_proxy_ssl_cert);
814
815	if (http_proxy_ssl_key)
816	curl_easy_setopt(result, CURLOPT_PROXY_SSLKEY,
817	http_proxy_ssl_key);
818
819	if (has_proxy_cert_password())
820	curl_easy_setopt(result, CURLOPT_PROXY_KEYPASSWD,
821	proxy_cert_auth.password);
822
823	return 0;
824	}
825
826	return -1;
827	}
828
829	/* Return 1 if redactions have been made, 0 otherwise. */
830	static int redact_sensitive_header(struct strbuf *header, size_t offset)
831	{
832	int ret = 0;
833	char *sensitive_header;
834
835	if (trace_curl_redact &&
836	(skip_iprefix(header->buf + offset, "Authorization:", &sensitive_header) \|\|
837	skip_iprefix(header->buf + offset, "Proxy-Authorization:", &sensitive_header))) {
838	/* The first token is the type, which is OK to log */
839	while (isspace(*sensitive_header))
840	sensitive_header++;
841	while (sensitive_header && !isspace(sensitive_header))
842	sensitive_header++;
843	/* Everything else is opaque and possibly sensitive */
844	strbuf_setlen(header, sensitive_header - header->buf);
845	strbuf_addstr(header, " <redacted>");
846	ret = 1;
847	} else if (trace_curl_redact &&
848	skip_iprefix(header->buf + offset, "Cookie:", &sensitive_header)) {
849	struct strbuf redacted_header = STRBUF_INIT;
850	char *cookie;
851
852	while (isspace(*sensitive_header))
853	sensitive_header++;
854
855	cookie = sensitive_header;
856
857	while (cookie) {
858	char *equals;
859	char *semicolon = strstr(cookie, "; ");
860	if (semicolon)
861	*semicolon = 0;
862	equals = strchrnul(cookie, '=');
863	if (!equals) {
864	/* invalid cookie, just append and continue */
865	strbuf_addstr(&redacted_header, cookie);
866	continue;
867	}
868	strbuf_add(&redacted_header, cookie, equals - cookie);
869	strbuf_addstr(&redacted_header, "=<redacted>");
870	if (semicolon) {
871	/*
872	* There are more cookies. (Or, for some
873	* reason, the input string ends in "; ".)
874	*/
875	strbuf_addstr(&redacted_header, "; ");
876	cookie = semicolon + strlen("; ");
877	} else {
878	cookie = NULL;
879	}
880	}
881
882	strbuf_setlen(header, sensitive_header - header->buf);
883	strbuf_addbuf(header, &redacted_header);
884	strbuf_release(&redacted_header);
885	ret = 1;
886	}
887	return ret;
888	}
889
890	static int match_curl_h2_trace(const char line, const char *out)
891	{
892	const char *p;
893
894	/*
895	* curl prior to 8.1.0 gives us:
896	*
897	* h2h3 [<header-name>: <header-val>]
898	*
899	* Starting in 8.1.0, the first token became just "h2".
900	*/
901	if (skip_iprefix(line, "h2h3 [", out) \|\|
902	skip_iprefix(line, "h2 [", out))
903	return 1;
904
905	/*
906	* curl 8.3.0 uses:
907	* [HTTP/2] [<stream-id>] [<header-name>: <header-val>]
908	* where <stream-id> is numeric.
909	*/
910	if (skip_iprefix(line, "[HTTP/2] [", &p)) {
911	while (isdigit(*p))
912	p++;
913	if (skip_prefix(p, "] [", out))
914	return 1;
915	}
916
917	return 0;
918	}
919
920	/* Redact headers in info */
921	static void redact_sensitive_info_header(struct strbuf *header)
922	{
923	const char *sensitive_header;
924
925	if (trace_curl_redact &&
926	match_curl_h2_trace(header->buf, &sensitive_header)) {
927	if (redact_sensitive_header(header, sensitive_header - header->buf)) {
928	/* redaction ate our closing bracket */
929	strbuf_addch(header, ']');
930	}
931	}
932	}
933
934	static void curl_dump_header(const char text, unsigned char ptr, size_t size, int hide_sensitive_header)
935	{
936	struct strbuf out = STRBUF_INIT;
937	struct strbuf headers, header;
938
939	strbuf_addf(&out, "%s, %10.10ld bytes (0x%8.8lx)\n",
940	text, (long)size, (long)size);
941	trace_strbuf(&trace_curl, &out);
942	strbuf_reset(&out);
943	strbuf_add(&out, ptr, size);
944	headers = strbuf_split_max(&out, '\n', 0);
945
946	for (header = headers; *header; header++) {
947	if (hide_sensitive_header)
948	redact_sensitive_header(*header, 0);
949	strbuf_insertstr((*header), 0, text);
950	strbuf_insertstr((*header), strlen(text), ": ");
951	strbuf_rtrim((*header));
952	strbuf_addch((*header), '\n');
953	trace_strbuf(&trace_curl, (*header));
954	}
955	strbuf_list_free(headers);
956	strbuf_release(&out);
957	}
958
959	static void curl_dump_data(const char text, unsigned char ptr, size_t size)
960	{
961	size_t i;
962	struct strbuf out = STRBUF_INIT;
963	unsigned int width = 60;
964
965	strbuf_addf(&out, "%s, %10.10ld bytes (0x%8.8lx)\n",
966	text, (long)size, (long)size);
967	trace_strbuf(&trace_curl, &out);
968
969	for (i = 0; i < size; i += width) {
970	size_t w;
971
972	strbuf_reset(&out);
973	strbuf_addf(&out, "%s: ", text);
974	for (w = 0; (w < width) && (i + w < size); w++) {
975	unsigned char ch = ptr[i + w];
976
977	strbuf_addch(&out,
978	(ch >= 0x20) && (ch < 0x80)
979	? ch : '.');
980	}
981	strbuf_addch(&out, '\n');
982	trace_strbuf(&trace_curl, &out);
983	}
984	strbuf_release(&out);
985	}
986
987	static void curl_dump_info(char *data, size_t size)
988	{
989	struct strbuf buf = STRBUF_INIT;
990
991	strbuf_add(&buf, data, size);
992
993	redact_sensitive_info_header(&buf);
994	trace_printf_key(&trace_curl, "== Info: %s", buf.buf);
995
996	strbuf_release(&buf);
997	}
998
999	static int curl_trace(CURL *handle UNUSED, curl_infotype type,
1000	char *data, size_t size,
1001	void *userp UNUSED)
1002	{
1003	const char *text;
1004	enum { NO_FILTER = 0, DO_FILTER = 1 };
1005
1006	switch (type) {
1007	case CURLINFO_TEXT:
1008	curl_dump_info(data, size);
1009	break;
1010	case CURLINFO_HEADER_OUT:
1011	text = "=> Send header";
1012	curl_dump_header(text, (unsigned char *)data, size, DO_FILTER);
1013	break;
1014	case CURLINFO_DATA_OUT:
1015	if (trace_curl_data) {
1016	text = "=> Send data";
1017	curl_dump_data(text, (unsigned char *)data, size);
1018	}
1019	break;
1020	case CURLINFO_SSL_DATA_OUT:
1021	if (trace_curl_data) {
1022	text = "=> Send SSL data";
1023	curl_dump_data(text, (unsigned char *)data, size);
1024	}
1025	break;
1026	case CURLINFO_HEADER_IN:
1027	text = "<= Recv header";
1028	curl_dump_header(text, (unsigned char *)data, size, NO_FILTER);
1029	break;
1030	case CURLINFO_DATA_IN:
1031	if (trace_curl_data) {
1032	text = "<= Recv data";
1033	curl_dump_data(text, (unsigned char *)data, size);
1034	}
1035	break;
1036	case CURLINFO_SSL_DATA_IN:
1037	if (trace_curl_data) {
1038	text = "<= Recv SSL data";
1039	curl_dump_data(text, (unsigned char *)data, size);
1040	}
1041	break;
1042
1043	default: /* we ignore unknown types by default */
1044	return 0;
1045	}
1046	return 0;
1047	}
1048
1049	void http_trace_curl_no_data(void)
1050	{
1051	trace_override_envvar(&trace_curl, "1");
1052	trace_curl_data = 0;
1053	}
1054
1055	void setup_curl_trace(CURL *handle)
1056	{
1057	if (!trace_want(&trace_curl))
1058	return;
1059	curl_easy_setopt(handle, CURLOPT_VERBOSE, 1L);
1060	curl_easy_setopt(handle, CURLOPT_DEBUGFUNCTION, curl_trace);
1061	curl_easy_setopt(handle, CURLOPT_DEBUGDATA, NULL);
1062	}
1063
1064	static void proto_list_append(struct strbuf list, const char proto)
1065	{
1066	if (!list)
1067	return;
1068	if (list->len)
1069	strbuf_addch(list, ',');
1070	strbuf_addstr(list, proto);
1071	}
1072
1073	static long get_curl_allowed_protocols(int from_user, struct strbuf *list)
1074	{
1075	long bits = 0;
1076
1077	if (is_transport_allowed("http", from_user)) {
1078	bits \|= CURLPROTO_HTTP;
1079	proto_list_append(list, "http");
1080	}
1081	if (is_transport_allowed("https", from_user)) {
1082	bits \|= CURLPROTO_HTTPS;
1083	proto_list_append(list, "https");
1084	}
1085	if (is_transport_allowed("ftp", from_user)) {
1086	bits \|= CURLPROTO_FTP;
1087	proto_list_append(list, "ftp");
1088	}
1089	if (is_transport_allowed("ftps", from_user)) {
1090	bits \|= CURLPROTO_FTPS;
1091	proto_list_append(list, "ftps");
1092	}
1093
1094	return bits;
1095	}
1096
1097	static int get_curl_http_version_opt(const char version_string, long opt)
1098	{
1099	int i;
1100	static struct {
1101	const char *name;
1102	long opt_token;
1103	} choice[] = {
1104	{ "HTTP/1.1", CURL_HTTP_VERSION_1_1 },
1105	{ "HTTP/2", CURL_HTTP_VERSION_2 }
1106	};
1107
1108	for (i = 0; i < ARRAY_SIZE(choice); i++) {
1109	if (!strcmp(version_string, choice[i].name)) {
1110	*opt = choice[i].opt_token;
1111	return 0;
1112	}
1113	}
1114
1115	warning("unknown value given to http.version: '%s'", version_string);
1116	return -1; /* not found */
1117	}
1118
1119	static CURL *get_curl_handle(void)
1120	{
1121	CURL *result = curl_easy_init();
1122
1123	if (!result)
1124	die("curl_easy_init failed");
1125
1126	if (!curl_ssl_verify) {
1127	curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 0L);
1128	curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 0L);
1129	} else {
1130	/* Verify authenticity of the peer's certificate */
1131	curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 1L);
1132	/* The name in the cert must match whom we tried to connect */
1133	curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 2L);
1134	}
1135
1136	if (curl_http_version) {
1137	long opt;
1138	if (!get_curl_http_version_opt(curl_http_version, &opt)) {
1139	/* Set request use http version */
1140	curl_easy_setopt(result, CURLOPT_HTTP_VERSION, opt);
1141	}
1142	}
1143
1144	curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
1145	curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
1146
1147	#ifdef CURLGSSAPI_DELEGATION_FLAG
1148	if (curl_deleg) {
1149	int i;
1150	for (i = 0; i < ARRAY_SIZE(curl_deleg_levels); i++) {
1151	if (!strcmp(curl_deleg, curl_deleg_levels[i].name)) {
1152	curl_easy_setopt(result, CURLOPT_GSSAPI_DELEGATION,
1153	curl_deleg_levels[i].curl_deleg_param);
1154	break;
1155	}
1156	}
1157	if (i == ARRAY_SIZE(curl_deleg_levels))
1158	warning("Unknown delegation method '%s': using default",
1159	curl_deleg);
1160	}
1161	#endif
1162
1163	if (http_ssl_backend && !strcmp("schannel", http_ssl_backend) &&
1164	!http_schannel_check_revoke) {
1165	curl_easy_setopt(result, CURLOPT_SSL_OPTIONS, (long)CURLSSLOPT_NO_REVOKE);
1166	}
1167
1168	if (http_proactive_auth != PROACTIVE_AUTH_NONE)
1169	init_curl_http_auth(result);
1170
1171	if (getenv("GIT_SSL_VERSION"))
1172	ssl_version = getenv("GIT_SSL_VERSION");
1173	if (ssl_version && *ssl_version) {
1174	int i;
1175	for (i = 0; i < ARRAY_SIZE(sslversions); i++) {
1176	if (!strcmp(ssl_version, sslversions[i].name)) {
1177	curl_easy_setopt(result, CURLOPT_SSLVERSION,
1178	sslversions[i].ssl_version);
1179	break;
1180	}
1181	}
1182	if (i == ARRAY_SIZE(sslversions))
1183	warning("unsupported ssl version %s: using default",
1184	ssl_version);
1185	}
1186
1187	if (getenv("GIT_SSL_CIPHER_LIST"))
1188	ssl_cipherlist = getenv("GIT_SSL_CIPHER_LIST");
1189	if (ssl_cipherlist != NULL && *ssl_cipherlist)
1190	curl_easy_setopt(result, CURLOPT_SSL_CIPHER_LIST,
1191	ssl_cipherlist);
1192
1193	if (ssl_cert)
1194	curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
1195	if (ssl_cert_type)
1196	curl_easy_setopt(result, CURLOPT_SSLCERTTYPE, ssl_cert_type);
1197	if (has_cert_password())
1198	curl_easy_setopt(result, CURLOPT_KEYPASSWD, cert_auth.password);
1199	if (ssl_key)
1200	curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
1201	if (ssl_key_type)
1202	curl_easy_setopt(result, CURLOPT_SSLKEYTYPE, ssl_key_type);
1203	if (ssl_capath)
1204	curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
1205	if (ssl_pinnedkey)
1206	curl_easy_setopt(result, CURLOPT_PINNEDPUBLICKEY, ssl_pinnedkey);
1207	if (http_ssl_backend && !strcmp("schannel", http_ssl_backend) &&
1208	!http_schannel_use_ssl_cainfo) {
1209	curl_easy_setopt(result, CURLOPT_CAINFO, NULL);
1210	curl_easy_setopt(result, CURLOPT_PROXY_CAINFO, NULL);
1211	} else if (ssl_cainfo != NULL \|\| http_proxy_ssl_ca_info != NULL) {
1212	if (ssl_cainfo)
1213	curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
1214	if (http_proxy_ssl_ca_info)
1215	curl_easy_setopt(result, CURLOPT_PROXY_CAINFO, http_proxy_ssl_ca_info);
1216	}
1217
1218	if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
1219	curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
1220	curl_low_speed_limit);
1221	curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
1222	curl_low_speed_time);
1223	}
1224
1225	curl_easy_setopt(result, CURLOPT_MAXREDIRS, 20L);
1226	curl_easy_setopt(result, CURLOPT_POSTREDIR, (long)CURL_REDIR_POST_ALL);
1227
1228	#ifdef GIT_CURL_HAVE_CURLOPT_PROTOCOLS_STR
1229	{
1230	struct strbuf buf = STRBUF_INIT;
1231
1232	get_curl_allowed_protocols(0, &buf);
1233	curl_easy_setopt(result, CURLOPT_REDIR_PROTOCOLS_STR, buf.buf);
1234	strbuf_reset(&buf);
1235
1236	get_curl_allowed_protocols(-1, &buf);
1237	curl_easy_setopt(result, CURLOPT_PROTOCOLS_STR, buf.buf);
1238	strbuf_release(&buf);
1239	}
1240	#else
1241	curl_easy_setopt(result, CURLOPT_REDIR_PROTOCOLS,
1242	get_curl_allowed_protocols(0, NULL));
1243	curl_easy_setopt(result, CURLOPT_PROTOCOLS,
1244	get_curl_allowed_protocols(-1, NULL));
1245	#endif
1246
1247	if (getenv("GIT_CURL_VERBOSE"))
1248	http_trace_curl_no_data();
1249	setup_curl_trace(result);
1250	if (getenv("GIT_TRACE_CURL_NO_DATA"))
1251	trace_curl_data = 0;
1252	if (!git_env_bool("GIT_TRACE_REDACT", 1))
1253	trace_curl_redact = 0;
1254
1255	curl_easy_setopt(result, CURLOPT_USERAGENT,
1256	user_agent ? user_agent : git_user_agent());
1257
1258	if (curl_ftp_no_epsv)
1259	curl_easy_setopt(result, CURLOPT_FTP_USE_EPSV, 0L);
1260
1261	if (curl_ssl_try)
1262	curl_easy_setopt(result, CURLOPT_USE_SSL, CURLUSESSL_TRY);
1263
1264	/*
1265	* CURL also examines these variables as a fallback; but we need to query
1266	* them here in order to decide whether to prompt for missing password (cf.
1267	* init_curl_proxy_auth()).
1268	*
1269	* Unlike many other common environment variables, these are historically
1270	* lowercase only. It appears that CURL did not know this and implemented
1271	* only uppercase variants, which was later corrected to take both - with
1272	* the exception of http_proxy, which is lowercase only also in CURL. As
1273	* the lowercase versions are the historical quasi-standard, they take
1274	* precedence here, as in CURL.
1275	*/
1276	if (!curl_http_proxy) {
1277	if (http_auth.protocol && !strcmp(http_auth.protocol, "https")) {
1278	var_override(&curl_http_proxy, getenv("HTTPS_PROXY"));
1279	var_override(&curl_http_proxy, getenv("https_proxy"));
1280	} else {
1281	var_override(&curl_http_proxy, getenv("http_proxy"));
1282	}
1283	if (!curl_http_proxy) {
1284	var_override(&curl_http_proxy, getenv("ALL_PROXY"));
1285	var_override(&curl_http_proxy, getenv("all_proxy"));
1286	}
1287	}
1288
1289	if (curl_http_proxy && curl_http_proxy[0] == '\0') {
1290	/*
1291	* Handle case with the empty http.proxy value here to keep
1292	* common code clean.
1293	* NB: empty option disables proxying at all.
1294	*/
1295	curl_easy_setopt(result, CURLOPT_PROXY, "");
1296	} else if (curl_http_proxy) {
1297	struct strbuf proxy = STRBUF_INIT;
1298
1299	if (strstr(curl_http_proxy, "://"))
1300	credential_from_url(&proxy_auth, curl_http_proxy);
1301	else {
1302	struct strbuf url = STRBUF_INIT;
1303	strbuf_addf(&url, "http://%s", curl_http_proxy);
1304	credential_from_url(&proxy_auth, url.buf);
1305	strbuf_release(&url);
1306	}
1307
1308	if (set_curl_proxy_type(result, proxy_auth.protocol) < 0)
1309	die("Invalid proxy URL '%s': unsupported proxy scheme '%s'",
1310	curl_http_proxy, proxy_auth.protocol);
1311
1312	if (!proxy_auth.host)
1313	die("Invalid proxy URL '%s'", curl_http_proxy);
1314
1315	strbuf_addstr(&proxy, proxy_auth.host);
1316	if (proxy_auth.path) {
1317	curl_version_info_data *ver = curl_version_info(CURLVERSION_NOW);
1318
1319	if (ver->version_num < 0x075400)
1320	die("libcurl 7.84 or later is required to support paths in proxy URLs");
1321
1322	if (!is_socks_proxy_protocol(proxy_auth.protocol))
1323	die("Invalid proxy URL '%s': only SOCKS proxies support paths",
1324	curl_http_proxy);
1325
1326	if (strcasecmp(proxy_auth.host, "localhost"))
1327	die("Invalid proxy URL '%s': host must be localhost if a path is present",
1328	curl_http_proxy);
1329
1330	strbuf_addch(&proxy, '/');
1331	strbuf_add_percentencode(&proxy, proxy_auth.path, 0);
1332	}
1333	curl_easy_setopt(result, CURLOPT_PROXY, proxy.buf);
1334	strbuf_release(&proxy);
1335
1336	var_override(&curl_no_proxy, getenv("NO_PROXY"));
1337	var_override(&curl_no_proxy, getenv("no_proxy"));
1338	curl_easy_setopt(result, CURLOPT_NOPROXY, curl_no_proxy);
1339	}
1340	init_curl_proxy_auth(result);
1341
1342	curl_easy_setopt(result, CURLOPT_TCP_KEEPALIVE, 1L);
1343
1344	if (curl_tcp_keepidle > -1)
1345	curl_easy_setopt(result, CURLOPT_TCP_KEEPIDLE,
1346	curl_tcp_keepidle);
1347	if (curl_tcp_keepintvl > -1)
1348	curl_easy_setopt(result, CURLOPT_TCP_KEEPINTVL,
1349	curl_tcp_keepintvl);
1350	#ifdef GIT_CURL_HAVE_CURLOPT_TCP_KEEPCNT
1351	if (curl_tcp_keepcnt > -1)
1352	curl_easy_setopt(result, CURLOPT_TCP_KEEPCNT, curl_tcp_keepcnt);
1353	#endif
1354
1355	return result;
1356	}
1357
1358	static void set_from_env(char *var, const char envname)
1359	{
1360	const char *val = getenv(envname);
1361	if (val) {
1362	FREE_AND_NULL(*var);
1363	*var = xstrdup(val);
1364	}
1365	}
1366
1367	static void set_long_from_env(long var, const char envname)
1368	{
1369	const char *val = getenv(envname);
1370	if (val) {
1371	long tmp;
1372	char *endp;
1373	int saved_errno = errno;
1374
1375	errno = 0;
1376	tmp = strtol(val, &endp, 10);
1377
1378	if (errno)
1379	warning_errno(_("failed to parse %s"), envname);
1380	else if (*endp \|\| endp == val)
1381	warning(_("failed to parse %s"), envname);
1382	else
1383	*var = tmp;
1384
1385	errno = saved_errno;
1386	}
1387	}
1388
1389	void http_init(struct remote remote, const char url, int proactive_auth)
1390	{
1391	char *normalized_url;
1392	struct urlmatch_config config = URLMATCH_CONFIG_INIT;
1393
1394	config.section = "http";
1395	config.key = NULL;
1396	config.collect_fn = http_options;
1397	config.cascade_fn = git_default_config;
1398	config.cb = NULL;
1399
1400	http_is_verbose = 0;
1401	normalized_url = url_normalize(url, &config.url);
1402
1403	repo_config(the_repository, urlmatch_config_entry, &config);
1404	free(normalized_url);
1405	string_list_clear(&config.vars, 1);
1406
1407	if (http_ssl_backend) {
1408	const curl_ssl_backend **backends;
1409	struct strbuf buf = STRBUF_INIT;
1410	int i;
1411
1412	switch (curl_global_sslset(-1, http_ssl_backend, &backends)) {
1413	case CURLSSLSET_UNKNOWN_BACKEND:
1414	strbuf_addf(&buf, _("Unsupported SSL backend '%s'. "
1415	"Supported SSL backends:"),
1416	http_ssl_backend);
1417	for (i = 0; backends[i]; i++)
1418	strbuf_addf(&buf, "\n\t%s", backends[i]->name);
1419	die("%s", buf.buf);
1420	case CURLSSLSET_NO_BACKENDS:
1421	die(_("Could not set SSL backend to '%s': "
1422	"cURL was built without SSL backends"),
1423	http_ssl_backend);
1424	case CURLSSLSET_TOO_LATE:
1425	die(_("Could not set SSL backend to '%s': already set"),
1426	http_ssl_backend);
1427	case CURLSSLSET_OK:
1428	break; /* Okay! */
1429	}
1430	}
1431
1432	if (curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK)
1433	die("curl_global_init failed");
1434
1435	#ifdef GIT_CURL_HAVE_GLOBAL_TRACE
1436	{
1437	const char *comp = getenv("GIT_TRACE_CURL_COMPONENTS");
1438	if (comp)
1439	curl_global_trace(comp);
1440	}
1441	#endif
1442
1443	if (proactive_auth && http_proactive_auth == PROACTIVE_AUTH_NONE)
1444	http_proactive_auth = PROACTIVE_AUTH_IF_CREDENTIALS;
1445
1446	if (remote && remote->http_proxy)
1447	curl_http_proxy = xstrdup(remote->http_proxy);
1448
1449	if (remote)
1450	var_override(&http_proxy_authmethod, remote->http_proxy_authmethod);
1451
1452	pragma_header = curl_slist_append(http_copy_default_headers(),
1453	"Pragma: no-cache");
1454
1455	{
1456	char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
1457	if (http_max_requests)
1458	max_requests = atoi(http_max_requests);
1459	}
1460
1461	curlm = curl_multi_init();
1462	if (!curlm)
1463	die("curl_multi_init failed");
1464
1465	if (getenv("GIT_SSL_NO_VERIFY"))
1466	curl_ssl_verify = 0;
1467
1468	set_from_env(&ssl_cert, "GIT_SSL_CERT");
1469	set_from_env(&ssl_cert_type, "GIT_SSL_CERT_TYPE");
1470	set_from_env(&ssl_key, "GIT_SSL_KEY");
1471	set_from_env(&ssl_key_type, "GIT_SSL_KEY_TYPE");
1472	set_from_env(&ssl_capath, "GIT_SSL_CAPATH");
1473	set_from_env(&ssl_cainfo, "GIT_SSL_CAINFO");
1474
1475	set_from_env(&user_agent, "GIT_HTTP_USER_AGENT");
1476
1477	set_long_from_env(&curl_low_speed_limit, "GIT_HTTP_LOW_SPEED_LIMIT");
1478	set_long_from_env(&curl_low_speed_time, "GIT_HTTP_LOW_SPEED_TIME");
1479
1480	if (curl_ssl_verify == -1)
1481	curl_ssl_verify = 1;
1482
1483	curl_session_count = 0;
1484	if (max_requests < 1)
1485	max_requests = DEFAULT_MAX_REQUESTS;
1486
1487	set_from_env(&http_proxy_ssl_cert, "GIT_PROXY_SSL_CERT");
1488	set_from_env(&http_proxy_ssl_key, "GIT_PROXY_SSL_KEY");
1489	set_from_env(&http_proxy_ssl_ca_info, "GIT_PROXY_SSL_CAINFO");
1490
1491	if (getenv("GIT_PROXY_SSL_CERT_PASSWORD_PROTECTED"))
1492	proxy_ssl_cert_password_required = 1;
1493
1494	if (getenv("GIT_CURL_FTP_NO_EPSV"))
1495	curl_ftp_no_epsv = 1;
1496
1497	if (url) {
1498	credential_from_url(&http_auth, url);
1499	if (!ssl_cert_password_required &&
1500	getenv("GIT_SSL_CERT_PASSWORD_PROTECTED") &&
1501	starts_with(url, "https://"))
1502	ssl_cert_password_required = 1;
1503	}
1504
1505	set_long_from_env(&curl_tcp_keepidle, "GIT_TCP_KEEPIDLE");
1506	set_long_from_env(&curl_tcp_keepintvl, "GIT_TCP_KEEPINTVL");
1507	set_long_from_env(&curl_tcp_keepcnt, "GIT_TCP_KEEPCNT");
1508
1509	set_long_from_env(&http_retry_after, "GIT_HTTP_RETRY_AFTER");
1510	set_long_from_env(&http_max_retries, "GIT_HTTP_MAX_RETRIES");
1511	set_long_from_env(&http_max_retry_time, "GIT_HTTP_MAX_RETRY_TIME");
1512
1513	curl_default = get_curl_handle();
1514	}
1515
1516	void http_cleanup(void)
1517	{
1518	struct active_request_slot *slot = active_queue_head;
1519
1520	while (slot != NULL) {
1521	struct active_request_slot *next = slot->next;
1522	if (slot->curl) {
1523	xmulti_remove_handle(slot);
1524	curl_easy_cleanup(slot->curl);
1525	}
1526	free(slot);
1527	slot = next;
1528	}
1529	active_queue_head = NULL;
1530
1531	curl_easy_cleanup(curl_default);
1532
1533	curl_multi_cleanup(curlm);
1534	curl_global_cleanup();
1535
1536	string_list_clear(&extra_http_headers, 0);
1537
1538	curl_slist_free_all(pragma_header);
1539	pragma_header = NULL;
1540
1541	curl_slist_free_all(host_resolutions);
1542	host_resolutions = NULL;
1543
1544	if (curl_http_proxy) {
1545	free((void *)curl_http_proxy);
1546	curl_http_proxy = NULL;
1547	}
1548
1549	if (proxy_auth.password) {
1550	memset(proxy_auth.password, 0, strlen(proxy_auth.password));
1551	FREE_AND_NULL(proxy_auth.password);
1552	}
1553
1554	free((void *)curl_proxyuserpwd);
1555	curl_proxyuserpwd = NULL;
1556
1557	free((void *)http_proxy_authmethod);
1558	http_proxy_authmethod = NULL;
1559
1560	if (cert_auth.password) {
1561	memset(cert_auth.password, 0, strlen(cert_auth.password));
1562	FREE_AND_NULL(cert_auth.password);
1563	}
1564	ssl_cert_password_required = 0;
1565
1566	if (proxy_cert_auth.password) {
1567	memset(proxy_cert_auth.password, 0, strlen(proxy_cert_auth.password));
1568	FREE_AND_NULL(proxy_cert_auth.password);
1569	}
1570	proxy_ssl_cert_password_required = 0;
1571
1572	FREE_AND_NULL(cached_accept_language);
1573	}
1574
1575	struct active_request_slot *get_active_slot(void)
1576	{
1577	struct active_request_slot *slot = active_queue_head;
1578	struct active_request_slot *newslot;
1579
1580	int num_transfers;
1581
1582	/* Wait for a slot to open up if the queue is full */
1583	while (active_requests >= max_requests) {
1584	curl_multi_perform(curlm, &num_transfers);
1585	if (num_transfers < active_requests)
1586	process_curl_messages();
1587	}
1588
1589	while (slot != NULL && slot->in_use)
1590	slot = slot->next;
1591
1592	if (!slot) {
1593	newslot = xmalloc(sizeof(*newslot));
1594	newslot->curl = NULL;
1595	newslot->in_use = 0;
1596	newslot->next = NULL;
1597
1598	slot = active_queue_head;
1599	if (!slot) {
1600	active_queue_head = newslot;
1601	} else {
1602	while (slot->next != NULL)
1603	slot = slot->next;
1604	slot->next = newslot;
1605	}
1606	slot = newslot;
1607	}
1608
1609	if (!slot->curl) {
1610	slot->curl = curl_easy_duphandle(curl_default);
1611	if (!slot->curl)
1612	die("curl_easy_duphandle failed");
1613	curl_session_count++;
1614	}
1615
1616	active_requests++;
1617	slot->in_use = 1;
1618	slot->results = NULL;
1619	slot->finished = NULL;
1620	slot->callback_data = NULL;
1621	slot->callback_func = NULL;
1622
1623	if (curl_cookie_file && !strcmp(curl_cookie_file, "-")) {
1624	warning(_("refusing to read cookies from http.cookiefile '-'"));
1625	FREE_AND_NULL(curl_cookie_file);
1626	}
1627	curl_easy_setopt(slot->curl, CURLOPT_COOKIEFILE, curl_cookie_file);
1628	if (curl_save_cookies && (!curl_cookie_file \|\| !curl_cookie_file[0])) {
1629	curl_save_cookies = 0;
1630	warning(_("ignoring http.savecookies for empty http.cookiefile"));
1631	}
1632	if (curl_save_cookies)
1633	curl_easy_setopt(slot->curl, CURLOPT_COOKIEJAR, curl_cookie_file);
1634	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
1635	curl_easy_setopt(slot->curl, CURLOPT_RESOLVE, host_resolutions);
1636	curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
1637	curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, NULL);
1638	curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, NULL);
1639	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, NULL);
1640	curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, NULL);
1641	curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, -1L);
1642	curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 0L);
1643	curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1L);
1644	curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 1L);
1645	curl_easy_setopt(slot->curl, CURLOPT_RANGE, NULL);
1646
1647	/*
1648	* Default following to off unless "ALWAYS" is configured; this gives
1649	* callers a sane starting point, and they can tweak for individual
1650	* HTTP_FOLLOW_* cases themselves.
1651	*/
1652	if (http_follow_config == HTTP_FOLLOW_ALWAYS)
1653	curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 1L);
1654	else
1655	curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 0L);
1656
1657	curl_easy_setopt(slot->curl, CURLOPT_IPRESOLVE, git_curl_ipresolve);
1658	curl_easy_setopt(slot->curl, CURLOPT_HTTPAUTH, http_auth_methods);
1659	if (http_auth.password \|\| http_auth.credential \|\| curl_empty_auth_enabled())
1660	init_curl_http_auth(slot->curl);
1661
1662	return slot;
1663	}
1664
1665	int start_active_slot(struct active_request_slot *slot)
1666	{
1667	CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
1668	int num_transfers;
1669
1670	if (curlm_result != CURLM_OK &&
1671	curlm_result != CURLM_CALL_MULTI_PERFORM) {
1672	warning("curl_multi_add_handle failed: %s",
1673	curl_multi_strerror(curlm_result));
1674	active_requests--;
1675	slot->in_use = 0;
1676	return 0;
1677	}
1678
1679	/*
1680	* We know there must be something to do, since we just added
1681	* something.
1682	*/
1683	curl_multi_perform(curlm, &num_transfers);
1684	return 1;
1685	}
1686
1687	struct fill_chain {
1688	void *data;
1689	int (fill)(void );
1690	struct fill_chain *next;
1691	};
1692
1693	static struct fill_chain *fill_cfg;
1694
1695	void add_fill_function(void data, int (fill)(void *))
1696	{
1697	struct fill_chain new_fill = xmalloc(sizeof(new_fill));
1698	struct fill_chain **linkp = &fill_cfg;
1699	new_fill->data = data;
1700	new_fill->fill = fill;
1701	new_fill->next = NULL;
1702	while (*linkp)
1703	linkp = &(*linkp)->next;
1704	*linkp = new_fill;
1705	}
1706
1707	void fill_active_slots(void)
1708	{
1709	struct active_request_slot *slot = active_queue_head;
1710
1711	while (active_requests < max_requests) {
1712	struct fill_chain *fill;
1713	for (fill = fill_cfg; fill; fill = fill->next)
1714	if (fill->fill(fill->data))
1715	break;
1716
1717	if (!fill)
1718	break;
1719	}
1720
1721	while (slot != NULL) {
1722	if (!slot->in_use && slot->curl != NULL
1723	&& curl_session_count > min_curl_sessions) {
1724	curl_easy_cleanup(slot->curl);
1725	slot->curl = NULL;
1726	curl_session_count--;
1727	}
1728	slot = slot->next;
1729	}
1730	}
1731
1732	void step_active_slots(void)
1733	{
1734	int num_transfers;
1735	CURLMcode curlm_result;
1736
1737	do {
1738	curlm_result = curl_multi_perform(curlm, &num_transfers);
1739	} while (curlm_result == CURLM_CALL_MULTI_PERFORM);
1740	if (num_transfers < active_requests) {
1741	process_curl_messages();
1742	fill_active_slots();
1743	}
1744	}
1745
1746	void run_active_slot(struct active_request_slot *slot)
1747	{
1748	fd_set readfds;
1749	fd_set writefds;
1750	fd_set excfds;
1751	int max_fd;
1752	struct timeval select_timeout;
1753	int finished = 0;
1754
1755	slot->finished = &finished;
1756	while (!finished) {
1757	step_active_slots();
1758
1759	if (slot->in_use) {
1760	long curl_timeout;
1761	curl_multi_timeout(curlm, &curl_timeout);
1762	if (curl_timeout == 0) {
1763	continue;
1764	} else if (curl_timeout == -1) {
1765	select_timeout.tv_sec = 0;
1766	select_timeout.tv_usec = 50000;
1767	} else {
1768	select_timeout.tv_sec = curl_timeout / 1000;
1769	select_timeout.tv_usec = (curl_timeout % 1000) * 1000;
1770	}
1771
1772	max_fd = -1;
1773	FD_ZERO(&readfds);
1774	FD_ZERO(&writefds);
1775	FD_ZERO(&excfds);
1776	curl_multi_fdset(curlm, &readfds, &writefds, &excfds, &max_fd);
1777
1778	/*
1779	* It can happen that curl_multi_timeout returns a pathologically
1780	* long timeout when curl_multi_fdset returns no file descriptors
1781	* to read. See commit message for more details.
1782	*/
1783	if (max_fd < 0 &&
1784	(select_timeout.tv_sec > 0 \|\|
1785	select_timeout.tv_usec > 50000)) {
1786	select_timeout.tv_sec = 0;
1787	select_timeout.tv_usec = 50000;
1788	}
1789
1790	select(max_fd+1, &readfds, &writefds, &excfds, &select_timeout);
1791	}
1792	}
1793
1794	/*
1795	* The value of slot->finished we set before the loop was used
1796	* to set our "finished" variable when our request completed.
1797	*
1798	* 1. The slot may not have been reused for another request
1799	* yet, in which case it still has &finished.
1800	*
1801	* 2. The slot may already be in-use to serve another request,
1802	* which can further be divided into two cases:
1803	*
1804	* (a) If call run_active_slot() hasn't been called for that
1805	* other request, slot->finished would have been cleared
1806	* by get_active_slot() and has NULL.
1807	*
1808	* (b) If the request did call run_active_slot(), then the
1809	* call would have updated slot->finished at the beginning
1810	* of this function, and with the clearing of the member
1811	* below, we would find that slot->finished is now NULL.
1812	*
1813	* In all cases, slot->finished has no useful information to
1814	* anybody at this point. Some compilers warn us for
1815	* attempting to smuggle a pointer that is about to become
1816	* invalid, i.e. &finished. We clear it here to assure them.
1817	*/
1818	slot->finished = NULL;
1819	}
1820
1821	static void release_active_slot(struct active_request_slot *slot)
1822	{
1823	closedown_active_slot(slot);
1824	if (slot->curl) {
1825	xmulti_remove_handle(slot);
1826	if (curl_session_count > min_curl_sessions) {
1827	curl_easy_cleanup(slot->curl);
1828	slot->curl = NULL;
1829	curl_session_count--;
1830	}
1831	}
1832	fill_active_slots();
1833	}
1834
1835	void finish_all_active_slots(void)
1836	{
1837	struct active_request_slot *slot = active_queue_head;
1838
1839	while (slot != NULL)
1840	if (slot->in_use) {
1841	run_active_slot(slot);
1842	slot = active_queue_head;
1843	} else {
1844	slot = slot->next;
1845	}
1846	}
1847
1848	/* Helpers for modifying and creating URLs */
1849	static inline int needs_quote(int ch)
1850	{
1851	if (((ch >= 'A') && (ch <= 'Z'))
1852	\|\| ((ch >= 'a') && (ch <= 'z'))
1853	\|\| ((ch >= '0') && (ch <= '9'))
1854	\|\| (ch == '/')
1855	\|\| (ch == '-')
1856	\|\| (ch == '.'))
1857	return 0;
1858	return 1;
1859	}
1860
1861	static char quote_ref_url(const char base, const char *ref)
1862	{
1863	struct strbuf buf = STRBUF_INIT;
1864	const char *cp;
1865	int ch;
1866
1867	end_url_with_slash(&buf, base);
1868
1869	for (cp = ref; (ch = *cp) != 0; cp++)
1870	if (needs_quote(ch))
1871	strbuf_addf(&buf, "%%%02x", ch);
1872	else
1873	strbuf_addch(&buf, *cp);
1874
1875	return strbuf_detach(&buf, NULL);
1876	}
1877
1878	void append_remote_object_url(struct strbuf buf, const char url,
1879	const char *hex,
1880	int only_two_digit_prefix)
1881	{
1882	end_url_with_slash(buf, url);
1883
1884	strbuf_addf(buf, "objects/%.*s/", 2, hex);
1885	if (!only_two_digit_prefix)
1886	strbuf_addstr(buf, hex + 2);
1887	}
1888
1889	char get_remote_object_url(const char url, const char *hex,
1890	int only_two_digit_prefix)
1891	{
1892	struct strbuf buf = STRBUF_INIT;
1893	append_remote_object_url(&buf, url, hex, only_two_digit_prefix);
1894	return strbuf_detach(&buf, NULL);
1895	}
1896
1897	void normalize_curl_result(CURLcode *result, long http_code,
1898	char *errorstr, size_t errorlen)
1899	{
1900	/*
1901	* If we see a failing http code with CURLE_OK, we have turned off
1902	* FAILONERROR (to keep the server's custom error response), and should
1903	* translate the code into failure here.
1904	*
1905	* Likewise, if we see a redirect (30x code), that means we turned off
1906	* redirect-following, and we should treat the result as an error.
1907	*/
1908	if (*result == CURLE_OK && http_code >= 300) {
1909	*result = CURLE_HTTP_RETURNED_ERROR;
1910	/*
1911	* Normally curl will already have put the "reason phrase"
1912	* from the server into curl_errorstr; unfortunately without
1913	* FAILONERROR it is lost, so we can give only the numeric
1914	* status code.
1915	*/
1916	xsnprintf(errorstr, errorlen,
1917	"The requested URL returned error: %ld",
1918	http_code);
1919	}
1920	}
1921
1922	static int handle_curl_result(struct slot_results *results)
1923	{
1924	normalize_curl_result(&results->curl_result, results->http_code,
1925	curl_errorstr, sizeof(curl_errorstr));
1926
1927	if (results->curl_result == CURLE_OK) {
1928	credential_approve(the_repository, &http_auth);
1929	credential_approve(the_repository, &proxy_auth);
1930	credential_approve(the_repository, &cert_auth);
1931	return HTTP_OK;
1932	} else if (results->curl_result == CURLE_SSL_CERTPROBLEM) {
1933	/*
1934	* We can't tell from here whether it's a bad path, bad
1935	* certificate, bad password, or something else wrong
1936	* with the certificate. So we reject the credential to
1937	* avoid caching or saving a bad password.
1938	*/
1939	credential_reject(the_repository, &cert_auth);
1940	return HTTP_NOAUTH;
1941	} else if (results->curl_result == CURLE_SSL_PINNEDPUBKEYNOTMATCH) {
1942	return HTTP_NOMATCHPUBLICKEY;
1943	} else if (missing_target(results))
1944	return HTTP_MISSING_TARGET;
1945	else if (results->http_code == 401) {
1946	if ((http_auth.username && http_auth.password) \|\|\
1947	(http_auth.authtype && http_auth.credential)) {
1948	if (http_auth.multistage) {
1949	credential_clear_secrets(&http_auth);
1950	return HTTP_REAUTH;
1951	}
1952	credential_reject(the_repository, &http_auth);
1953	if (always_auth_proactively())
1954	http_proactive_auth = PROACTIVE_AUTH_NONE;
1955	return HTTP_NOAUTH;
1956	} else {
1957	if (curl_empty_auth == -1 &&
1958	!empty_auth_try_negotiate &&
1959	(results->auth_avail & CURLAUTH_GSSNEGOTIATE)) {
1960	/*
1961	* In auto mode, give Negotiate a chance via
1962	* empty auth before stripping it. If it fails,
1963	* we will strip it on the next 401.
1964	*/
1965	empty_auth_try_negotiate = 1;
1966	} else {
1967	http_auth_methods &= ~CURLAUTH_GSSNEGOTIATE;
1968	}
1969	if (results->auth_avail) {
1970	http_auth_methods &= results->auth_avail;
1971	http_auth_methods_restricted = 1;
1972	}
1973	return HTTP_REAUTH;
1974	}
1975	} else if (results->http_code == 429) {
1976	trace2_data_intmax("http", the_repository, "http/429-retry-after",
1977	results->retry_after);
1978	return HTTP_RATE_LIMITED;
1979	} else {
1980	if (results->http_connectcode == 407)
1981	credential_reject(the_repository, &proxy_auth);
1982	if (!curl_errorstr[0])
1983	strlcpy(curl_errorstr,
1984	curl_easy_strerror(results->curl_result),
1985	sizeof(curl_errorstr));
1986	return HTTP_ERROR;
1987	}
1988	}
1989
1990	int run_one_slot(struct active_request_slot *slot,
1991	struct slot_results *results)
1992	{
1993	slot->results = results;
1994
1995	if (!start_active_slot(slot)) {
1996	xsnprintf(curl_errorstr, sizeof(curl_errorstr),
1997	"failed to start HTTP request");
1998	return HTTP_START_FAILED;
1999	}
2000
2001	run_active_slot(slot);
2002	return handle_curl_result(results);
2003	}
2004
2005	struct curl_slist *http_copy_default_headers(void)
2006	{
2007	struct curl_slist *headers = NULL;
2008	const struct string_list_item *item;
2009
2010	for_each_string_list_item(item, &extra_http_headers)
2011	headers = curl_slist_append(headers, item->string);
2012
2013	return headers;
2014	}
2015
2016	static CURLcode curlinfo_strbuf(CURL curl, CURLINFO info, struct strbuf buf)
2017	{
2018	char *ptr;
2019	CURLcode ret;
2020
2021	strbuf_reset(buf);
2022	ret = curl_easy_getinfo(curl, info, &ptr);
2023	if (!ret && ptr)
2024	strbuf_addstr(buf, ptr);
2025	return ret;
2026	}
2027
2028	/*
2029	* Check for and extract a content-type parameter. "raw"
2030	* should be positioned at the start of the potential
2031	* parameter, with any whitespace already removed.
2032	*
2033	* "name" is the name of the parameter. The value is appended
2034	* to "out".
2035	*/
2036	static int extract_param(const char raw, const char name,
2037	struct strbuf *out)
2038	{
2039	size_t len = strlen(name);
2040
2041	if (strncasecmp(raw, name, len))
2042	return -1;
2043	raw += len;
2044
2045	if (*raw != '=')
2046	return -1;
2047	raw++;
2048
2049	while (raw && !isspace(raw) && *raw != ';')
2050	strbuf_addch(out, *raw++);
2051	return 0;
2052	}
2053
2054	/*
2055	* Extract a normalized version of the content type, with any
2056	* spaces suppressed, all letters lowercased, and no trailing ";"
2057	* or parameters.
2058	*
2059	* Note that we will silently remove even invalid whitespace. For
2060	* example, "text / plain" is specifically forbidden by RFC 2616,
2061	* but "text/plain" is the only reasonable output, and this keeps
2062	* our code simple.
2063	*
2064	* If the "charset" argument is not NULL, store the value of any
2065	* charset parameter there.
2066	*
2067	* Example:
2068	* "TEXT/PLAIN; charset=utf-8" -> "text/plain", "utf-8"
2069	* "text / plain" -> "text/plain"
2070	*/
2071	static void extract_content_type(struct strbuf raw, struct strbuf type,
2072	struct strbuf *charset)
2073	{
2074	const char *p;
2075
2076	strbuf_reset(type);
2077	strbuf_grow(type, raw->len);
2078	for (p = raw->buf; *p; p++) {
2079	if (isspace(*p))
2080	continue;
2081	if (*p == ';') {
2082	p++;
2083	break;
2084	}
2085	strbuf_addch(type, tolower(*p));
2086	}
2087
2088	if (!charset)
2089	return;
2090
2091	strbuf_reset(charset);
2092	while (*p) {
2093	while (isspace(p) \|\| p == ';')
2094	p++;
2095	if (!extract_param(p, "charset", charset))
2096	return;
2097	while (p && !isspace(p))
2098	p++;
2099	}
2100
2101	if (!charset->len && starts_with(type->buf, "text/"))
2102	strbuf_addstr(charset, "ISO-8859-1");
2103	}
2104
2105	static void write_accept_language(struct strbuf *buf)
2106	{
2107	/*
2108	* MAX_DECIMAL_PLACES must not be larger than 3. If it is larger than
2109	* that, q-value will be smaller than 0.001, the minimum q-value the
2110	* HTTP specification allows. See
2111	* https://datatracker.ietf.org/doc/html/rfc7231#section-5.3.1 for q-value.
2112	*/
2113	const int MAX_DECIMAL_PLACES = 3;
2114	const int MAX_LANGUAGE_TAGS = 1000;
2115	const int MAX_ACCEPT_LANGUAGE_HEADER_SIZE = 4000;
2116	char **language_tags = NULL;
2117	int num_langs = 0;
2118	const char *s = get_preferred_languages();
2119	int i;
2120	struct strbuf tag = STRBUF_INIT;
2121
2122	/* Don't add Accept-Language header if no language is preferred. */
2123	if (!s)
2124	return;
2125
2126	/*
2127	* Split the colon-separated string of preferred languages into
2128	* language_tags array.
2129	*/
2130	do {
2131	/* collect language tag */
2132	for (; s && (isalnum(s) \|\| *s == '_'); s++)
2133	strbuf_addch(&tag, s == '_' ? '-' : s);
2134
2135	/* skip .codeset, @modifier and any other unnecessary parts */
2136	while (s && s != ':')
2137	s++;
2138
2139	if (tag.len) {
2140	num_langs++;
2141	REALLOC_ARRAY(language_tags, num_langs);
2142	language_tags[num_langs - 1] = strbuf_detach(&tag, NULL);
2143	if (num_langs >= MAX_LANGUAGE_TAGS - 1) /* -1 for '' /
2144	break;
2145	}
2146	} while (*s++);
2147
2148	/* write Accept-Language header into buf */
2149	if (num_langs) {
2150	int last_buf_len = 0;
2151	int max_q;
2152	int decimal_places;
2153	char q_format[32];
2154
2155	/* add '' /
2156	REALLOC_ARRAY(language_tags, num_langs + 1);
2157	language_tags[num_langs++] = xstrdup("*");
2158
2159	/* compute decimal_places */
2160	for (max_q = 1, decimal_places = 0;
2161	max_q < num_langs && decimal_places <= MAX_DECIMAL_PLACES;
2162	decimal_places++, max_q *= 10)
2163	;
2164
2165	xsnprintf(q_format, sizeof(q_format), ";q=0.%%0%dd", decimal_places);
2166
2167	strbuf_addstr(buf, "Accept-Language: ");
2168
2169	for (i = 0; i < num_langs; i++) {
2170	if (i > 0)
2171	strbuf_addstr(buf, ", ");
2172
2173	strbuf_addstr(buf, language_tags[i]);
2174
2175	if (i > 0)
2176	strbuf_addf(buf, q_format, max_q - i);
2177
2178	if (buf->len > MAX_ACCEPT_LANGUAGE_HEADER_SIZE) {
2179	strbuf_remove(buf, last_buf_len, buf->len - last_buf_len);
2180	break;
2181	}
2182
2183	last_buf_len = buf->len;
2184	}
2185	}
2186
2187	for (i = 0; i < num_langs; i++)
2188	free(language_tags[i]);
2189	free(language_tags);
2190	}
2191
2192	/*
2193	* Get an Accept-Language header which indicates user's preferred languages.
2194	*
2195	* Examples:
2196	* LANGUAGE= -> ""
2197	* LANGUAGE=ko:en -> "Accept-Language: ko, en; q=0.9, *; q=0.1"
2198	* LANGUAGE=ko_KR.UTF-8:sr@latin -> "Accept-Language: ko-KR, sr; q=0.9, *; q=0.1"
2199	* LANGUAGE=ko LANG=en_US.UTF-8 -> "Accept-Language: ko, *; q=0.1"
2200	* LANGUAGE= LANG=en_US.UTF-8 -> "Accept-Language: en-US, *; q=0.1"
2201	* LANGUAGE= LANG=C -> ""
2202	*/
2203	const char *http_get_accept_language_header(void)
2204	{
2205	if (!cached_accept_language) {
2206	struct strbuf buf = STRBUF_INIT;
2207	write_accept_language(&buf);
2208	if (buf.len > 0)
2209	cached_accept_language = strbuf_detach(&buf, NULL);
2210	}
2211
2212	return cached_accept_language;
2213	}
2214
2215	static void http_opt_request_remainder(CURL *curl, off_t pos)
2216	{
2217	char buf[128];
2218	xsnprintf(buf, sizeof(buf), "%"PRIuMAX"-", (uintmax_t)pos);
2219	curl_easy_setopt(curl, CURLOPT_RANGE, buf);
2220	}
2221
2222	/* http_request() targets */
2223	#define HTTP_REQUEST_STRBUF 0
2224	#define HTTP_REQUEST_FILE 1
2225
2226	static int http_request(const char *url,
2227	void *result, int target,
2228	struct http_get_options *options)
2229	{
2230	struct active_request_slot *slot;
2231	struct slot_results results = { .retry_after = -1 };
2232	struct curl_slist *headers = http_copy_default_headers();
2233	struct strbuf buf = STRBUF_INIT;
2234	const char *accept_language;
2235	int ret;
2236
2237	slot = get_active_slot();
2238	curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1L);
2239
2240	if (!result) {
2241	curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1L);
2242	} else {
2243	curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0L);
2244	curl_easy_setopt(slot->curl, CURLOPT_WRITEDATA, result);
2245
2246	if (target == HTTP_REQUEST_FILE) {
2247	off_t posn = ftello(result);
2248	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
2249	fwrite);
2250	if (posn > 0)
2251	http_opt_request_remainder(slot->curl, posn);
2252	} else
2253	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
2254	fwrite_buffer);
2255	}
2256
2257	curl_easy_setopt(slot->curl, CURLOPT_HEADERFUNCTION, fwrite_wwwauth);
2258
2259	accept_language = http_get_accept_language_header();
2260
2261	if (accept_language)
2262	headers = curl_slist_append(headers, accept_language);
2263
2264	strbuf_addstr(&buf, "Pragma:");
2265	if (options->no_cache)
2266	strbuf_addstr(&buf, " no-cache");
2267	if (options->initial_request &&
2268	http_follow_config == HTTP_FOLLOW_INITIAL)
2269	curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 1L);
2270
2271	headers = curl_slist_append(headers, buf.buf);
2272
2273	/* Add additional headers here */
2274	if (options->extra_headers) {
2275	const struct string_list_item *item;
2276	for_each_string_list_item(item, options->extra_headers)
2277	headers = curl_slist_append(headers, item->string);
2278	}
2279
2280	headers = http_append_auth_header(&http_auth, headers);
2281
2282	curl_easy_setopt(slot->curl, CURLOPT_URL, url);
2283	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers);
2284	curl_easy_setopt(slot->curl, CURLOPT_ENCODING, "");
2285	curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 0L);
2286
2287	ret = run_one_slot(slot, &results);
2288
2289	#ifdef GIT_CURL_HAVE_CURLINFO_RETRY_AFTER
2290	if (ret == HTTP_RATE_LIMITED) {
2291	curl_off_t retry_after;
2292	if (curl_easy_getinfo(slot->curl, CURLINFO_RETRY_AFTER,
2293	&retry_after) == CURLE_OK && retry_after > 0)
2294	results.retry_after = (long)retry_after;
2295	}
2296	#endif
2297
2298	options->retry_after = results.retry_after;
2299
2300	if (options->content_type) {
2301	struct strbuf raw = STRBUF_INIT;
2302	curlinfo_strbuf(slot->curl, CURLINFO_CONTENT_TYPE, &raw);
2303	extract_content_type(&raw, options->content_type,
2304	options->charset);
2305	strbuf_release(&raw);
2306	}
2307
2308	if (options->effective_url)
2309	curlinfo_strbuf(slot->curl, CURLINFO_EFFECTIVE_URL,
2310	options->effective_url);
2311
2312	curl_slist_free_all(headers);
2313	strbuf_release(&buf);
2314
2315	return ret;
2316	}
2317
2318	/*
2319	* Update the "base" url to a more appropriate value, as deduced by
2320	* redirects seen when requesting a URL starting with "url".
2321	*
2322	* The "asked" parameter is a URL that we asked curl to access, and must begin
2323	* with "base".
2324	*
2325	* The "got" parameter is the URL that curl reported to us as where we ended
2326	* up.
2327	*
2328	* Returns 1 if we updated the base url, 0 otherwise.
2329	*
2330	* Our basic strategy is to compare "base" and "asked" to find the bits
2331	* specific to our request. We then strip those bits off of "got" to yield the
2332	* new base. So for example, if our base is "http://example.com/foo.git",
2333	* and we ask for "http://example.com/foo.git/info/refs", we might end up
2334	* with "https://other.example.com/foo.git/info/refs". We would want the
2335	* new URL to become "https://other.example.com/foo.git".
2336	*
2337	* Note that this assumes a sane redirect scheme. It's entirely possible
2338	* in the example above to end up at a URL that does not even end in
2339	* "info/refs". In such a case we die. There's not much we can do, such a
2340	* scheme is unlikely to represent a real git repository, and failing to
2341	* rewrite the base opens options for malicious redirects to do funny things.
2342	*/
2343	static int update_url_from_redirect(struct strbuf *base,
2344	const char *asked,
2345	const struct strbuf *got)
2346	{
2347	const char *tail;
2348	size_t new_len;
2349
2350	if (!strcmp(asked, got->buf))
2351	return 0;
2352
2353	if (!skip_prefix(asked, base->buf, &tail))
2354	BUG("update_url_from_redirect: %s is not a superset of %s",
2355	asked, base->buf);
2356
2357	new_len = got->len;
2358	if (!strip_suffix_mem(got->buf, &new_len, tail))
2359	die(_("unable to update url base from redirection:\n"
2360	" asked for: %s\n"
2361	" redirect: %s"),
2362	asked, got->buf);
2363
2364	strbuf_reset(base);
2365	strbuf_add(base, got->buf, new_len);
2366
2367	return 1;
2368	}
2369
2370	/*
2371	* Compute the retry delay for an HTTP 429 response.
2372	* Returns a negative value if configuration is invalid (delay exceeds
2373	* http.maxRetryTime), otherwise returns the delay in seconds (>= 0).
2374	*/
2375	static long handle_rate_limit_retry(long slot_retry_after)
2376	{
2377	/* Use the slot-specific retry_after value or configured default */
2378	if (slot_retry_after >= 0) {
2379	/* Check if retry delay exceeds maximum allowed */
2380	if (slot_retry_after > http_max_retry_time) {
2381	error(_("response requested a delay greater than http.maxRetryTime (%ld > %ld seconds)"),
2382	slot_retry_after, http_max_retry_time);
2383	trace2_data_string("http", the_repository,
2384	"http/429-error", "exceeds-max-retry-time");
2385	trace2_data_intmax("http", the_repository,
2386	"http/429-requested-delay", slot_retry_after);
2387	return -1;
2388	}
2389	return slot_retry_after;
2390	} else {
2391	/* No Retry-After header provided, use configured default */
2392	if (http_retry_after > http_max_retry_time) {
2393	error(_("configured http.retryAfter exceeds http.maxRetryTime (%ld > %ld seconds)"),
2394	http_retry_after, http_max_retry_time);
2395	trace2_data_string("http", the_repository,
2396	"http/429-error", "config-exceeds-max-retry-time");
2397	return -1;
2398	}
2399	trace2_data_string("http", the_repository,
2400	"http/429-retry-source", "config-default");
2401	return http_retry_after;
2402	}
2403	}
2404
2405	static int http_request_recoverable(const char *url,
2406	void *result, int target,
2407	struct http_get_options *options)
2408	{
2409	static struct http_get_options empty_opts;
2410	int i = 3;
2411	int ret;
2412	int rate_limit_retries = http_max_retries;
2413
2414	if (!options)
2415	options = &empty_opts;
2416
2417	if (always_auth_proactively())
2418	credential_fill(the_repository, &http_auth, 1);
2419
2420	ret = http_request(url, result, target, options);
2421
2422	if (ret != HTTP_OK && ret != HTTP_REAUTH && ret != HTTP_RATE_LIMITED)
2423	return ret;
2424
2425	/* If retries are disabled and we got a 429, fail immediately */
2426	if (ret == HTTP_RATE_LIMITED && !http_max_retries)
2427	return HTTP_ERROR;
2428
2429	if (options->effective_url && options->base_url) {
2430	if (update_url_from_redirect(options->base_url,
2431	url, options->effective_url)) {
2432	credential_from_url(&http_auth, options->base_url->buf);
2433	url = options->effective_url->buf;
2434	}
2435	}
2436
2437	while ((ret == HTTP_REAUTH && --i) \|\|
2438	(ret == HTTP_RATE_LIMITED && --rate_limit_retries)) {
2439	long retry_delay = -1;
2440	/*
2441	* The previous request may have put cruft into our output stream; we
2442	* should clear it out before making our next request.
2443	*/
2444	switch (target) {
2445	case HTTP_REQUEST_STRBUF:
2446	strbuf_reset(result);
2447	break;
2448	case HTTP_REQUEST_FILE: {
2449	FILE *f = result;
2450	if (fflush(f)) {
2451	error_errno("unable to flush a file");
2452	return HTTP_START_FAILED;
2453	}
2454	rewind(f);
2455	if (ftruncate(fileno(f), 0) < 0) {
2456	error_errno("unable to truncate a file");
2457	return HTTP_START_FAILED;
2458	}
2459	break;
2460	}
2461	default:
2462	BUG("Unknown http_request target");
2463	}
2464	if (ret == HTTP_RATE_LIMITED) {
2465	retry_delay = handle_rate_limit_retry(options->retry_after);
2466	if (retry_delay < 0)
2467	return HTTP_ERROR;
2468
2469	if (retry_delay > 0) {
2470	warning(_("rate limited, waiting %ld seconds before retry"), retry_delay);
2471	trace2_data_intmax("http", the_repository,
2472	"http/retry-sleep-seconds", retry_delay);
2473	sleep(retry_delay);
2474	}
2475	} else if (ret == HTTP_REAUTH) {
2476	http_reauth_prepare(1);
2477	}
2478
2479	ret = http_request(url, result, target, options);
2480	}
2481	if (ret == HTTP_RATE_LIMITED) {
2482	trace2_data_string("http", the_repository,
2483	"http/429-error", "retries-exhausted");
2484	return HTTP_RATE_LIMITED;
2485	}
2486	return ret;
2487	}
2488
2489	int http_get_strbuf(const char *url,
2490	struct strbuf *result,
2491	struct http_get_options *options)
2492	{
2493	return http_request_recoverable(url, result, HTTP_REQUEST_STRBUF, options);
2494	}
2495
2496	/*
2497	* Downloads a URL and stores the result in the given file.
2498	*
2499	* If a previous interrupted download is detected (i.e. a previous temporary
2500	* file is still around) the download is resumed.
2501	*/
2502	int http_get_file(const char url, const char filename,
2503	struct http_get_options *options)
2504	{
2505	int ret;
2506	struct strbuf tmpfile = STRBUF_INIT;
2507	FILE *result;
2508
2509	strbuf_addf(&tmpfile, "%s.temp", filename);
2510	result = fopen(tmpfile.buf, "a");
2511	if (!result) {
2512	error("Unable to open local file %s", tmpfile.buf);
2513	ret = HTTP_ERROR;
2514	goto cleanup;
2515	}
2516
2517	ret = http_request_recoverable(url, result, HTTP_REQUEST_FILE, options);
2518	fclose(result);
2519
2520	if (ret == HTTP_OK && finalize_object_file(the_repository, tmpfile.buf, filename))
2521	ret = HTTP_ERROR;
2522	cleanup:
2523	strbuf_release(&tmpfile);
2524	return ret;
2525	}
2526
2527	int http_fetch_ref(const char base, struct ref ref)
2528	{
2529	struct http_get_options options = {0};
2530	char *url;
2531	struct strbuf buffer = STRBUF_INIT;
2532	int ret = -1;
2533
2534	options.no_cache = 1;
2535
2536	url = quote_ref_url(base, ref->name);
2537	if (http_get_strbuf(url, &buffer, &options) == HTTP_OK) {
2538	strbuf_rtrim(&buffer);
2539	if (buffer.len == the_hash_algo->hexsz)
2540	ret = get_oid_hex(buffer.buf, &ref->old_oid);
2541	else if (starts_with(buffer.buf, "ref: ")) {
2542	ref->symref = xstrdup(buffer.buf + 5);
2543	ret = 0;
2544	}
2545	}
2546
2547	strbuf_release(&buffer);
2548	free(url);
2549	return ret;
2550	}
2551
2552	/* Helpers for fetching packs */
2553	static char fetch_pack_index(unsigned char hash, const char *base_url)
2554	{
2555	char url, tmp;
2556	struct strbuf buf = STRBUF_INIT;
2557
2558	if (http_is_verbose)
2559	fprintf(stderr, "Getting index for pack %s\n", hash_to_hex(hash));
2560
2561	end_url_with_slash(&buf, base_url);
2562	strbuf_addf(&buf, "objects/pack/pack-%s.idx", hash_to_hex(hash));
2563	url = strbuf_detach(&buf, NULL);
2564
2565	/*
2566	* Don't put this into packs/, since it's just temporary and we don't
2567	* want to confuse it with our local .idx files. We'll generate our
2568	* own index if we choose to download the matching packfile.
2569	*
2570	* It's tempting to use xmks_tempfile() here, but it's important that
2571	* the file not exist, otherwise http_get_file() complains. So we
2572	* create a filename that should be unique, and then just register it
2573	* as a tempfile so that it will get cleaned up on exit.
2574	*
2575	* In theory we could hold on to the tempfile and delete these as soon
2576	* as we download the matching pack, but it would take a bit of
2577	* refactoring. Leaving them until the process ends is probably OK.
2578	*/
2579	tmp = xstrfmt("%s/tmp_pack_%s.idx",
2580	repo_get_object_directory(the_repository),
2581	hash_to_hex(hash));
2582	register_tempfile(tmp);
2583
2584	if (http_get_file(url, tmp, NULL) != HTTP_OK) {
2585	error("Unable to get pack index %s", url);
2586	FREE_AND_NULL(tmp);
2587	}
2588
2589	free(url);
2590	return tmp;
2591	}
2592
2593	static int fetch_and_setup_pack_index(struct packfile_list *packs,
2594	unsigned char *sha1,
2595	const char *base_url)
2596	{
2597	struct packed_git new_pack, p;
2598	char *tmp_idx = NULL;
2599	int ret;
2600
2601	/*
2602	* If we already have the pack locally, no need to fetch its index or
2603	* even add it to list; we already have all of its objects.
2604	*/
2605	repo_for_each_pack(the_repository, p) {
2606	if (hasheq(p->hash, sha1, the_repository->hash_algo))
2607	return 0;
2608	}
2609
2610	tmp_idx = fetch_pack_index(sha1, base_url);
2611	if (!tmp_idx)
2612	return -1;
2613
2614	new_pack = parse_pack_index(the_repository, sha1, tmp_idx);
2615	if (!new_pack) {
2616	free(tmp_idx);
2617	return -1; /* parse_pack_index() already issued error message */
2618	}
2619
2620	ret = verify_pack_index(new_pack);
2621
2622	close_pack_index(new_pack);
2623	free(tmp_idx);
2624	if (ret) {
2625	free(new_pack);
2626	return -1;
2627	}
2628
2629	packfile_list_prepend(packs, new_pack);
2630	return 0;
2631	}
2632
2633	int http_get_info_packs(const char base_url, struct packfile_list packs)
2634	{
2635	struct http_get_options options = {0};
2636	int ret = 0;
2637	char *url;
2638	const char *data;
2639	struct strbuf buf = STRBUF_INIT;
2640	struct object_id oid;
2641
2642	end_url_with_slash(&buf, base_url);
2643	strbuf_addstr(&buf, "objects/info/packs");
2644	url = strbuf_detach(&buf, NULL);
2645
2646	options.no_cache = 1;
2647	ret = http_get_strbuf(url, &buf, &options);
2648	if (ret != HTTP_OK)
2649	goto cleanup;
2650
2651	data = buf.buf;
2652	while (*data) {
2653	if (skip_prefix(data, "P pack-", &data) &&
2654	!parse_oid_hex(data, &oid, &data) &&
2655	skip_prefix(data, ".pack", &data) &&
2656	(data == '\n' \|\| data == '\0')) {
2657	fetch_and_setup_pack_index(packs, oid.hash, base_url);
2658	} else {
2659	data = strchrnul(data, '\n');
2660	}
2661	if (*data)
2662	data++; /* skip past newline */
2663	}
2664
2665	cleanup:
2666	free(url);
2667	strbuf_release(&buf);
2668	return ret;
2669	}
2670
2671	void release_http_pack_request(struct http_pack_request *preq)
2672	{
2673	if (preq->packfile) {
2674	fclose(preq->packfile);
2675	preq->packfile = NULL;
2676	}
2677	preq->slot = NULL;
2678	strbuf_release(&preq->tmpfile);
2679	curl_slist_free_all(preq->headers);
2680	free(preq->url);
2681	free(preq);
2682	}
2683
2684	static const char *default_index_pack_args[] =
2685	{"index-pack", "--stdin", NULL};
2686
2687	int finish_http_pack_request(struct http_pack_request *preq)
2688	{
2689	struct child_process ip = CHILD_PROCESS_INIT;
2690	int tmpfile_fd;
2691	int ret = 0;
2692
2693	/* Another downloader may unlink the staging path while we index it. */
2694	tmpfile_fd = xdup(fileno(preq->packfile));
2695	fclose(preq->packfile);
2696	preq->packfile = NULL;
2697	if (lseek(tmpfile_fd, 0, SEEK_SET) < 0)
2698	die_errno("unable to seek local file %s for pack",
2699	preq->tmpfile.buf);
2700
2701	ip.git_cmd = 1;
2702	ip.in = tmpfile_fd;
2703	strvec_pushv(&ip.args, preq->index_pack_args ?
2704	preq->index_pack_args :
2705	default_index_pack_args);
2706
2707	if (preq->preserve_index_pack_stdout)
2708	ip.out = 0;
2709	else
2710	ip.no_stdout = 1;
2711
2712	if (run_command(&ip))
2713	ret = -1;
2714	unlink(preq->tmpfile.buf);
2715	return ret;
2716	}
2717
2718	void http_install_packfile(struct packed_git *p,
2719	struct packfile_list *list_to_remove_from)
2720	{
2721	struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources);
2722	packfile_list_remove(list_to_remove_from, p);
2723	packfile_store_add_pack(files->packed, p);
2724	}
2725
2726	struct http_pack_request *new_http_pack_request(
2727	const unsigned char packed_git_hash, const char base_url) {
2728
2729	struct strbuf buf = STRBUF_INIT;
2730
2731	end_url_with_slash(&buf, base_url);
2732	strbuf_addf(&buf, "objects/pack/pack-%s.pack",
2733	hash_to_hex(packed_git_hash));
2734	return new_direct_http_pack_request(packed_git_hash,
2735	strbuf_detach(&buf, NULL));
2736	}
2737
2738	struct http_pack_request *new_direct_http_pack_request(
2739	const unsigned char packed_git_hash, char url)
2740	{
2741	off_t prev_posn;
2742	struct http_pack_request *preq;
2743	int fd;
2744
2745	CALLOC_ARRAY(preq, 1);
2746	strbuf_init(&preq->tmpfile, 0);
2747	preq->url = url;
2748
2749	odb_pack_name(the_repository, &preq->tmpfile, packed_git_hash, "pack");
2750	strbuf_addstr(&preq->tmpfile, ".temp");
2751	/*
2752	* MinGW's non-append O_RDWR open grants FILE_SHARE_DELETE only for an
2753	* existing file; reopen a newly created file so others may unlink it.
2754	*/
2755	for (;;) {
2756	fd = open(preq->tmpfile.buf, O_RDWR);
2757	if (fd >= 0 \|\| errno != ENOENT)
2758	break;
2759	fd = open(preq->tmpfile.buf, O_RDWR \| O_CREAT \| O_EXCL, 0666);
2760	if (fd >= 0) {
2761	close(fd);
2762	continue;
2763	}
2764	if (errno != EEXIST)
2765	break;
2766	}
2767	if (fd < 0) {
2768	error_errno("unable to open local file %s for pack",
2769	preq->tmpfile.buf);
2770	goto abort;
2771	}
2772	prev_posn = lseek(fd, 0, SEEK_END);
2773	if (prev_posn < 0) {
2774	error_errno("unable to seek local file %s for pack",
2775	preq->tmpfile.buf);
2776	close(fd);
2777	goto abort;
2778	}
2779	preq->packfile = xfdopen(fd, "w");
2780
2781	preq->slot = get_active_slot();
2782	preq->headers = object_request_headers();
2783	curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEDATA, preq->packfile);
2784	curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
2785	curl_easy_setopt(preq->slot->curl, CURLOPT_URL, preq->url);
2786	curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER, preq->headers);
2787
2788	if (prev_posn > 0) {
2789	if (http_is_verbose)
2790	fprintf(stderr,
2791	"Resuming fetch of pack %s at byte %"PRIuMAX"\n",
2792	hash_to_hex(packed_git_hash),
2793	(uintmax_t)prev_posn);
2794	http_opt_request_remainder(preq->slot->curl, prev_posn);
2795	}
2796
2797	return preq;
2798
2799	abort:
2800	strbuf_release(&preq->tmpfile);
2801	free(preq->url);
2802	free(preq);
2803	return NULL;
2804	}
2805
2806	/* Helpers for fetching objects (loose) */
2807	static size_t fwrite_sha1_file(char *ptr, size_t eltsize, size_t nmemb,
2808	void *data)
2809	{
2810	unsigned char expn[4096];
2811	size_t size = eltsize * nmemb;
2812	int posn = 0;
2813	struct http_object_request *freq = data;
2814	struct active_request_slot *slot = freq->slot;
2815
2816	if (slot) {
2817	CURLcode c = curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE,
2818	&slot->http_code);
2819	if (c != CURLE_OK)
2820	BUG("curl_easy_getinfo for HTTP code failed: %s",
2821	curl_easy_strerror(c));
2822	if (slot->http_code >= 300)
2823	return nmemb;
2824	}
2825
2826	do {
2827	ssize_t retval = xwrite(freq->localfile,
2828	(char *) ptr + posn, size - posn);
2829	if (retval < 0)
2830	return posn / eltsize;
2831	posn += retval;
2832	} while (posn < size);
2833
2834	freq->stream.avail_in = size;
2835	freq->stream.next_in = (void *)ptr;
2836	do {
2837	freq->stream.next_out = expn;
2838	freq->stream.avail_out = sizeof(expn);
2839	freq->zret = git_inflate(&freq->stream, Z_SYNC_FLUSH);
2840	git_hash_update(&freq->c, expn,
2841	sizeof(expn) - freq->stream.avail_out);
2842	} while (freq->stream.avail_in && freq->zret == Z_OK);
2843	return nmemb;
2844	}
2845
2846	struct http_object_request new_http_object_request(const char base_url,
2847	const struct object_id *oid)
2848	{
2849	struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources);
2850	char *hex = oid_to_hex(oid);
2851	struct strbuf filename = STRBUF_INIT;
2852	struct strbuf prevfile = STRBUF_INIT;
2853	int prevlocal;
2854	char prev_buf[PREV_BUF_SIZE];
2855	ssize_t prev_read = 0;
2856	off_t prev_posn = 0;
2857	struct http_object_request *freq;
2858
2859	CALLOC_ARRAY(freq, 1);
2860	strbuf_init(&freq->tmpfile, 0);
2861	oidcpy(&freq->oid, oid);
2862	freq->localfile = -1;
2863
2864	odb_loose_path(files->loose, &filename, oid);
2865	strbuf_addf(&freq->tmpfile, "%s.temp", filename.buf);
2866
2867	strbuf_addf(&prevfile, "%s.prev", filename.buf);
2868	unlink_or_warn(prevfile.buf);
2869	rename(freq->tmpfile.buf, prevfile.buf);
2870	unlink_or_warn(freq->tmpfile.buf);
2871	strbuf_release(&filename);
2872
2873	if (freq->localfile != -1)
2874	error("fd leakage in start: %d", freq->localfile);
2875	freq->localfile = open(freq->tmpfile.buf,
2876	O_WRONLY \| O_CREAT \| O_EXCL, 0666);
2877	/*
2878	* This could have failed due to the "lazy directory creation";
2879	* try to mkdir the last path component.
2880	*/
2881	if (freq->localfile < 0 && errno == ENOENT) {
2882	char *dir = strrchr(freq->tmpfile.buf, '/');
2883	if (dir) {
2884	*dir = 0;
2885	mkdir(freq->tmpfile.buf, 0777);
2886	*dir = '/';
2887	}
2888	freq->localfile = open(freq->tmpfile.buf,
2889	O_WRONLY \| O_CREAT \| O_EXCL, 0666);
2890	}
2891
2892	if (freq->localfile < 0) {
2893	error_errno("Couldn't create temporary file %s",
2894	freq->tmpfile.buf);
2895	goto abort;
2896	}
2897
2898	git_inflate_init(&freq->stream);
2899
2900	git_hash_init(&freq->c, the_hash_algo);
2901
2902	freq->url = get_remote_object_url(base_url, hex, 0);
2903
2904	/*
2905	* If a previous temp file is present, process what was already
2906	* fetched.
2907	*/
2908	prevlocal = open(prevfile.buf, O_RDONLY);
2909	if (prevlocal != -1) {
2910	do {
2911	prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
2912	if (prev_read>0) {
2913	if (fwrite_sha1_file(prev_buf,
2914	1,
2915	prev_read,
2916	freq) == prev_read) {
2917	prev_posn += prev_read;
2918	} else {
2919	prev_read = -1;
2920	}
2921	}
2922	} while (prev_read > 0);
2923	close(prevlocal);
2924	}
2925	unlink_or_warn(prevfile.buf);
2926	strbuf_release(&prevfile);
2927
2928	/*
2929	* Reset inflate/SHA1 if there was an error reading the previous temp
2930	* file; also rewind to the beginning of the local file.
2931	*/
2932	if (prev_read == -1) {
2933	git_inflate_end(&freq->stream);
2934	memset(&freq->stream, 0, sizeof(freq->stream));
2935	git_inflate_init(&freq->stream);
2936	git_hash_init(&freq->c, the_hash_algo);
2937	if (prev_posn>0) {
2938	prev_posn = 0;
2939	lseek(freq->localfile, 0, SEEK_SET);
2940	if (ftruncate(freq->localfile, 0) < 0) {
2941	error_errno("Couldn't truncate temporary file %s",
2942	freq->tmpfile.buf);
2943	goto abort;
2944	}
2945	}
2946	}
2947
2948	freq->slot = get_active_slot();
2949	freq->headers = object_request_headers();
2950
2951	curl_easy_setopt(freq->slot->curl, CURLOPT_WRITEDATA, freq);
2952	curl_easy_setopt(freq->slot->curl, CURLOPT_FAILONERROR, 0L);
2953	curl_easy_setopt(freq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
2954	curl_easy_setopt(freq->slot->curl, CURLOPT_ERRORBUFFER, freq->errorstr);
2955	curl_easy_setopt(freq->slot->curl, CURLOPT_URL, freq->url);
2956	curl_easy_setopt(freq->slot->curl, CURLOPT_HTTPHEADER, freq->headers);
2957
2958	/*
2959	* If we have successfully processed data from a previous fetch
2960	* attempt, only fetch the data we don't already have.
2961	*/
2962	if (prev_posn>0) {
2963	if (http_is_verbose)
2964	fprintf(stderr,
2965	"Resuming fetch of object %s at byte %"PRIuMAX"\n",
2966	hex, (uintmax_t)prev_posn);
2967	http_opt_request_remainder(freq->slot->curl, prev_posn);
2968	}
2969
2970	return freq;
2971
2972	abort:
2973	strbuf_release(&prevfile);
2974	free(freq->url);
2975	free(freq);
2976	return NULL;
2977	}
2978
2979	void process_http_object_request(struct http_object_request *freq)
2980	{
2981	if (!freq->slot)
2982	return;
2983	freq->curl_result = freq->slot->curl_result;
2984	freq->http_code = freq->slot->http_code;
2985	freq->slot = NULL;
2986	}
2987
2988	int finish_http_object_request(struct http_object_request *freq)
2989	{
2990	struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources);
2991	struct stat st;
2992	struct strbuf filename = STRBUF_INIT;
2993
2994	close(freq->localfile);
2995	freq->localfile = -1;
2996
2997	process_http_object_request(freq);
2998
2999	if (freq->http_code == 416) {
3000	warning("requested range invalid; we may already have all the data.");
3001	} else if (freq->curl_result != CURLE_OK) {
3002	if (stat(freq->tmpfile.buf, &st) == 0)
3003	if (st.st_size == 0)
3004	unlink_or_warn(freq->tmpfile.buf);
3005	return -1;
3006	}
3007
3008	git_hash_final_oid(&freq->real_oid, &freq->c);
3009	if (freq->zret != Z_STREAM_END) {
3010	unlink_or_warn(freq->tmpfile.buf);
3011	return -1;
3012	}
3013	if (!oideq(&freq->oid, &freq->real_oid)) {
3014	unlink_or_warn(freq->tmpfile.buf);
3015	return -1;
3016	}
3017	odb_loose_path(files->loose, &filename, &freq->oid);
3018	freq->rename = finalize_object_file(the_repository, freq->tmpfile.buf, filename.buf);
3019	strbuf_release(&filename);
3020
3021	return freq->rename;
3022	}
3023
3024	void abort_http_object_request(struct http_object_request **freq_p)
3025	{
3026	struct http_object_request freq = freq_p;
3027	unlink_or_warn(freq->tmpfile.buf);
3028
3029	release_http_object_request(freq_p);
3030	}
3031
3032	void release_http_object_request(struct http_object_request **freq_p)
3033	{
3034	struct http_object_request freq = freq_p;
3035	if (freq->localfile != -1) {
3036	close(freq->localfile);
3037	freq->localfile = -1;
3038	}
3039	FREE_AND_NULL(freq->url);
3040	if (freq->slot) {
3041	freq->slot->callback_func = NULL;
3042	freq->slot->callback_data = NULL;
3043	release_active_slot(freq->slot);
3044	freq->slot = NULL;
3045	}
3046	curl_slist_free_all(freq->headers);
3047	strbuf_release(&freq->tmpfile);
3048	git_inflate_end(&freq->stream);
3049	git_hash_discard(&freq->c);
3050
3051	free(freq);
3052	*freq_p = NULL;
3053	}