http.c · matheusbsbs/matheus-bagliano

bisect jch maint master next seen test todo

c 3,035 lines 81.3 KB

Raw

1	#define USE_THE_REPOSITORY_VARIABLE
2	#define DISABLE_SIGN_COMPARE_WARNINGS
3
4	#include "git-compat-util.h"
5	#include "git-curl-compat.h"
6	#include "environment.h"
7	#include "hex.h"
8	#include "http.h"
9	#include "config.h"
10	#include "pack.h"
11	#include "run-command.h"
12	#include "url.h"
13	#include "urlmatch.h"
14	#include "credential.h"
15	#include "version.h"
16	#include "pkt-line.h"
17	#include "gettext.h"
18	#include "trace.h"
19	#include "transport.h"
20	#include "packfile.h"
21	#include "string-list.h"
22	#include "object-file.h"
23	#include "odb.h"
24	#include "tempfile.h"
25	#include "date.h"
26	#include "trace2.h"
27
28	static struct trace_key trace_curl = TRACE_KEY_INIT(CURL);
29	static int trace_curl_data = 1;
30	static int trace_curl_redact = 1;
31	long int git_curl_ipresolve = CURL_IPRESOLVE_WHATEVER;
32	int active_requests;
33	int http_is_verbose;
34	ssize_t http_post_buffer = 16 * LARGE_PACKET_MAX;
35
36	static int min_curl_sessions = 1;
37	static int curl_session_count;
38	static int max_requests = -1;
39	static CURLM *curlm;
40	static CURL *curl_default;
41
42	#define PREV_BUF_SIZE 4096
43
44	char curl_errorstr[CURL_ERROR_SIZE];
45
46	static int curl_ssl_verify = -1;
47	static int curl_ssl_try;
48	static char *curl_http_version;
49	static char *ssl_cert;
50	static char *ssl_cert_type;
51	static char *ssl_cipherlist;
52	static char *ssl_version;
53	static struct {
54	const char *name;
55	long ssl_version;
56	} sslversions[] = {
57	{ "sslv2", CURL_SSLVERSION_SSLv2 },
58	{ "sslv3", CURL_SSLVERSION_SSLv3 },
59	{ "tlsv1", CURL_SSLVERSION_TLSv1 },
60	{ "tlsv1.0", CURL_SSLVERSION_TLSv1_0 },
61	{ "tlsv1.1", CURL_SSLVERSION_TLSv1_1 },
62	{ "tlsv1.2", CURL_SSLVERSION_TLSv1_2 },
63	{ "tlsv1.3", CURL_SSLVERSION_TLSv1_3 },
64	};
65	static char *ssl_key;
66	static char *ssl_key_type;
67	static char *ssl_capath;
68	static char *curl_no_proxy;
69	static char *ssl_pinnedkey;
70	static char *ssl_cainfo;
71	static long curl_low_speed_limit = -1;
72	static long curl_low_speed_time = -1;
73	static int curl_ftp_no_epsv;
74	static char *curl_http_proxy;
75	static char *http_proxy_authmethod;
76
77	static char *http_proxy_ssl_cert;
78	static char *http_proxy_ssl_key;
79	static char *http_proxy_ssl_ca_info;
80	static struct credential proxy_cert_auth = CREDENTIAL_INIT;
81	static int proxy_ssl_cert_password_required;
82
83	static struct {
84	const char *name;
85	long curlauth_param;
86	} proxy_authmethods[] = {
87	{ "basic", CURLAUTH_BASIC },
88	{ "digest", CURLAUTH_DIGEST },
89	{ "negotiate", CURLAUTH_GSSNEGOTIATE },
90	{ "ntlm", CURLAUTH_NTLM },
91	{ "anyauth", CURLAUTH_ANY },
92	/*
93	* CURLAUTH_DIGEST_IE has no corresponding command-line option in
94	* curl(1) and is not included in CURLAUTH_ANY, so we leave it out
95	* here, too
96	*/
97	};
98	#ifdef CURLGSSAPI_DELEGATION_FLAG
99	static char *curl_deleg;
100	static struct {
101	const char *name;
102	long curl_deleg_param;
103	} curl_deleg_levels[] = {
104	{ "none", CURLGSSAPI_DELEGATION_NONE },
105	{ "policy", CURLGSSAPI_DELEGATION_POLICY_FLAG },
106	{ "always", CURLGSSAPI_DELEGATION_FLAG },
107	};
108	#endif
109
110	static long curl_tcp_keepidle = -1;
111	static long curl_tcp_keepintvl = -1;
112	static long curl_tcp_keepcnt = -1;
113
114	enum proactive_auth {
115	PROACTIVE_AUTH_NONE = 0,
116	PROACTIVE_AUTH_IF_CREDENTIALS,
117	PROACTIVE_AUTH_AUTO,
118	PROACTIVE_AUTH_BASIC,
119	};
120
121	static struct credential proxy_auth = CREDENTIAL_INIT;
122	static const char *curl_proxyuserpwd;
123	static char *curl_cookie_file;
124	static int curl_save_cookies;
125	struct credential http_auth = CREDENTIAL_INIT;
126	static enum proactive_auth http_proactive_auth;
127	static char *user_agent;
128	static int curl_empty_auth = -1;
129
130	enum http_follow_config http_follow_config = HTTP_FOLLOW_INITIAL;
131
132	static struct credential cert_auth = CREDENTIAL_INIT;
133	static int ssl_cert_password_required;
134	static unsigned long http_auth_methods = CURLAUTH_ANY;
135	static int http_auth_methods_restricted;
136	/* Modes for which empty_auth cannot actually help us. */
137	static unsigned long empty_auth_useless =
138	CURLAUTH_BASIC
139	\| CURLAUTH_DIGEST_IE
140	\| CURLAUTH_DIGEST;
141	static int empty_auth_try_negotiate;
142
143	static struct curl_slist *pragma_header;
144	static struct string_list extra_http_headers = STRING_LIST_INIT_DUP;
145
146	static struct curl_slist *host_resolutions;
147
148	static struct active_request_slot *active_queue_head;
149
150	static char *cached_accept_language;
151
152	static char *http_ssl_backend;
153
154	static int http_schannel_check_revoke = 1;
155
156	static long http_retry_after = 0;
157	static long http_max_retries = 0;
158	static long http_max_retry_time = 300;
159
160	/*
161	* With the backend being set to `schannel`, setting sslCAinfo would override
162	* the Certificate Store in cURL v7.60.0 and later, which is not what we want
163	* by default.
164	*/
165	static int http_schannel_use_ssl_cainfo;
166
167	static int always_auth_proactively(void)
168	{
169	return http_proactive_auth != PROACTIVE_AUTH_NONE &&
170	http_proactive_auth != PROACTIVE_AUTH_IF_CREDENTIALS;
171	}
172
173	size_t fread_buffer(char ptr, size_t eltsize, size_t nmemb, void buffer_)
174	{
175	size_t size = eltsize * nmemb;
176	struct buffer *buffer = buffer_;
177
178	if (size > buffer->buf.len - buffer->posn)
179	size = buffer->buf.len - buffer->posn;
180	memcpy(ptr, buffer->buf.buf + buffer->posn, size);
181	buffer->posn += size;
182
183	return size / eltsize;
184	}
185
186	int seek_buffer(void *clientp, curl_off_t offset, int origin)
187	{
188	struct buffer *buffer = clientp;
189
190	if (origin != SEEK_SET)
191	BUG("seek_buffer only handles SEEK_SET");
192	if (offset < 0 \|\| offset >= buffer->buf.len) {
193	error("curl seek would be outside of buffer");
194	return CURL_SEEKFUNC_FAIL;
195	}
196
197	buffer->posn = offset;
198	return CURL_SEEKFUNC_OK;
199	}
200
201	size_t fwrite_buffer(char ptr, size_t eltsize, size_t nmemb, void buffer_)
202	{
203	size_t size = eltsize * nmemb;
204	struct strbuf *buffer = buffer_;
205
206	strbuf_add(buffer, ptr, size);
207	return nmemb;
208	}
209
210	/*
211	* A folded header continuation line starts with any number of spaces or
212	* horizontal tab characters (SP or HTAB) as per RFC 7230 section 3.2.
213	* It is not a continuation line if the line starts with any other character.
214	*/
215	static inline int is_hdr_continuation(const char *ptr, const size_t size)
216	{
217	return size && (ptr == ' ' \|\| ptr == '\t');
218	}
219
220	static size_t fwrite_wwwauth(char ptr, size_t eltsize, size_t nmemb, void p MAYBE_UNUSED)
221	{
222	size_t size = eltsize * nmemb;
223	struct strvec *values = &http_auth.wwwauth_headers;
224	struct strbuf buf = STRBUF_INIT;
225	const char *val;
226	size_t val_len;
227
228	/*
229	* Header lines may not come NULL-terminated from libcurl so we must
230	* limit all scans to the maximum length of the header line, or leverage
231	* strbufs for all operations.
232	*
233	* In addition, it is possible that header values can be split over
234	* multiple lines as per RFC 7230. 'Line folding' has been deprecated
235	* but older servers may still emit them. A continuation header field
236	* value is identified as starting with a space or horizontal tab.
237	*
238	* The formal definition of a header field as given in RFC 7230 is:
239	*
240	* header-field = field-name ":" OWS field-value OWS
241	*
242	* field-name = token
243	* field-value = *( field-content / obs-fold )
244	* field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ]
245	* field-vchar = VCHAR / obs-text
246	*
247	* obs-fold = CRLF 1*( SP / HTAB )
248	* ; obsolete line folding
249	* ; see Section 3.2.4
250	*/
251
252	/* Start of a new WWW-Authenticate header */
253	if (skip_iprefix_mem(ptr, size, "www-authenticate:", &val, &val_len)) {
254	strbuf_add(&buf, val, val_len);
255
256	/*
257	* Strip the CRLF that should be present at the end of each
258	* field as well as any trailing or leading whitespace from the
259	* value.
260	*/
261	strbuf_trim(&buf);
262
263	strvec_push(values, buf.buf);
264	http_auth.header_is_last_match = 1;
265	goto exit;
266	}
267
268	/*
269	* This line could be a continuation of the previously matched header
270	* field. If this is the case then we should append this value to the
271	* end of the previously consumed value.
272	*/
273	if (http_auth.header_is_last_match && is_hdr_continuation(ptr, size)) {
274	/*
275	* Trim the CRLF and any leading or trailing from this line.
276	*/
277	strbuf_add(&buf, ptr, size);
278	strbuf_trim(&buf);
279
280	/*
281	* At this point we should always have at least one existing
282	* value, even if it is empty. Do not bother appending the new
283	* value if this continuation header is itself empty.
284	*/
285	if (!values->nr) {
286	BUG("should have at least one existing header value");
287	} else if (buf.len) {
288	char *prev = xstrdup(values->v[values->nr - 1]);
289
290	/* Join two non-empty values with a single space. */
291	const char const sp = prev ? " " : "";
292
293	strvec_pop(values);
294	strvec_pushf(values, "%s%s%s", prev, sp, buf.buf);
295	free(prev);
296	}
297
298	goto exit;
299	}
300
301	/* Not a continuation of a previously matched auth header line. */
302	http_auth.header_is_last_match = 0;
303
304	/*
305	* If this is a HTTP status line and not a header field, this signals
306	* a different HTTP response. libcurl writes all the output of all
307	* response headers of all responses, including redirects.
308	* We only care about the last HTTP request response's headers so clear
309	* the existing array.
310	*/
311	if (skip_iprefix_mem(ptr, size, "http/", &val, &val_len))
312	strvec_clear(values);
313
314	exit:
315	strbuf_release(&buf);
316	return size;
317	}
318
319	size_t fwrite_null(char *ptr UNUSED, size_t eltsize UNUSED, size_t nmemb,
320	void *data UNUSED)
321	{
322	return nmemb;
323	}
324
325	static struct curl_slist *object_request_headers(void)
326	{
327	return curl_slist_append(http_copy_default_headers(), "Pragma:");
328	}
329
330	static void closedown_active_slot(struct active_request_slot *slot)
331	{
332	active_requests--;
333	slot->in_use = 0;
334	}
335
336	static void finish_active_slot(struct active_request_slot *slot)
337	{
338	closedown_active_slot(slot);
339	curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code);
340
341	if (slot->finished)
342	(*slot->finished) = 1;
343
344	/* Store slot results so they can be read after the slot is reused */
345	if (slot->results) {
346	slot->results->curl_result = slot->curl_result;
347	slot->results->http_code = slot->http_code;
348	curl_easy_getinfo(slot->curl, CURLINFO_HTTPAUTH_AVAIL,
349	&slot->results->auth_avail);
350
351	curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CONNECTCODE,
352	&slot->results->http_connectcode);
353	}
354
355	/* Run callback if appropriate */
356	if (slot->callback_func)
357	slot->callback_func(slot->callback_data);
358	}
359
360	static void xmulti_remove_handle(struct active_request_slot *slot)
361	{
362	curl_multi_remove_handle(curlm, slot->curl);
363	}
364
365	static void process_curl_messages(void)
366	{
367	int num_messages;
368	struct active_request_slot *slot;
369	CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
370
371	while (curl_message != NULL) {
372	if (curl_message->msg == CURLMSG_DONE) {
373	int curl_result = curl_message->data.result;
374	slot = active_queue_head;
375	while (slot != NULL &&
376	slot->curl != curl_message->easy_handle)
377	slot = slot->next;
378	if (slot) {
379	xmulti_remove_handle(slot);
380	slot->curl_result = curl_result;
381	finish_active_slot(slot);
382	} else {
383	fprintf(stderr, "Received DONE message for unknown request!\n");
384	}
385	} else {
386	fprintf(stderr, "Unknown CURL message received: %d\n",
387	(int)curl_message->msg);
388	}
389	curl_message = curl_multi_info_read(curlm, &num_messages);
390	}
391	}
392
393	static int http_options(const char var, const char value,
394	const struct config_context ctx, void data)
395	{
396	if (!strcmp("http.version", var)) {
397	return git_config_string(&curl_http_version, var, value);
398	}
399	if (!strcmp("http.sslverify", var)) {
400	curl_ssl_verify = git_config_bool(var, value);
401	return 0;
402	}
403	if (!strcmp("http.sslcipherlist", var))
404	return git_config_string(&ssl_cipherlist, var, value);
405	if (!strcmp("http.sslversion", var))
406	return git_config_string(&ssl_version, var, value);
407	if (!strcmp("http.sslcert", var))
408	return git_config_pathname(&ssl_cert, var, value);
409	if (!strcmp("http.sslcerttype", var))
410	return git_config_string(&ssl_cert_type, var, value);
411	if (!strcmp("http.sslkey", var))
412	return git_config_pathname(&ssl_key, var, value);
413	if (!strcmp("http.sslkeytype", var))
414	return git_config_string(&ssl_key_type, var, value);
415	if (!strcmp("http.sslcapath", var))
416	return git_config_pathname(&ssl_capath, var, value);
417	if (!strcmp("http.sslcainfo", var))
418	return git_config_pathname(&ssl_cainfo, var, value);
419	if (!strcmp("http.sslcertpasswordprotected", var)) {
420	ssl_cert_password_required = git_config_bool(var, value);
421	return 0;
422	}
423	if (!strcmp("http.ssltry", var)) {
424	curl_ssl_try = git_config_bool(var, value);
425	return 0;
426	}
427	if (!strcmp("http.sslbackend", var)) {
428	free(http_ssl_backend);
429	http_ssl_backend = xstrdup_or_null(value);
430	return 0;
431	}
432
433	if (!strcmp("http.schannelcheckrevoke", var)) {
434	http_schannel_check_revoke = git_config_bool(var, value);
435	return 0;
436	}
437
438	if (!strcmp("http.schannelusesslcainfo", var)) {
439	http_schannel_use_ssl_cainfo = git_config_bool(var, value);
440	return 0;
441	}
442
443	if (!strcmp("http.minsessions", var)) {
444	min_curl_sessions = git_config_int(var, value, ctx->kvi);
445	if (min_curl_sessions > 1)
446	min_curl_sessions = 1;
447	return 0;
448	}
449	if (!strcmp("http.maxrequests", var)) {
450	max_requests = git_config_int(var, value, ctx->kvi);
451	return 0;
452	}
453	if (!strcmp("http.lowspeedlimit", var)) {
454	curl_low_speed_limit = git_config_int(var, value, ctx->kvi);
455	return 0;
456	}
457	if (!strcmp("http.lowspeedtime", var)) {
458	curl_low_speed_time = git_config_int(var, value, ctx->kvi);
459	return 0;
460	}
461
462	if (!strcmp("http.noepsv", var)) {
463	curl_ftp_no_epsv = git_config_bool(var, value);
464	return 0;
465	}
466	if (!strcmp("http.proxy", var))
467	return git_config_string(&curl_http_proxy, var, value);
468
469	if (!strcmp("http.proxyauthmethod", var))
470	return git_config_string(&http_proxy_authmethod, var, value);
471
472	if (!strcmp("http.proxysslcert", var))
473	return git_config_string(&http_proxy_ssl_cert, var, value);
474
475	if (!strcmp("http.proxysslkey", var))
476	return git_config_string(&http_proxy_ssl_key, var, value);
477
478	if (!strcmp("http.proxysslcainfo", var))
479	return git_config_string(&http_proxy_ssl_ca_info, var, value);
480
481	if (!strcmp("http.proxysslcertpasswordprotected", var)) {
482	proxy_ssl_cert_password_required = git_config_bool(var, value);
483	return 0;
484	}
485
486	if (!strcmp("http.cookiefile", var))
487	return git_config_pathname(&curl_cookie_file, var, value);
488	if (!strcmp("http.savecookies", var)) {
489	curl_save_cookies = git_config_bool(var, value);
490	return 0;
491	}
492
493	if (!strcmp("http.postbuffer", var)) {
494	http_post_buffer = git_config_ssize_t(var, value, ctx->kvi);
495	if (http_post_buffer < 0)
496	warning(_("negative value for http.postBuffer; defaulting to %d"), LARGE_PACKET_MAX);
497	if (http_post_buffer < LARGE_PACKET_MAX)
498	http_post_buffer = LARGE_PACKET_MAX;
499	return 0;
500	}
501
502	if (!strcmp("http.useragent", var))
503	return git_config_string(&user_agent, var, value);
504
505	if (!strcmp("http.emptyauth", var)) {
506	if (value && !strcmp("auto", value))
507	curl_empty_auth = -1;
508	else
509	curl_empty_auth = git_config_bool(var, value);
510	return 0;
511	}
512
513	if (!strcmp("http.delegation", var)) {
514	#ifdef CURLGSSAPI_DELEGATION_FLAG
515	return git_config_string(&curl_deleg, var, value);
516	#else
517	warning(_("Delegation control is not supported with cURL < 7.22.0"));
518	return 0;
519	#endif
520	}
521
522	if (!strcmp("http.pinnedpubkey", var)) {
523	return git_config_pathname(&ssl_pinnedkey, var, value);
524	}
525
526	if (!strcmp("http.extraheader", var)) {
527	if (!value) {
528	return config_error_nonbool(var);
529	} else if (!*value) {
530	string_list_clear(&extra_http_headers, 0);
531	} else {
532	string_list_append(&extra_http_headers, value);
533	}
534	return 0;
535	}
536
537	if (!strcmp("http.curloptresolve", var)) {
538	if (!value) {
539	return config_error_nonbool(var);
540	} else if (!*value) {
541	curl_slist_free_all(host_resolutions);
542	host_resolutions = NULL;
543	} else {
544	host_resolutions = curl_slist_append(host_resolutions, value);
545	}
546	return 0;
547	}
548
549	if (!strcmp("http.followredirects", var)) {
550	if (value && !strcmp(value, "initial"))
551	http_follow_config = HTTP_FOLLOW_INITIAL;
552	else if (git_config_bool(var, value))
553	http_follow_config = HTTP_FOLLOW_ALWAYS;
554	else
555	http_follow_config = HTTP_FOLLOW_NONE;
556	return 0;
557	}
558
559	if (!strcmp("http.proactiveauth", var)) {
560	if (!value)
561	return config_error_nonbool(var);
562	if (!strcmp(value, "auto"))
563	http_proactive_auth = PROACTIVE_AUTH_AUTO;
564	else if (!strcmp(value, "basic"))
565	http_proactive_auth = PROACTIVE_AUTH_BASIC;
566	else if (!strcmp(value, "none"))
567	http_proactive_auth = PROACTIVE_AUTH_NONE;
568	else
569	warning(_("Unknown value for http.proactiveauth"));
570	return 0;
571	}
572
573	if (!strcmp("http.keepaliveidle", var)) {
574	curl_tcp_keepidle = git_config_int(var, value, ctx->kvi);
575	return 0;
576	}
577	if (!strcmp("http.keepaliveinterval", var)) {
578	curl_tcp_keepintvl = git_config_int(var, value, ctx->kvi);
579	return 0;
580	}
581	if (!strcmp("http.keepalivecount", var)) {
582	curl_tcp_keepcnt = git_config_int(var, value, ctx->kvi);
583	return 0;
584	}
585
586	if (!strcmp("http.retryafter", var)) {
587	http_retry_after = git_config_int(var, value, ctx->kvi);
588	return 0;
589	}
590
591	if (!strcmp("http.maxretries", var)) {
592	http_max_retries = git_config_int(var, value, ctx->kvi);
593	return 0;
594	}
595
596	if (!strcmp("http.maxretrytime", var)) {
597	http_max_retry_time = git_config_int(var, value, ctx->kvi);
598	return 0;
599	}
600
601	/* Fall back on the default ones */
602	return git_default_config(var, value, ctx, data);
603	}
604
605	static int curl_empty_auth_enabled(void)
606	{
607	if (curl_empty_auth >= 0)
608	return curl_empty_auth;
609
610	/*
611	* In the automatic case, kick in the empty-auth
612	* hack as long as we would potentially try some
613	* method more exotic than "Basic" or "Digest".
614	*
615	* But only do this when this is our second or
616	* subsequent request, as by then we know what
617	* methods are available.
618	*/
619	if (http_auth_methods_restricted &&
620	(http_auth_methods & ~empty_auth_useless))
621	return 1;
622	return 0;
623	}
624
625	struct curl_slist http_append_auth_header(const struct credential c,
626	struct curl_slist *headers)
627	{
628	if (c->authtype && c->credential) {
629	struct strbuf auth = STRBUF_INIT;
630	strbuf_addf(&auth, "Authorization: %s %s",
631	c->authtype, c->credential);
632	headers = curl_slist_append(headers, auth.buf);
633	strbuf_release(&auth);
634	}
635	return headers;
636	}
637
638	static void init_curl_http_auth(CURL *result)
639	{
640	if ((!http_auth.username \|\| !*http_auth.username) &&
641	(!http_auth.credential \|\| !*http_auth.credential)) {
642	if (!always_auth_proactively() && curl_empty_auth_enabled()) {
643	curl_easy_setopt(result, CURLOPT_USERPWD, ":");
644	return;
645	} else if (!always_auth_proactively()) {
646	return;
647	} else if (http_proactive_auth == PROACTIVE_AUTH_BASIC) {
648	strvec_push(&http_auth.wwwauth_headers, "Basic");
649	}
650	}
651
652	credential_fill(the_repository, &http_auth, 1);
653
654	if (http_auth.password) {
655	if (always_auth_proactively()) {
656	/*
657	* We got a credential without an authtype and we don't
658	* know what's available. Since our only two options at
659	* the moment are auto (which defaults to basic) and
660	* basic, use basic for now.
661	*/
662	curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
663	}
664	curl_easy_setopt(result, CURLOPT_USERNAME, http_auth.username);
665	curl_easy_setopt(result, CURLOPT_PASSWORD, http_auth.password);
666	}
667	}
668
669	void http_reauth_prepare(int all_capabilities)
670	{
671	/*
672	* If we deferred stripping Negotiate to give empty auth a
673	* chance (auto mode), skip credential_fill on this retry so
674	* that init_curl_http_auth() sends empty credentials and
675	* libcurl can attempt Negotiate with the system ticket cache.
676	*/
677	if (empty_auth_try_negotiate &&
678	!http_auth.password && !http_auth.credential &&
679	(http_auth_methods & CURLAUTH_GSSNEGOTIATE))
680	return;
681
682	credential_fill(the_repository, &http_auth, all_capabilities);
683	}
684
685	/* var must be free-able /
686	static void var_override(char *var, char value)
687	{
688	if (value) {
689	free(*var);
690	*var = xstrdup(value);
691	}
692	}
693
694	static void set_proxyauth_name_password(CURL *result)
695	{
696	if (proxy_auth.password) {
697	curl_easy_setopt(result, CURLOPT_PROXYUSERNAME,
698	proxy_auth.username);
699	curl_easy_setopt(result, CURLOPT_PROXYPASSWORD,
700	proxy_auth.password);
701	} else if (proxy_auth.authtype && proxy_auth.credential) {
702	curl_easy_setopt(result, CURLOPT_PROXYHEADER,
703	http_append_auth_header(&proxy_auth, NULL));
704	}
705	}
706
707	static void init_curl_proxy_auth(CURL *result)
708	{
709	if (proxy_auth.username) {
710	if (!proxy_auth.password && !proxy_auth.credential)
711	credential_fill(the_repository, &proxy_auth, 1);
712	set_proxyauth_name_password(result);
713	}
714
715	var_override(&http_proxy_authmethod, getenv("GIT_HTTP_PROXY_AUTHMETHOD"));
716
717	if (http_proxy_authmethod) {
718	int i;
719	for (i = 0; i < ARRAY_SIZE(proxy_authmethods); i++) {
720	if (!strcmp(http_proxy_authmethod, proxy_authmethods[i].name)) {
721	curl_easy_setopt(result, CURLOPT_PROXYAUTH,
722	proxy_authmethods[i].curlauth_param);
723	break;
724	}
725	}
726	if (i == ARRAY_SIZE(proxy_authmethods)) {
727	warning("unsupported proxy authentication method %s: using anyauth",
728	http_proxy_authmethod);
729	curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
730	}
731	}
732	else
733	curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
734	}
735
736	static int has_cert_password(void)
737	{
738	if (ssl_cert == NULL \|\| ssl_cert_password_required != 1)
739	return 0;
740	if (!cert_auth.password) {
741	cert_auth.protocol = xstrdup("cert");
742	cert_auth.host = xstrdup("");
743	cert_auth.username = xstrdup("");
744	cert_auth.path = xstrdup(ssl_cert);
745	credential_fill(the_repository, &cert_auth, 0);
746	}
747	return 1;
748	}
749
750	static int has_proxy_cert_password(void)
751	{
752	if (http_proxy_ssl_cert == NULL \|\| proxy_ssl_cert_password_required != 1)
753	return 0;
754	if (!proxy_cert_auth.password) {
755	proxy_cert_auth.protocol = xstrdup("cert");
756	proxy_cert_auth.host = xstrdup("");
757	proxy_cert_auth.username = xstrdup("");
758	proxy_cert_auth.path = xstrdup(http_proxy_ssl_cert);
759	credential_fill(the_repository, &proxy_cert_auth, 0);
760	}
761	return 1;
762	}
763
764	static const struct socks_proxy_type {
765	const char *name;
766	long curlsym;
767	} socks_proxy_types[] = {
768	{ "socks", CURLPROXY_SOCKS4 },
769	{ "socks4", CURLPROXY_SOCKS4 },
770	{ "socks4a", CURLPROXY_SOCKS4A },
771	{ "socks5", CURLPROXY_SOCKS5 },
772	{ "socks5h", CURLPROXY_SOCKS5_HOSTNAME },
773	};
774
775	static const struct socks_proxy_type find_socks_proxy_type(const char protocol)
776	{
777	int i;
778
779	if (!protocol)
780	return NULL;
781
782	for (i = 0; i < ARRAY_SIZE(socks_proxy_types); i++) {
783	if (!strcmp(socks_proxy_types[i].name, protocol))
784	return &socks_proxy_types[i];
785	}
786
787	return NULL;
788	}
789
790	static int is_socks_proxy_protocol(const char *protocol)
791	{
792	return !!find_socks_proxy_type(protocol);
793	}
794
795	static int set_curl_proxy_type(CURL result, const char protocol)
796	{
797	const struct socks_proxy_type *socks_proxy_type;
798
799	if (!protocol \|\| !strcmp(protocol, "http"))
800	return 0;
801
802	socks_proxy_type = find_socks_proxy_type(protocol);
803	if (socks_proxy_type) {
804	curl_easy_setopt(result, CURLOPT_PROXYTYPE, socks_proxy_type->curlsym);
805	return 0;
806	}
807
808	if (!strcmp(protocol, "https")) {
809	curl_easy_setopt(result, CURLOPT_PROXYTYPE, (long)CURLPROXY_HTTPS);
810
811	if (http_proxy_ssl_cert)
812	curl_easy_setopt(result, CURLOPT_PROXY_SSLCERT,
813	http_proxy_ssl_cert);
814
815	if (http_proxy_ssl_key)
816	curl_easy_setopt(result, CURLOPT_PROXY_SSLKEY,
817	http_proxy_ssl_key);
818
819	if (has_proxy_cert_password())
820	curl_easy_setopt(result, CURLOPT_PROXY_KEYPASSWD,
821	proxy_cert_auth.password);
822
823	return 0;
824	}
825
826	return -1;
827	}
828
829	/* Return 1 if redactions have been made, 0 otherwise. */
830	static int redact_sensitive_header(struct strbuf *header, size_t offset)
831	{
832	int ret = 0;
833	char *sensitive_header;
834
835	if (trace_curl_redact &&
836	(skip_iprefix(header->buf + offset, "Authorization:", &sensitive_header) \|\|
837	skip_iprefix(header->buf + offset, "Proxy-Authorization:", &sensitive_header))) {
838	/* The first token is the type, which is OK to log */
839	while (isspace(*sensitive_header))
840	sensitive_header++;
841	while (sensitive_header && !isspace(sensitive_header))
842	sensitive_header++;
843	/* Everything else is opaque and possibly sensitive */
844	strbuf_setlen(header, sensitive_header - header->buf);
845	strbuf_addstr(header, " <redacted>");
846	ret = 1;
847	} else if (trace_curl_redact &&
848	skip_iprefix(header->buf + offset, "Cookie:", &sensitive_header)) {
849	struct strbuf redacted_header = STRBUF_INIT;
850	char *cookie;
851
852	while (isspace(*sensitive_header))
853	sensitive_header++;
854
855	cookie = sensitive_header;
856
857	while (cookie) {
858	char *equals;
859	char *semicolon = strstr(cookie, "; ");
860	if (semicolon)
861	*semicolon = 0;
862	equals = strchrnul(cookie, '=');
863	if (!equals) {
864	/* invalid cookie, just append and continue */
865	strbuf_addstr(&redacted_header, cookie);
866	continue;
867	}
868	strbuf_add(&redacted_header, cookie, equals - cookie);
869	strbuf_addstr(&redacted_header, "=<redacted>");
870	if (semicolon) {
871	/*
872	* There are more cookies. (Or, for some
873	* reason, the input string ends in "; ".)
874	*/
875	strbuf_addstr(&redacted_header, "; ");
876	cookie = semicolon + strlen("; ");
877	} else {
878	cookie = NULL;
879	}
880	}
881
882	strbuf_setlen(header, sensitive_header - header->buf);
883	strbuf_addbuf(header, &redacted_header);
884	strbuf_release(&redacted_header);
885	ret = 1;
886	}
887	return ret;
888	}
889
890	static int match_curl_h2_trace(const char line, const char *out)
891	{
892	const char *p;
893
894	/*
895	* curl prior to 8.1.0 gives us:
896	*
897	* h2h3 [<header-name>: <header-val>]
898	*
899	* Starting in 8.1.0, the first token became just "h2".
900	*/
901	if (skip_iprefix(line, "h2h3 [", out) \|\|
902	skip_iprefix(line, "h2 [", out))
903	return 1;
904
905	/*
906	* curl 8.3.0 uses:
907	* [HTTP/2] [<stream-id>] [<header-name>: <header-val>]
908	* where <stream-id> is numeric.
909	*/
910	if (skip_iprefix(line, "[HTTP/2] [", &p)) {
911	while (isdigit(*p))
912	p++;
913	if (skip_prefix(p, "] [", out))
914	return 1;
915	}
916
917	return 0;
918	}
919
920	/* Redact headers in info */
921	static void redact_sensitive_info_header(struct strbuf *header)
922	{
923	const char *sensitive_header;
924
925	if (trace_curl_redact &&
926	match_curl_h2_trace(header->buf, &sensitive_header)) {
927	if (redact_sensitive_header(header, sensitive_header - header->buf)) {
928	/* redaction ate our closing bracket */
929	strbuf_addch(header, ']');
930	}
931	}
932	}
933
934	static void curl_dump_header(const char text, unsigned char ptr, size_t size, int hide_sensitive_header)
935	{
936	struct strbuf out = STRBUF_INIT;
937	struct strbuf headers, header;
938
939	strbuf_addf(&out, "%s, %10.10ld bytes (0x%8.8lx)\n",
940	text, (long)size, (long)size);
941	trace_strbuf(&trace_curl, &out);
942	strbuf_reset(&out);
943	strbuf_add(&out, ptr, size);
944	headers = strbuf_split_max(&out, '\n', 0);
945
946	for (header = headers; *header; header++) {
947	if (hide_sensitive_header)
948	redact_sensitive_header(*header, 0);
949	strbuf_insertstr((*header), 0, text);
950	strbuf_insertstr((*header), strlen(text), ": ");
951	strbuf_rtrim((*header));
952	strbuf_addch((*header), '\n');
953	trace_strbuf(&trace_curl, (*header));
954	}
955	strbuf_list_free(headers);
956	strbuf_release(&out);
957	}
958
959	static void curl_dump_data(const char text, unsigned char ptr, size_t size)
960	{
961	size_t i;
962	struct strbuf out = STRBUF_INIT;
963	unsigned int width = 60;
964
965	strbuf_addf(&out, "%s, %10.10ld bytes (0x%8.8lx)\n",
966	text, (long)size, (long)size);
967	trace_strbuf(&trace_curl, &out);
968
969	for (i = 0; i < size; i += width) {
970	size_t w;
971
972	strbuf_reset(&out);
973	strbuf_addf(&out, "%s: ", text);
974	for (w = 0; (w < width) && (i + w < size); w++) {
975	unsigned char ch = ptr[i + w];
976
977	strbuf_addch(&out,
978	(ch >= 0x20) && (ch < 0x80)
979	? ch : '.');
980	}
981	strbuf_addch(&out, '\n');
982	trace_strbuf(&trace_curl, &out);
983	}
984	strbuf_release(&out);
985	}
986
987	static void curl_dump_info(char *data, size_t size)
988	{
989	struct strbuf buf = STRBUF_INIT;
990
991	strbuf_add(&buf, data, size);
992
993	redact_sensitive_info_header(&buf);
994	trace_printf_key(&trace_curl, "== Info: %s", buf.buf);
995
996	strbuf_release(&buf);
997	}
998
999	static int curl_trace(CURL *handle UNUSED, curl_infotype type,
1000	char *data, size_t size,
1001	void *userp UNUSED)
1002	{
1003	const char *text;
1004	enum { NO_FILTER = 0, DO_FILTER = 1 };
1005
1006	switch (type) {
1007	case CURLINFO_TEXT:
1008	curl_dump_info(data, size);
1009	break;
1010	case CURLINFO_HEADER_OUT:
1011	text = "=> Send header";
1012	curl_dump_header(text, (unsigned char *)data, size, DO_FILTER);
1013	break;
1014	case CURLINFO_DATA_OUT:
1015	if (trace_curl_data) {
1016	text = "=> Send data";
1017	curl_dump_data(text, (unsigned char *)data, size);
1018	}
1019	break;
1020	case CURLINFO_SSL_DATA_OUT:
1021	if (trace_curl_data) {
1022	text = "=> Send SSL data";
1023	curl_dump_data(text, (unsigned char *)data, size);
1024	}
1025	break;
1026	case CURLINFO_HEADER_IN:
1027	text = "<= Recv header";
1028	curl_dump_header(text, (unsigned char *)data, size, NO_FILTER);
1029	break;
1030	case CURLINFO_DATA_IN:
1031	if (trace_curl_data) {
1032	text = "<= Recv data";
1033	curl_dump_data(text, (unsigned char *)data, size);
1034	}
1035	break;
1036	case CURLINFO_SSL_DATA_IN:
1037	if (trace_curl_data) {
1038	text = "<= Recv SSL data";
1039	curl_dump_data(text, (unsigned char *)data, size);
1040	}
1041	break;
1042
1043	default: /* we ignore unknown types by default */
1044	return 0;
1045	}
1046	return 0;
1047	}
1048
1049	void http_trace_curl_no_data(void)
1050	{
1051	trace_override_envvar(&trace_curl, "1");
1052	trace_curl_data = 0;
1053	}
1054
1055	void setup_curl_trace(CURL *handle)
1056	{
1057	if (!trace_want(&trace_curl))
1058	return;
1059	curl_easy_setopt(handle, CURLOPT_VERBOSE, 1L);
1060	curl_easy_setopt(handle, CURLOPT_DEBUGFUNCTION, curl_trace);
1061	curl_easy_setopt(handle, CURLOPT_DEBUGDATA, NULL);
1062	}
1063
1064	static void proto_list_append(struct strbuf list, const char proto)
1065	{
1066	if (!list)
1067	return;
1068	if (list->len)
1069	strbuf_addch(list, ',');
1070	strbuf_addstr(list, proto);
1071	}
1072
1073	static long get_curl_allowed_protocols(int from_user, struct strbuf *list)
1074	{
1075	long bits = 0;
1076
1077	if (is_transport_allowed("http", from_user)) {
1078	bits \|= CURLPROTO_HTTP;
1079	proto_list_append(list, "http");
1080	}
1081	if (is_transport_allowed("https", from_user)) {
1082	bits \|= CURLPROTO_HTTPS;
1083	proto_list_append(list, "https");
1084	}
1085	if (is_transport_allowed("ftp", from_user)) {
1086	bits \|= CURLPROTO_FTP;
1087	proto_list_append(list, "ftp");
1088	}
1089	if (is_transport_allowed("ftps", from_user)) {
1090	bits \|= CURLPROTO_FTPS;
1091	proto_list_append(list, "ftps");
1092	}
1093
1094	return bits;
1095	}
1096
1097	static int get_curl_http_version_opt(const char version_string, long opt)
1098	{
1099	int i;
1100	static struct {
1101	const char *name;
1102	long opt_token;
1103	} choice[] = {
1104	{ "HTTP/1.1", CURL_HTTP_VERSION_1_1 },
1105	{ "HTTP/2", CURL_HTTP_VERSION_2 }
1106	};
1107
1108	for (i = 0; i < ARRAY_SIZE(choice); i++) {
1109	if (!strcmp(version_string, choice[i].name)) {
1110	*opt = choice[i].opt_token;
1111	return 0;
1112	}
1113	}
1114
1115	warning("unknown value given to http.version: '%s'", version_string);
1116	return -1; /* not found */
1117	}
1118
1119	static CURL *get_curl_handle(void)
1120	{
1121	CURL *result = curl_easy_init();
1122
1123	if (!result)
1124	die("curl_easy_init failed");
1125
1126	if (!curl_ssl_verify) {
1127	curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 0L);
1128	curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 0L);
1129	} else {
1130	/* Verify authenticity of the peer's certificate */
1131	curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 1L);
1132	/* The name in the cert must match whom we tried to connect */
1133	curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 2L);
1134	}
1135
1136	if (curl_http_version) {
1137	long opt;
1138	if (!get_curl_http_version_opt(curl_http_version, &opt)) {
1139	/* Set request use http version */
1140	curl_easy_setopt(result, CURLOPT_HTTP_VERSION, opt);
1141	}
1142	}
1143
1144	curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
1145	curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
1146
1147	#ifdef CURLGSSAPI_DELEGATION_FLAG
1148	if (curl_deleg) {
1149	int i;
1150	for (i = 0; i < ARRAY_SIZE(curl_deleg_levels); i++) {
1151	if (!strcmp(curl_deleg, curl_deleg_levels[i].name)) {
1152	curl_easy_setopt(result, CURLOPT_GSSAPI_DELEGATION,
1153	curl_deleg_levels[i].curl_deleg_param);
1154	break;
1155	}
1156	}
1157	if (i == ARRAY_SIZE(curl_deleg_levels))
1158	warning("Unknown delegation method '%s': using default",
1159	curl_deleg);
1160	}
1161	#endif
1162
1163	if (http_ssl_backend && !strcmp("schannel", http_ssl_backend) &&
1164	!http_schannel_check_revoke) {
1165	curl_easy_setopt(result, CURLOPT_SSL_OPTIONS, (long)CURLSSLOPT_NO_REVOKE);
1166	}
1167
1168	if (http_proactive_auth != PROACTIVE_AUTH_NONE)
1169	init_curl_http_auth(result);
1170
1171	if (getenv("GIT_SSL_VERSION"))
1172	ssl_version = getenv("GIT_SSL_VERSION");
1173	if (ssl_version && *ssl_version) {
1174	int i;
1175	for (i = 0; i < ARRAY_SIZE(sslversions); i++) {
1176	if (!strcmp(ssl_version, sslversions[i].name)) {
1177	curl_easy_setopt(result, CURLOPT_SSLVERSION,
1178	sslversions[i].ssl_version);
1179	break;
1180	}
1181	}
1182	if (i == ARRAY_SIZE(sslversions))
1183	warning("unsupported ssl version %s: using default",
1184	ssl_version);
1185	}
1186
1187	if (getenv("GIT_SSL_CIPHER_LIST"))
1188	ssl_cipherlist = getenv("GIT_SSL_CIPHER_LIST");
1189	if (ssl_cipherlist != NULL && *ssl_cipherlist)
1190	curl_easy_setopt(result, CURLOPT_SSL_CIPHER_LIST,
1191	ssl_cipherlist);
1192
1193	if (ssl_cert)
1194	curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
1195	if (ssl_cert_type)
1196	curl_easy_setopt(result, CURLOPT_SSLCERTTYPE, ssl_cert_type);
1197	if (has_cert_password())
1198	curl_easy_setopt(result, CURLOPT_KEYPASSWD, cert_auth.password);
1199	if (ssl_key)
1200	curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
1201	if (ssl_key_type)
1202	curl_easy_setopt(result, CURLOPT_SSLKEYTYPE, ssl_key_type);
1203	if (ssl_capath)
1204	curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
1205	if (ssl_pinnedkey)
1206	curl_easy_setopt(result, CURLOPT_PINNEDPUBLICKEY, ssl_pinnedkey);
1207	if (http_ssl_backend && !strcmp("schannel", http_ssl_backend) &&
1208	!http_schannel_use_ssl_cainfo) {
1209	curl_easy_setopt(result, CURLOPT_CAINFO, NULL);
1210	curl_easy_setopt(result, CURLOPT_PROXY_CAINFO, NULL);
1211	} else if (ssl_cainfo != NULL \|\| http_proxy_ssl_ca_info != NULL) {
1212	if (ssl_cainfo)
1213	curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
1214	if (http_proxy_ssl_ca_info)
1215	curl_easy_setopt(result, CURLOPT_PROXY_CAINFO, http_proxy_ssl_ca_info);
1216	}
1217
1218	if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
1219	curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
1220	curl_low_speed_limit);
1221	curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
1222	curl_low_speed_time);
1223	}
1224
1225	curl_easy_setopt(result, CURLOPT_MAXREDIRS, 20L);
1226	curl_easy_setopt(result, CURLOPT_POSTREDIR, (long)CURL_REDIR_POST_ALL);
1227
1228	#ifdef GIT_CURL_HAVE_CURLOPT_PROTOCOLS_STR
1229	{
1230	struct strbuf buf = STRBUF_INIT;
1231
1232	get_curl_allowed_protocols(0, &buf);
1233	curl_easy_setopt(result, CURLOPT_REDIR_PROTOCOLS_STR, buf.buf);
1234	strbuf_reset(&buf);
1235
1236	get_curl_allowed_protocols(-1, &buf);
1237	curl_easy_setopt(result, CURLOPT_PROTOCOLS_STR, buf.buf);
1238	strbuf_release(&buf);
1239	}
1240	#else
1241	curl_easy_setopt(result, CURLOPT_REDIR_PROTOCOLS,
1242	get_curl_allowed_protocols(0, NULL));
1243	curl_easy_setopt(result, CURLOPT_PROTOCOLS,
1244	get_curl_allowed_protocols(-1, NULL));
1245	#endif
1246
1247	if (getenv("GIT_CURL_VERBOSE"))
1248	http_trace_curl_no_data();
1249	setup_curl_trace(result);
1250	if (getenv("GIT_TRACE_CURL_NO_DATA"))
1251	trace_curl_data = 0;
1252	if (!git_env_bool("GIT_TRACE_REDACT", 1))
1253	trace_curl_redact = 0;
1254
1255	curl_easy_setopt(result, CURLOPT_USERAGENT,
1256	user_agent ? user_agent : git_user_agent());
1257
1258	if (curl_ftp_no_epsv)
1259	curl_easy_setopt(result, CURLOPT_FTP_USE_EPSV, 0L);
1260
1261	if (curl_ssl_try)
1262	curl_easy_setopt(result, CURLOPT_USE_SSL, CURLUSESSL_TRY);
1263
1264	/*
1265	* CURL also examines these variables as a fallback; but we need to query
1266	* them here in order to decide whether to prompt for missing password (cf.
1267	* init_curl_proxy_auth()).
1268	*
1269	* Unlike many other common environment variables, these are historically
1270	* lowercase only. It appears that CURL did not know this and implemented
1271	* only uppercase variants, which was later corrected to take both - with
1272	* the exception of http_proxy, which is lowercase only also in CURL. As
1273	* the lowercase versions are the historical quasi-standard, they take
1274	* precedence here, as in CURL.
1275	*/
1276	if (!curl_http_proxy) {
1277	if (http_auth.protocol && !strcmp(http_auth.protocol, "https")) {
1278	var_override(&curl_http_proxy, getenv("HTTPS_PROXY"));
1279	var_override(&curl_http_proxy, getenv("https_proxy"));
1280	} else {
1281	var_override(&curl_http_proxy, getenv("http_proxy"));
1282	}
1283	if (!curl_http_proxy) {
1284	var_override(&curl_http_proxy, getenv("ALL_PROXY"));
1285	var_override(&curl_http_proxy, getenv("all_proxy"));
1286	}
1287	}
1288
1289	if (curl_http_proxy && curl_http_proxy[0] == '\0') {
1290	/*
1291	* Handle case with the empty http.proxy value here to keep
1292	* common code clean.
1293	* NB: empty option disables proxying at all.
1294	*/
1295	curl_easy_setopt(result, CURLOPT_PROXY, "");
1296	} else if (curl_http_proxy) {
1297	struct strbuf proxy = STRBUF_INIT;
1298
1299	if (strstr(curl_http_proxy, "://"))
1300	credential_from_url(&proxy_auth, curl_http_proxy);
1301	else {
1302	struct strbuf url = STRBUF_INIT;
1303	strbuf_addf(&url, "http://%s", curl_http_proxy);
1304	credential_from_url(&proxy_auth, url.buf);
1305	strbuf_release(&url);
1306	}
1307
1308	if (set_curl_proxy_type(result, proxy_auth.protocol) < 0)
1309	die("Invalid proxy URL '%s': unsupported proxy scheme '%s'",
1310	curl_http_proxy, proxy_auth.protocol);
1311
1312	if (!proxy_auth.host)
1313	die("Invalid proxy URL '%s'", curl_http_proxy);
1314
1315	strbuf_addstr(&proxy, proxy_auth.host);
1316	if (proxy_auth.path) {
1317	curl_version_info_data *ver = curl_version_info(CURLVERSION_NOW);
1318
1319	if (ver->version_num < 0x075400)
1320	die("libcurl 7.84 or later is required to support paths in proxy URLs");
1321
1322	if (!is_socks_proxy_protocol(proxy_auth.protocol))
1323	die("Invalid proxy URL '%s': only SOCKS proxies support paths",
1324	curl_http_proxy);
1325
1326	if (strcasecmp(proxy_auth.host, "localhost"))
1327	die("Invalid proxy URL '%s': host must be localhost if a path is present",
1328	curl_http_proxy);
1329
1330	strbuf_addch(&proxy, '/');
1331	strbuf_add_percentencode(&proxy, proxy_auth.path, 0);
1332	}
1333	curl_easy_setopt(result, CURLOPT_PROXY, proxy.buf);
1334	strbuf_release(&proxy);
1335
1336	var_override(&curl_no_proxy, getenv("NO_PROXY"));
1337	var_override(&curl_no_proxy, getenv("no_proxy"));
1338	curl_easy_setopt(result, CURLOPT_NOPROXY, curl_no_proxy);
1339	}
1340	init_curl_proxy_auth(result);
1341
1342	curl_easy_setopt(result, CURLOPT_TCP_KEEPALIVE, 1L);
1343
1344	if (curl_tcp_keepidle > -1)
1345	curl_easy_setopt(result, CURLOPT_TCP_KEEPIDLE,
1346	curl_tcp_keepidle);
1347	if (curl_tcp_keepintvl > -1)
1348	curl_easy_setopt(result, CURLOPT_TCP_KEEPINTVL,
1349	curl_tcp_keepintvl);
1350	#ifdef GIT_CURL_HAVE_CURLOPT_TCP_KEEPCNT
1351	if (curl_tcp_keepcnt > -1)
1352	curl_easy_setopt(result, CURLOPT_TCP_KEEPCNT, curl_tcp_keepcnt);
1353	#endif
1354
1355	return result;
1356	}
1357
1358	static void set_from_env(char *var, const char envname)
1359	{
1360	const char *val = getenv(envname);
1361	if (val) {
1362	FREE_AND_NULL(*var);
1363	*var = xstrdup(val);
1364	}
1365	}
1366
1367	static void set_long_from_env(long var, const char envname)
1368	{
1369	const char *val = getenv(envname);
1370	if (val) {
1371	long tmp;
1372	char *endp;
1373	int saved_errno = errno;
1374
1375	errno = 0;
1376	tmp = strtol(val, &endp, 10);
1377
1378	if (errno)
1379	warning_errno(_("failed to parse %s"), envname);
1380	else if (*endp \|\| endp == val)
1381	warning(_("failed to parse %s"), envname);
1382	else
1383	*var = tmp;
1384
1385	errno = saved_errno;
1386	}
1387	}
1388
1389	void http_init(struct remote remote, const char url, int proactive_auth)
1390	{
1391	char *normalized_url;
1392	struct urlmatch_config config = URLMATCH_CONFIG_INIT;
1393
1394	config.section = "http";
1395	config.key = NULL;
1396	config.collect_fn = http_options;
1397	config.cascade_fn = git_default_config;
1398	config.cb = NULL;
1399
1400	http_is_verbose = 0;
1401	normalized_url = url_normalize(url, &config.url);
1402
1403	repo_config(the_repository, urlmatch_config_entry, &config);
1404	free(normalized_url);
1405	string_list_clear(&config.vars, 1);
1406
1407	if (http_ssl_backend) {
1408	const curl_ssl_backend **backends;
1409	struct strbuf buf = STRBUF_INIT;
1410	int i;
1411
1412	switch (curl_global_sslset(-1, http_ssl_backend, &backends)) {
1413	case CURLSSLSET_UNKNOWN_BACKEND:
1414	strbuf_addf(&buf, _("Unsupported SSL backend '%s'. "
1415	"Supported SSL backends:"),
1416	http_ssl_backend);
1417	for (i = 0; backends[i]; i++)
1418	strbuf_addf(&buf, "\n\t%s", backends[i]->name);
1419	die("%s", buf.buf);
1420	case CURLSSLSET_NO_BACKENDS:
1421	die(_("Could not set SSL backend to '%s': "
1422	"cURL was built without SSL backends"),
1423	http_ssl_backend);
1424	case CURLSSLSET_TOO_LATE:
1425	die(_("Could not set SSL backend to '%s': already set"),
1426	http_ssl_backend);
1427	case CURLSSLSET_OK:
1428	break; /* Okay! */
1429	}
1430	}
1431
1432	if (curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK)
1433	die("curl_global_init failed");
1434
1435	#ifdef GIT_CURL_HAVE_GLOBAL_TRACE
1436	{
1437	const char *comp = getenv("GIT_TRACE_CURL_COMPONENTS");
1438	if (comp)
1439	curl_global_trace(comp);
1440	}
1441	#endif
1442
1443	if (proactive_auth && http_proactive_auth == PROACTIVE_AUTH_NONE)
1444	http_proactive_auth = PROACTIVE_AUTH_IF_CREDENTIALS;
1445
1446	if (remote && remote->http_proxy)
1447	curl_http_proxy = xstrdup(remote->http_proxy);
1448
1449	if (remote)
1450	var_override(&http_proxy_authmethod, remote->http_proxy_authmethod);
1451
1452	pragma_header = curl_slist_append(http_copy_default_headers(),
1453	"Pragma: no-cache");
1454
1455	{
1456	char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
1457	if (http_max_requests)
1458	max_requests = atoi(http_max_requests);
1459	}
1460
1461	curlm = curl_multi_init();
1462	if (!curlm)
1463	die("curl_multi_init failed");
1464
1465	if (getenv("GIT_SSL_NO_VERIFY"))
1466	curl_ssl_verify = 0;
1467
1468	set_from_env(&ssl_cert, "GIT_SSL_CERT");
1469	set_from_env(&ssl_cert_type, "GIT_SSL_CERT_TYPE");
1470	set_from_env(&ssl_key, "GIT_SSL_KEY");
1471	set_from_env(&ssl_key_type, "GIT_SSL_KEY_TYPE");
1472	set_from_env(&ssl_capath, "GIT_SSL_CAPATH");
1473	set_from_env(&ssl_cainfo, "GIT_SSL_CAINFO");
1474
1475	set_from_env(&user_agent, "GIT_HTTP_USER_AGENT");
1476
1477	set_long_from_env(&curl_low_speed_limit, "GIT_HTTP_LOW_SPEED_LIMIT");
1478	set_long_from_env(&curl_low_speed_time, "GIT_HTTP_LOW_SPEED_TIME");
1479
1480	if (curl_ssl_verify == -1)
1481	curl_ssl_verify = 1;
1482
1483	curl_session_count = 0;
1484	if (max_requests < 1)
1485	max_requests = DEFAULT_MAX_REQUESTS;
1486
1487	set_from_env(&http_proxy_ssl_cert, "GIT_PROXY_SSL_CERT");
1488	set_from_env(&http_proxy_ssl_key, "GIT_PROXY_SSL_KEY");
1489	set_from_env(&http_proxy_ssl_ca_info, "GIT_PROXY_SSL_CAINFO");
1490
1491	if (getenv("GIT_PROXY_SSL_CERT_PASSWORD_PROTECTED"))
1492	proxy_ssl_cert_password_required = 1;
1493
1494	if (getenv("GIT_CURL_FTP_NO_EPSV"))
1495	curl_ftp_no_epsv = 1;
1496
1497	if (url) {
1498	credential_from_url(&http_auth, url);
1499	if (!ssl_cert_password_required &&
1500	getenv("GIT_SSL_CERT_PASSWORD_PROTECTED") &&
1501	starts_with(url, "https://"))
1502	ssl_cert_password_required = 1;
1503	}
1504
1505	set_long_from_env(&curl_tcp_keepidle, "GIT_TCP_KEEPIDLE");
1506	set_long_from_env(&curl_tcp_keepintvl, "GIT_TCP_KEEPINTVL");
1507	set_long_from_env(&curl_tcp_keepcnt, "GIT_TCP_KEEPCNT");
1508
1509	set_long_from_env(&http_retry_after, "GIT_HTTP_RETRY_AFTER");
1510	set_long_from_env(&http_max_retries, "GIT_HTTP_MAX_RETRIES");
1511	set_long_from_env(&http_max_retry_time, "GIT_HTTP_MAX_RETRY_TIME");
1512
1513	curl_default = get_curl_handle();
1514	}
1515
1516	void http_cleanup(void)
1517	{
1518	struct active_request_slot *slot = active_queue_head;
1519
1520	while (slot != NULL) {
1521	struct active_request_slot *next = slot->next;
1522	if (slot->curl) {
1523	xmulti_remove_handle(slot);
1524	curl_easy_cleanup(slot->curl);
1525	}
1526	free(slot);
1527	slot = next;
1528	}
1529	active_queue_head = NULL;
1530
1531	curl_easy_cleanup(curl_default);
1532
1533	curl_multi_cleanup(curlm);
1534	curl_global_cleanup();
1535
1536	string_list_clear(&extra_http_headers, 0);
1537
1538	curl_slist_free_all(pragma_header);
1539	pragma_header = NULL;
1540
1541	curl_slist_free_all(host_resolutions);
1542	host_resolutions = NULL;
1543
1544	if (curl_http_proxy) {
1545	free((void *)curl_http_proxy);
1546	curl_http_proxy = NULL;
1547	}
1548
1549	if (proxy_auth.password) {
1550	memset(proxy_auth.password, 0, strlen(proxy_auth.password));
1551	FREE_AND_NULL(proxy_auth.password);
1552	}
1553
1554	free((void *)curl_proxyuserpwd);
1555	curl_proxyuserpwd = NULL;
1556
1557	free((void *)http_proxy_authmethod);
1558	http_proxy_authmethod = NULL;
1559
1560	if (cert_auth.password) {
1561	memset(cert_auth.password, 0, strlen(cert_auth.password));
1562	FREE_AND_NULL(cert_auth.password);
1563	}
1564	ssl_cert_password_required = 0;
1565
1566	if (proxy_cert_auth.password) {
1567	memset(proxy_cert_auth.password, 0, strlen(proxy_cert_auth.password));
1568	FREE_AND_NULL(proxy_cert_auth.password);
1569	}
1570	proxy_ssl_cert_password_required = 0;
1571
1572	FREE_AND_NULL(cached_accept_language);
1573	}
1574
1575	struct active_request_slot *get_active_slot(void)
1576	{
1577	struct active_request_slot *slot = active_queue_head;
1578	struct active_request_slot *newslot;
1579
1580	int num_transfers;
1581
1582	/* Wait for a slot to open up if the queue is full */
1583	while (active_requests >= max_requests) {
1584	curl_multi_perform(curlm, &num_transfers);
1585	if (num_transfers < active_requests)
1586	process_curl_messages();
1587	}
1588
1589	while (slot != NULL && slot->in_use)
1590	slot = slot->next;
1591
1592	if (!slot) {
1593	newslot = xmalloc(sizeof(*newslot));
1594	newslot->curl = NULL;
1595	newslot->in_use = 0;
1596	newslot->next = NULL;
1597
1598	slot = active_queue_head;
1599	if (!slot) {
1600	active_queue_head = newslot;
1601	} else {
1602	while (slot->next != NULL)
1603	slot = slot->next;
1604	slot->next = newslot;
1605	}
1606	slot = newslot;
1607	}
1608
1609	if (!slot->curl) {
1610	slot->curl = curl_easy_duphandle(curl_default);
1611	curl_session_count++;
1612	}
1613
1614	active_requests++;
1615	slot->in_use = 1;
1616	slot->results = NULL;
1617	slot->finished = NULL;
1618	slot->callback_data = NULL;
1619	slot->callback_func = NULL;
1620
1621	if (curl_cookie_file && !strcmp(curl_cookie_file, "-")) {
1622	warning(_("refusing to read cookies from http.cookiefile '-'"));
1623	FREE_AND_NULL(curl_cookie_file);
1624	}
1625	curl_easy_setopt(slot->curl, CURLOPT_COOKIEFILE, curl_cookie_file);
1626	if (curl_save_cookies && (!curl_cookie_file \|\| !curl_cookie_file[0])) {
1627	curl_save_cookies = 0;
1628	warning(_("ignoring http.savecookies for empty http.cookiefile"));
1629	}
1630	if (curl_save_cookies)
1631	curl_easy_setopt(slot->curl, CURLOPT_COOKIEJAR, curl_cookie_file);
1632	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
1633	curl_easy_setopt(slot->curl, CURLOPT_RESOLVE, host_resolutions);
1634	curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
1635	curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, NULL);
1636	curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, NULL);
1637	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, NULL);
1638	curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, NULL);
1639	curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, -1L);
1640	curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 0L);
1641	curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1L);
1642	curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 1L);
1643	curl_easy_setopt(slot->curl, CURLOPT_RANGE, NULL);
1644
1645	/*
1646	* Default following to off unless "ALWAYS" is configured; this gives
1647	* callers a sane starting point, and they can tweak for individual
1648	* HTTP_FOLLOW_* cases themselves.
1649	*/
1650	if (http_follow_config == HTTP_FOLLOW_ALWAYS)
1651	curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 1L);
1652	else
1653	curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 0L);
1654
1655	curl_easy_setopt(slot->curl, CURLOPT_IPRESOLVE, git_curl_ipresolve);
1656	curl_easy_setopt(slot->curl, CURLOPT_HTTPAUTH, http_auth_methods);
1657	if (http_auth.password \|\| http_auth.credential \|\| curl_empty_auth_enabled())
1658	init_curl_http_auth(slot->curl);
1659
1660	return slot;
1661	}
1662
1663	int start_active_slot(struct active_request_slot *slot)
1664	{
1665	CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
1666	int num_transfers;
1667
1668	if (curlm_result != CURLM_OK &&
1669	curlm_result != CURLM_CALL_MULTI_PERFORM) {
1670	warning("curl_multi_add_handle failed: %s",
1671	curl_multi_strerror(curlm_result));
1672	active_requests--;
1673	slot->in_use = 0;
1674	return 0;
1675	}
1676
1677	/*
1678	* We know there must be something to do, since we just added
1679	* something.
1680	*/
1681	curl_multi_perform(curlm, &num_transfers);
1682	return 1;
1683	}
1684
1685	struct fill_chain {
1686	void *data;
1687	int (fill)(void );
1688	struct fill_chain *next;
1689	};
1690
1691	static struct fill_chain *fill_cfg;
1692
1693	void add_fill_function(void data, int (fill)(void *))
1694	{
1695	struct fill_chain new_fill = xmalloc(sizeof(new_fill));
1696	struct fill_chain **linkp = &fill_cfg;
1697	new_fill->data = data;
1698	new_fill->fill = fill;
1699	new_fill->next = NULL;
1700	while (*linkp)
1701	linkp = &(*linkp)->next;
1702	*linkp = new_fill;
1703	}
1704
1705	void fill_active_slots(void)
1706	{
1707	struct active_request_slot *slot = active_queue_head;
1708
1709	while (active_requests < max_requests) {
1710	struct fill_chain *fill;
1711	for (fill = fill_cfg; fill; fill = fill->next)
1712	if (fill->fill(fill->data))
1713	break;
1714
1715	if (!fill)
1716	break;
1717	}
1718
1719	while (slot != NULL) {
1720	if (!slot->in_use && slot->curl != NULL
1721	&& curl_session_count > min_curl_sessions) {
1722	curl_easy_cleanup(slot->curl);
1723	slot->curl = NULL;
1724	curl_session_count--;
1725	}
1726	slot = slot->next;
1727	}
1728	}
1729
1730	void step_active_slots(void)
1731	{
1732	int num_transfers;
1733	CURLMcode curlm_result;
1734
1735	do {
1736	curlm_result = curl_multi_perform(curlm, &num_transfers);
1737	} while (curlm_result == CURLM_CALL_MULTI_PERFORM);
1738	if (num_transfers < active_requests) {
1739	process_curl_messages();
1740	fill_active_slots();
1741	}
1742	}
1743
1744	void run_active_slot(struct active_request_slot *slot)
1745	{
1746	fd_set readfds;
1747	fd_set writefds;
1748	fd_set excfds;
1749	int max_fd;
1750	struct timeval select_timeout;
1751	int finished = 0;
1752
1753	slot->finished = &finished;
1754	while (!finished) {
1755	step_active_slots();
1756
1757	if (slot->in_use) {
1758	long curl_timeout;
1759	curl_multi_timeout(curlm, &curl_timeout);
1760	if (curl_timeout == 0) {
1761	continue;
1762	} else if (curl_timeout == -1) {
1763	select_timeout.tv_sec = 0;
1764	select_timeout.tv_usec = 50000;
1765	} else {
1766	select_timeout.tv_sec = curl_timeout / 1000;
1767	select_timeout.tv_usec = (curl_timeout % 1000) * 1000;
1768	}
1769
1770	max_fd = -1;
1771	FD_ZERO(&readfds);
1772	FD_ZERO(&writefds);
1773	FD_ZERO(&excfds);
1774	curl_multi_fdset(curlm, &readfds, &writefds, &excfds, &max_fd);
1775
1776	/*
1777	* It can happen that curl_multi_timeout returns a pathologically
1778	* long timeout when curl_multi_fdset returns no file descriptors
1779	* to read. See commit message for more details.
1780	*/
1781	if (max_fd < 0 &&
1782	(select_timeout.tv_sec > 0 \|\|
1783	select_timeout.tv_usec > 50000)) {
1784	select_timeout.tv_sec = 0;
1785	select_timeout.tv_usec = 50000;
1786	}
1787
1788	select(max_fd+1, &readfds, &writefds, &excfds, &select_timeout);
1789	}
1790	}
1791
1792	/*
1793	* The value of slot->finished we set before the loop was used
1794	* to set our "finished" variable when our request completed.
1795	*
1796	* 1. The slot may not have been reused for another request
1797	* yet, in which case it still has &finished.
1798	*
1799	* 2. The slot may already be in-use to serve another request,
1800	* which can further be divided into two cases:
1801	*
1802	* (a) If call run_active_slot() hasn't been called for that
1803	* other request, slot->finished would have been cleared
1804	* by get_active_slot() and has NULL.
1805	*
1806	* (b) If the request did call run_active_slot(), then the
1807	* call would have updated slot->finished at the beginning
1808	* of this function, and with the clearing of the member
1809	* below, we would find that slot->finished is now NULL.
1810	*
1811	* In all cases, slot->finished has no useful information to
1812	* anybody at this point. Some compilers warn us for
1813	* attempting to smuggle a pointer that is about to become
1814	* invalid, i.e. &finished. We clear it here to assure them.
1815	*/
1816	slot->finished = NULL;
1817	}
1818
1819	static void release_active_slot(struct active_request_slot *slot)
1820	{
1821	closedown_active_slot(slot);
1822	if (slot->curl) {
1823	xmulti_remove_handle(slot);
1824	if (curl_session_count > min_curl_sessions) {
1825	curl_easy_cleanup(slot->curl);
1826	slot->curl = NULL;
1827	curl_session_count--;
1828	}
1829	}
1830	fill_active_slots();
1831	}
1832
1833	void finish_all_active_slots(void)
1834	{
1835	struct active_request_slot *slot = active_queue_head;
1836
1837	while (slot != NULL)
1838	if (slot->in_use) {
1839	run_active_slot(slot);
1840	slot = active_queue_head;
1841	} else {
1842	slot = slot->next;
1843	}
1844	}
1845
1846	/* Helpers for modifying and creating URLs */
1847	static inline int needs_quote(int ch)
1848	{
1849	if (((ch >= 'A') && (ch <= 'Z'))
1850	\|\| ((ch >= 'a') && (ch <= 'z'))
1851	\|\| ((ch >= '0') && (ch <= '9'))
1852	\|\| (ch == '/')
1853	\|\| (ch == '-')
1854	\|\| (ch == '.'))
1855	return 0;
1856	return 1;
1857	}
1858
1859	static char quote_ref_url(const char base, const char *ref)
1860	{
1861	struct strbuf buf = STRBUF_INIT;
1862	const char *cp;
1863	int ch;
1864
1865	end_url_with_slash(&buf, base);
1866
1867	for (cp = ref; (ch = *cp) != 0; cp++)
1868	if (needs_quote(ch))
1869	strbuf_addf(&buf, "%%%02x", ch);
1870	else
1871	strbuf_addch(&buf, *cp);
1872
1873	return strbuf_detach(&buf, NULL);
1874	}
1875
1876	void append_remote_object_url(struct strbuf buf, const char url,
1877	const char *hex,
1878	int only_two_digit_prefix)
1879	{
1880	end_url_with_slash(buf, url);
1881
1882	strbuf_addf(buf, "objects/%.*s/", 2, hex);
1883	if (!only_two_digit_prefix)
1884	strbuf_addstr(buf, hex + 2);
1885	}
1886
1887	char get_remote_object_url(const char url, const char *hex,
1888	int only_two_digit_prefix)
1889	{
1890	struct strbuf buf = STRBUF_INIT;
1891	append_remote_object_url(&buf, url, hex, only_two_digit_prefix);
1892	return strbuf_detach(&buf, NULL);
1893	}
1894
1895	void normalize_curl_result(CURLcode *result, long http_code,
1896	char *errorstr, size_t errorlen)
1897	{
1898	/*
1899	* If we see a failing http code with CURLE_OK, we have turned off
1900	* FAILONERROR (to keep the server's custom error response), and should
1901	* translate the code into failure here.
1902	*
1903	* Likewise, if we see a redirect (30x code), that means we turned off
1904	* redirect-following, and we should treat the result as an error.
1905	*/
1906	if (*result == CURLE_OK && http_code >= 300) {
1907	*result = CURLE_HTTP_RETURNED_ERROR;
1908	/*
1909	* Normally curl will already have put the "reason phrase"
1910	* from the server into curl_errorstr; unfortunately without
1911	* FAILONERROR it is lost, so we can give only the numeric
1912	* status code.
1913	*/
1914	xsnprintf(errorstr, errorlen,
1915	"The requested URL returned error: %ld",
1916	http_code);
1917	}
1918	}
1919
1920	static int handle_curl_result(struct slot_results *results)
1921	{
1922	normalize_curl_result(&results->curl_result, results->http_code,
1923	curl_errorstr, sizeof(curl_errorstr));
1924
1925	if (results->curl_result == CURLE_OK) {
1926	credential_approve(the_repository, &http_auth);
1927	credential_approve(the_repository, &proxy_auth);
1928	credential_approve(the_repository, &cert_auth);
1929	return HTTP_OK;
1930	} else if (results->curl_result == CURLE_SSL_CERTPROBLEM) {
1931	/*
1932	* We can't tell from here whether it's a bad path, bad
1933	* certificate, bad password, or something else wrong
1934	* with the certificate. So we reject the credential to
1935	* avoid caching or saving a bad password.
1936	*/
1937	credential_reject(the_repository, &cert_auth);
1938	return HTTP_NOAUTH;
1939	} else if (results->curl_result == CURLE_SSL_PINNEDPUBKEYNOTMATCH) {
1940	return HTTP_NOMATCHPUBLICKEY;
1941	} else if (missing_target(results))
1942	return HTTP_MISSING_TARGET;
1943	else if (results->http_code == 401) {
1944	if ((http_auth.username && http_auth.password) \|\|\
1945	(http_auth.authtype && http_auth.credential)) {
1946	if (http_auth.multistage) {
1947	credential_clear_secrets(&http_auth);
1948	return HTTP_REAUTH;
1949	}
1950	credential_reject(the_repository, &http_auth);
1951	if (always_auth_proactively())
1952	http_proactive_auth = PROACTIVE_AUTH_NONE;
1953	return HTTP_NOAUTH;
1954	} else {
1955	if (curl_empty_auth == -1 &&
1956	!empty_auth_try_negotiate &&
1957	(results->auth_avail & CURLAUTH_GSSNEGOTIATE)) {
1958	/*
1959	* In auto mode, give Negotiate a chance via
1960	* empty auth before stripping it. If it fails,
1961	* we will strip it on the next 401.
1962	*/
1963	empty_auth_try_negotiate = 1;
1964	} else {
1965	http_auth_methods &= ~CURLAUTH_GSSNEGOTIATE;
1966	}
1967	if (results->auth_avail) {
1968	http_auth_methods &= results->auth_avail;
1969	http_auth_methods_restricted = 1;
1970	}
1971	return HTTP_REAUTH;
1972	}
1973	} else if (results->http_code == 429) {
1974	trace2_data_intmax("http", the_repository, "http/429-retry-after",
1975	results->retry_after);
1976	return HTTP_RATE_LIMITED;
1977	} else {
1978	if (results->http_connectcode == 407)
1979	credential_reject(the_repository, &proxy_auth);
1980	if (!curl_errorstr[0])
1981	strlcpy(curl_errorstr,
1982	curl_easy_strerror(results->curl_result),
1983	sizeof(curl_errorstr));
1984	return HTTP_ERROR;
1985	}
1986	}
1987
1988	int run_one_slot(struct active_request_slot *slot,
1989	struct slot_results *results)
1990	{
1991	slot->results = results;
1992
1993	if (!start_active_slot(slot)) {
1994	xsnprintf(curl_errorstr, sizeof(curl_errorstr),
1995	"failed to start HTTP request");
1996	return HTTP_START_FAILED;
1997	}
1998
1999	run_active_slot(slot);
2000	return handle_curl_result(results);
2001	}
2002
2003	struct curl_slist *http_copy_default_headers(void)
2004	{
2005	struct curl_slist *headers = NULL;
2006	const struct string_list_item *item;
2007
2008	for_each_string_list_item(item, &extra_http_headers)
2009	headers = curl_slist_append(headers, item->string);
2010
2011	return headers;
2012	}
2013
2014	static CURLcode curlinfo_strbuf(CURL curl, CURLINFO info, struct strbuf buf)
2015	{
2016	char *ptr;
2017	CURLcode ret;
2018
2019	strbuf_reset(buf);
2020	ret = curl_easy_getinfo(curl, info, &ptr);
2021	if (!ret && ptr)
2022	strbuf_addstr(buf, ptr);
2023	return ret;
2024	}
2025
2026	/*
2027	* Check for and extract a content-type parameter. "raw"
2028	* should be positioned at the start of the potential
2029	* parameter, with any whitespace already removed.
2030	*
2031	* "name" is the name of the parameter. The value is appended
2032	* to "out".
2033	*/
2034	static int extract_param(const char raw, const char name,
2035	struct strbuf *out)
2036	{
2037	size_t len = strlen(name);
2038
2039	if (strncasecmp(raw, name, len))
2040	return -1;
2041	raw += len;
2042
2043	if (*raw != '=')
2044	return -1;
2045	raw++;
2046
2047	while (raw && !isspace(raw) && *raw != ';')
2048	strbuf_addch(out, *raw++);
2049	return 0;
2050	}
2051
2052	/*
2053	* Extract a normalized version of the content type, with any
2054	* spaces suppressed, all letters lowercased, and no trailing ";"
2055	* or parameters.
2056	*
2057	* Note that we will silently remove even invalid whitespace. For
2058	* example, "text / plain" is specifically forbidden by RFC 2616,
2059	* but "text/plain" is the only reasonable output, and this keeps
2060	* our code simple.
2061	*
2062	* If the "charset" argument is not NULL, store the value of any
2063	* charset parameter there.
2064	*
2065	* Example:
2066	* "TEXT/PLAIN; charset=utf-8" -> "text/plain", "utf-8"
2067	* "text / plain" -> "text/plain"
2068	*/
2069	static void extract_content_type(struct strbuf raw, struct strbuf type,
2070	struct strbuf *charset)
2071	{
2072	const char *p;
2073
2074	strbuf_reset(type);
2075	strbuf_grow(type, raw->len);
2076	for (p = raw->buf; *p; p++) {
2077	if (isspace(*p))
2078	continue;
2079	if (*p == ';') {
2080	p++;
2081	break;
2082	}
2083	strbuf_addch(type, tolower(*p));
2084	}
2085
2086	if (!charset)
2087	return;
2088
2089	strbuf_reset(charset);
2090	while (*p) {
2091	while (isspace(p) \|\| p == ';')
2092	p++;
2093	if (!extract_param(p, "charset", charset))
2094	return;
2095	while (p && !isspace(p))
2096	p++;
2097	}
2098
2099	if (!charset->len && starts_with(type->buf, "text/"))
2100	strbuf_addstr(charset, "ISO-8859-1");
2101	}
2102
2103	static void write_accept_language(struct strbuf *buf)
2104	{
2105	/*
2106	* MAX_DECIMAL_PLACES must not be larger than 3. If it is larger than
2107	* that, q-value will be smaller than 0.001, the minimum q-value the
2108	* HTTP specification allows. See
2109	* https://datatracker.ietf.org/doc/html/rfc7231#section-5.3.1 for q-value.
2110	*/
2111	const int MAX_DECIMAL_PLACES = 3;
2112	const int MAX_LANGUAGE_TAGS = 1000;
2113	const int MAX_ACCEPT_LANGUAGE_HEADER_SIZE = 4000;
2114	char **language_tags = NULL;
2115	int num_langs = 0;
2116	const char *s = get_preferred_languages();
2117	int i;
2118	struct strbuf tag = STRBUF_INIT;
2119
2120	/* Don't add Accept-Language header if no language is preferred. */
2121	if (!s)
2122	return;
2123
2124	/*
2125	* Split the colon-separated string of preferred languages into
2126	* language_tags array.
2127	*/
2128	do {
2129	/* collect language tag */
2130	for (; s && (isalnum(s) \|\| *s == '_'); s++)
2131	strbuf_addch(&tag, s == '_' ? '-' : s);
2132
2133	/* skip .codeset, @modifier and any other unnecessary parts */
2134	while (s && s != ':')
2135	s++;
2136
2137	if (tag.len) {
2138	num_langs++;
2139	REALLOC_ARRAY(language_tags, num_langs);
2140	language_tags[num_langs - 1] = strbuf_detach(&tag, NULL);
2141	if (num_langs >= MAX_LANGUAGE_TAGS - 1) /* -1 for '' /
2142	break;
2143	}
2144	} while (*s++);
2145
2146	/* write Accept-Language header into buf */
2147	if (num_langs) {
2148	int last_buf_len = 0;
2149	int max_q;
2150	int decimal_places;
2151	char q_format[32];
2152
2153	/* add '' /
2154	REALLOC_ARRAY(language_tags, num_langs + 1);
2155	language_tags[num_langs++] = xstrdup("*");
2156
2157	/* compute decimal_places */
2158	for (max_q = 1, decimal_places = 0;
2159	max_q < num_langs && decimal_places <= MAX_DECIMAL_PLACES;
2160	decimal_places++, max_q *= 10)
2161	;
2162
2163	xsnprintf(q_format, sizeof(q_format), ";q=0.%%0%dd", decimal_places);
2164
2165	strbuf_addstr(buf, "Accept-Language: ");
2166
2167	for (i = 0; i < num_langs; i++) {
2168	if (i > 0)
2169	strbuf_addstr(buf, ", ");
2170
2171	strbuf_addstr(buf, language_tags[i]);
2172
2173	if (i > 0)
2174	strbuf_addf(buf, q_format, max_q - i);
2175
2176	if (buf->len > MAX_ACCEPT_LANGUAGE_HEADER_SIZE) {
2177	strbuf_remove(buf, last_buf_len, buf->len - last_buf_len);
2178	break;
2179	}
2180
2181	last_buf_len = buf->len;
2182	}
2183	}
2184
2185	for (i = 0; i < num_langs; i++)
2186	free(language_tags[i]);
2187	free(language_tags);
2188	}
2189
2190	/*
2191	* Get an Accept-Language header which indicates user's preferred languages.
2192	*
2193	* Examples:
2194	* LANGUAGE= -> ""
2195	* LANGUAGE=ko:en -> "Accept-Language: ko, en; q=0.9, *; q=0.1"
2196	* LANGUAGE=ko_KR.UTF-8:sr@latin -> "Accept-Language: ko-KR, sr; q=0.9, *; q=0.1"
2197	* LANGUAGE=ko LANG=en_US.UTF-8 -> "Accept-Language: ko, *; q=0.1"
2198	* LANGUAGE= LANG=en_US.UTF-8 -> "Accept-Language: en-US, *; q=0.1"
2199	* LANGUAGE= LANG=C -> ""
2200	*/
2201	const char *http_get_accept_language_header(void)
2202	{
2203	if (!cached_accept_language) {
2204	struct strbuf buf = STRBUF_INIT;
2205	write_accept_language(&buf);
2206	if (buf.len > 0)
2207	cached_accept_language = strbuf_detach(&buf, NULL);
2208	}
2209
2210	return cached_accept_language;
2211	}
2212
2213	static void http_opt_request_remainder(CURL *curl, off_t pos)
2214	{
2215	char buf[128];
2216	xsnprintf(buf, sizeof(buf), "%"PRIuMAX"-", (uintmax_t)pos);
2217	curl_easy_setopt(curl, CURLOPT_RANGE, buf);
2218	}
2219
2220	/* http_request() targets */
2221	#define HTTP_REQUEST_STRBUF 0
2222	#define HTTP_REQUEST_FILE 1
2223
2224	static int http_request(const char *url,
2225	void *result, int target,
2226	struct http_get_options *options)
2227	{
2228	struct active_request_slot *slot;
2229	struct slot_results results = { .retry_after = -1 };
2230	struct curl_slist *headers = http_copy_default_headers();
2231	struct strbuf buf = STRBUF_INIT;
2232	const char *accept_language;
2233	int ret;
2234
2235	slot = get_active_slot();
2236	curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1L);
2237
2238	if (!result) {
2239	curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1L);
2240	} else {
2241	curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0L);
2242	curl_easy_setopt(slot->curl, CURLOPT_WRITEDATA, result);
2243
2244	if (target == HTTP_REQUEST_FILE) {
2245	off_t posn = ftello(result);
2246	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
2247	fwrite);
2248	if (posn > 0)
2249	http_opt_request_remainder(slot->curl, posn);
2250	} else
2251	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
2252	fwrite_buffer);
2253	}
2254
2255	curl_easy_setopt(slot->curl, CURLOPT_HEADERFUNCTION, fwrite_wwwauth);
2256
2257	accept_language = http_get_accept_language_header();
2258
2259	if (accept_language)
2260	headers = curl_slist_append(headers, accept_language);
2261
2262	strbuf_addstr(&buf, "Pragma:");
2263	if (options->no_cache)
2264	strbuf_addstr(&buf, " no-cache");
2265	if (options->initial_request &&
2266	http_follow_config == HTTP_FOLLOW_INITIAL)
2267	curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 1L);
2268
2269	headers = curl_slist_append(headers, buf.buf);
2270
2271	/* Add additional headers here */
2272	if (options->extra_headers) {
2273	const struct string_list_item *item;
2274	for_each_string_list_item(item, options->extra_headers)
2275	headers = curl_slist_append(headers, item->string);
2276	}
2277
2278	headers = http_append_auth_header(&http_auth, headers);
2279
2280	curl_easy_setopt(slot->curl, CURLOPT_URL, url);
2281	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers);
2282	curl_easy_setopt(slot->curl, CURLOPT_ENCODING, "");
2283	curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 0L);
2284
2285	ret = run_one_slot(slot, &results);
2286
2287	#ifdef GIT_CURL_HAVE_CURLINFO_RETRY_AFTER
2288	if (ret == HTTP_RATE_LIMITED) {
2289	curl_off_t retry_after;
2290	if (curl_easy_getinfo(slot->curl, CURLINFO_RETRY_AFTER,
2291	&retry_after) == CURLE_OK && retry_after > 0)
2292	results.retry_after = (long)retry_after;
2293	}
2294	#endif
2295
2296	options->retry_after = results.retry_after;
2297
2298	if (options->content_type) {
2299	struct strbuf raw = STRBUF_INIT;
2300	curlinfo_strbuf(slot->curl, CURLINFO_CONTENT_TYPE, &raw);
2301	extract_content_type(&raw, options->content_type,
2302	options->charset);
2303	strbuf_release(&raw);
2304	}
2305
2306	if (options->effective_url)
2307	curlinfo_strbuf(slot->curl, CURLINFO_EFFECTIVE_URL,
2308	options->effective_url);
2309
2310	curl_slist_free_all(headers);
2311	strbuf_release(&buf);
2312
2313	return ret;
2314	}
2315
2316	/*
2317	* Update the "base" url to a more appropriate value, as deduced by
2318	* redirects seen when requesting a URL starting with "url".
2319	*
2320	* The "asked" parameter is a URL that we asked curl to access, and must begin
2321	* with "base".
2322	*
2323	* The "got" parameter is the URL that curl reported to us as where we ended
2324	* up.
2325	*
2326	* Returns 1 if we updated the base url, 0 otherwise.
2327	*
2328	* Our basic strategy is to compare "base" and "asked" to find the bits
2329	* specific to our request. We then strip those bits off of "got" to yield the
2330	* new base. So for example, if our base is "http://example.com/foo.git",
2331	* and we ask for "http://example.com/foo.git/info/refs", we might end up
2332	* with "https://other.example.com/foo.git/info/refs". We would want the
2333	* new URL to become "https://other.example.com/foo.git".
2334	*
2335	* Note that this assumes a sane redirect scheme. It's entirely possible
2336	* in the example above to end up at a URL that does not even end in
2337	* "info/refs". In such a case we die. There's not much we can do, such a
2338	* scheme is unlikely to represent a real git repository, and failing to
2339	* rewrite the base opens options for malicious redirects to do funny things.
2340	*/
2341	static int update_url_from_redirect(struct strbuf *base,
2342	const char *asked,
2343	const struct strbuf *got)
2344	{
2345	const char *tail;
2346	size_t new_len;
2347
2348	if (!strcmp(asked, got->buf))
2349	return 0;
2350
2351	if (!skip_prefix(asked, base->buf, &tail))
2352	BUG("update_url_from_redirect: %s is not a superset of %s",
2353	asked, base->buf);
2354
2355	new_len = got->len;
2356	if (!strip_suffix_mem(got->buf, &new_len, tail))
2357	die(_("unable to update url base from redirection:\n"
2358	" asked for: %s\n"
2359	" redirect: %s"),
2360	asked, got->buf);
2361
2362	strbuf_reset(base);
2363	strbuf_add(base, got->buf, new_len);
2364
2365	return 1;
2366	}
2367
2368	/*
2369	* Compute the retry delay for an HTTP 429 response.
2370	* Returns a negative value if configuration is invalid (delay exceeds
2371	* http.maxRetryTime), otherwise returns the delay in seconds (>= 0).
2372	*/
2373	static long handle_rate_limit_retry(long slot_retry_after)
2374	{
2375	/* Use the slot-specific retry_after value or configured default */
2376	if (slot_retry_after >= 0) {
2377	/* Check if retry delay exceeds maximum allowed */
2378	if (slot_retry_after > http_max_retry_time) {
2379	error(_("response requested a delay greater than http.maxRetryTime (%ld > %ld seconds)"),
2380	slot_retry_after, http_max_retry_time);
2381	trace2_data_string("http", the_repository,
2382	"http/429-error", "exceeds-max-retry-time");
2383	trace2_data_intmax("http", the_repository,
2384	"http/429-requested-delay", slot_retry_after);
2385	return -1;
2386	}
2387	return slot_retry_after;
2388	} else {
2389	/* No Retry-After header provided, use configured default */
2390	if (http_retry_after > http_max_retry_time) {
2391	error(_("configured http.retryAfter exceeds http.maxRetryTime (%ld > %ld seconds)"),
2392	http_retry_after, http_max_retry_time);
2393	trace2_data_string("http", the_repository,
2394	"http/429-error", "config-exceeds-max-retry-time");
2395	return -1;
2396	}
2397	trace2_data_string("http", the_repository,
2398	"http/429-retry-source", "config-default");
2399	return http_retry_after;
2400	}
2401	}
2402
2403	static int http_request_recoverable(const char *url,
2404	void *result, int target,
2405	struct http_get_options *options)
2406	{
2407	static struct http_get_options empty_opts;
2408	int i = 3;
2409	int ret;
2410	int rate_limit_retries = http_max_retries;
2411
2412	if (!options)
2413	options = &empty_opts;
2414
2415	if (always_auth_proactively())
2416	credential_fill(the_repository, &http_auth, 1);
2417
2418	ret = http_request(url, result, target, options);
2419
2420	if (ret != HTTP_OK && ret != HTTP_REAUTH && ret != HTTP_RATE_LIMITED)
2421	return ret;
2422
2423	/* If retries are disabled and we got a 429, fail immediately */
2424	if (ret == HTTP_RATE_LIMITED && !http_max_retries)
2425	return HTTP_ERROR;
2426
2427	if (options->effective_url && options->base_url) {
2428	if (update_url_from_redirect(options->base_url,
2429	url, options->effective_url)) {
2430	credential_from_url(&http_auth, options->base_url->buf);
2431	url = options->effective_url->buf;
2432	}
2433	}
2434
2435	while ((ret == HTTP_REAUTH && --i) \|\|
2436	(ret == HTTP_RATE_LIMITED && --rate_limit_retries)) {
2437	long retry_delay = -1;
2438	/*
2439	* The previous request may have put cruft into our output stream; we
2440	* should clear it out before making our next request.
2441	*/
2442	switch (target) {
2443	case HTTP_REQUEST_STRBUF:
2444	strbuf_reset(result);
2445	break;
2446	case HTTP_REQUEST_FILE: {
2447	FILE *f = result;
2448	if (fflush(f)) {
2449	error_errno("unable to flush a file");
2450	return HTTP_START_FAILED;
2451	}
2452	rewind(f);
2453	if (ftruncate(fileno(f), 0) < 0) {
2454	error_errno("unable to truncate a file");
2455	return HTTP_START_FAILED;
2456	}
2457	break;
2458	}
2459	default:
2460	BUG("Unknown http_request target");
2461	}
2462	if (ret == HTTP_RATE_LIMITED) {
2463	retry_delay = handle_rate_limit_retry(options->retry_after);
2464	if (retry_delay < 0)
2465	return HTTP_ERROR;
2466
2467	if (retry_delay > 0) {
2468	warning(_("rate limited, waiting %ld seconds before retry"), retry_delay);
2469	trace2_data_intmax("http", the_repository,
2470	"http/retry-sleep-seconds", retry_delay);
2471	sleep(retry_delay);
2472	}
2473	} else if (ret == HTTP_REAUTH) {
2474	http_reauth_prepare(1);
2475	}
2476
2477	ret = http_request(url, result, target, options);
2478	}
2479	if (ret == HTTP_RATE_LIMITED) {
2480	trace2_data_string("http", the_repository,
2481	"http/429-error", "retries-exhausted");
2482	return HTTP_RATE_LIMITED;
2483	}
2484	return ret;
2485	}
2486
2487	int http_get_strbuf(const char *url,
2488	struct strbuf *result,
2489	struct http_get_options *options)
2490	{
2491	return http_request_recoverable(url, result, HTTP_REQUEST_STRBUF, options);
2492	}
2493
2494	/*
2495	* Downloads a URL and stores the result in the given file.
2496	*
2497	* If a previous interrupted download is detected (i.e. a previous temporary
2498	* file is still around) the download is resumed.
2499	*/
2500	int http_get_file(const char url, const char filename,
2501	struct http_get_options *options)
2502	{
2503	int ret;
2504	struct strbuf tmpfile = STRBUF_INIT;
2505	FILE *result;
2506
2507	strbuf_addf(&tmpfile, "%s.temp", filename);
2508	result = fopen(tmpfile.buf, "a");
2509	if (!result) {
2510	error("Unable to open local file %s", tmpfile.buf);
2511	ret = HTTP_ERROR;
2512	goto cleanup;
2513	}
2514
2515	ret = http_request_recoverable(url, result, HTTP_REQUEST_FILE, options);
2516	fclose(result);
2517
2518	if (ret == HTTP_OK && finalize_object_file(the_repository, tmpfile.buf, filename))
2519	ret = HTTP_ERROR;
2520	cleanup:
2521	strbuf_release(&tmpfile);
2522	return ret;
2523	}
2524
2525	int http_fetch_ref(const char base, struct ref ref)
2526	{
2527	struct http_get_options options = {0};
2528	char *url;
2529	struct strbuf buffer = STRBUF_INIT;
2530	int ret = -1;
2531
2532	options.no_cache = 1;
2533
2534	url = quote_ref_url(base, ref->name);
2535	if (http_get_strbuf(url, &buffer, &options) == HTTP_OK) {
2536	strbuf_rtrim(&buffer);
2537	if (buffer.len == the_hash_algo->hexsz)
2538	ret = get_oid_hex(buffer.buf, &ref->old_oid);
2539	else if (starts_with(buffer.buf, "ref: ")) {
2540	ref->symref = xstrdup(buffer.buf + 5);
2541	ret = 0;
2542	}
2543	}
2544
2545	strbuf_release(&buffer);
2546	free(url);
2547	return ret;
2548	}
2549
2550	/* Helpers for fetching packs */
2551	static char fetch_pack_index(unsigned char hash, const char *base_url)
2552	{
2553	char url, tmp;
2554	struct strbuf buf = STRBUF_INIT;
2555
2556	if (http_is_verbose)
2557	fprintf(stderr, "Getting index for pack %s\n", hash_to_hex(hash));
2558
2559	end_url_with_slash(&buf, base_url);
2560	strbuf_addf(&buf, "objects/pack/pack-%s.idx", hash_to_hex(hash));
2561	url = strbuf_detach(&buf, NULL);
2562
2563	/*
2564	* Don't put this into packs/, since it's just temporary and we don't
2565	* want to confuse it with our local .idx files. We'll generate our
2566	* own index if we choose to download the matching packfile.
2567	*
2568	* It's tempting to use xmks_tempfile() here, but it's important that
2569	* the file not exist, otherwise http_get_file() complains. So we
2570	* create a filename that should be unique, and then just register it
2571	* as a tempfile so that it will get cleaned up on exit.
2572	*
2573	* In theory we could hold on to the tempfile and delete these as soon
2574	* as we download the matching pack, but it would take a bit of
2575	* refactoring. Leaving them until the process ends is probably OK.
2576	*/
2577	tmp = xstrfmt("%s/tmp_pack_%s.idx",
2578	repo_get_object_directory(the_repository),
2579	hash_to_hex(hash));
2580	register_tempfile(tmp);
2581
2582	if (http_get_file(url, tmp, NULL) != HTTP_OK) {
2583	error("Unable to get pack index %s", url);
2584	FREE_AND_NULL(tmp);
2585	}
2586
2587	free(url);
2588	return tmp;
2589	}
2590
2591	static int fetch_and_setup_pack_index(struct packfile_list *packs,
2592	unsigned char *sha1,
2593	const char *base_url)
2594	{
2595	struct packed_git new_pack, p;
2596	char *tmp_idx = NULL;
2597	int ret;
2598
2599	/*
2600	* If we already have the pack locally, no need to fetch its index or
2601	* even add it to list; we already have all of its objects.
2602	*/
2603	repo_for_each_pack(the_repository, p) {
2604	if (hasheq(p->hash, sha1, the_repository->hash_algo))
2605	return 0;
2606	}
2607
2608	tmp_idx = fetch_pack_index(sha1, base_url);
2609	if (!tmp_idx)
2610	return -1;
2611
2612	new_pack = parse_pack_index(the_repository, sha1, tmp_idx);
2613	if (!new_pack) {
2614	free(tmp_idx);
2615	return -1; /* parse_pack_index() already issued error message */
2616	}
2617
2618	ret = verify_pack_index(new_pack);
2619
2620	close_pack_index(new_pack);
2621	free(tmp_idx);
2622	if (ret) {
2623	free(new_pack);
2624	return -1;
2625	}
2626
2627	packfile_list_prepend(packs, new_pack);
2628	return 0;
2629	}
2630
2631	int http_get_info_packs(const char base_url, struct packfile_list packs)
2632	{
2633	struct http_get_options options = {0};
2634	int ret = 0;
2635	char *url;
2636	const char *data;
2637	struct strbuf buf = STRBUF_INIT;
2638	struct object_id oid;
2639
2640	end_url_with_slash(&buf, base_url);
2641	strbuf_addstr(&buf, "objects/info/packs");
2642	url = strbuf_detach(&buf, NULL);
2643
2644	options.no_cache = 1;
2645	ret = http_get_strbuf(url, &buf, &options);
2646	if (ret != HTTP_OK)
2647	goto cleanup;
2648
2649	data = buf.buf;
2650	while (*data) {
2651	if (skip_prefix(data, "P pack-", &data) &&
2652	!parse_oid_hex(data, &oid, &data) &&
2653	skip_prefix(data, ".pack", &data) &&
2654	(data == '\n' \|\| data == '\0')) {
2655	fetch_and_setup_pack_index(packs, oid.hash, base_url);
2656	} else {
2657	data = strchrnul(data, '\n');
2658	}
2659	if (*data)
2660	data++; /* skip past newline */
2661	}
2662
2663	cleanup:
2664	free(url);
2665	strbuf_release(&buf);
2666	return ret;
2667	}
2668
2669	void release_http_pack_request(struct http_pack_request *preq)
2670	{
2671	if (preq->packfile) {
2672	fclose(preq->packfile);
2673	preq->packfile = NULL;
2674	}
2675	preq->slot = NULL;
2676	strbuf_release(&preq->tmpfile);
2677	curl_slist_free_all(preq->headers);
2678	free(preq->url);
2679	free(preq);
2680	}
2681
2682	static const char *default_index_pack_args[] =
2683	{"index-pack", "--stdin", NULL};
2684
2685	int finish_http_pack_request(struct http_pack_request *preq)
2686	{
2687	struct child_process ip = CHILD_PROCESS_INIT;
2688	int tmpfile_fd;
2689	int ret = 0;
2690
2691	fclose(preq->packfile);
2692	preq->packfile = NULL;
2693
2694	tmpfile_fd = xopen(preq->tmpfile.buf, O_RDONLY);
2695
2696	ip.git_cmd = 1;
2697	ip.in = tmpfile_fd;
2698	strvec_pushv(&ip.args, preq->index_pack_args ?
2699	preq->index_pack_args :
2700	default_index_pack_args);
2701
2702	if (preq->preserve_index_pack_stdout)
2703	ip.out = 0;
2704	else
2705	ip.no_stdout = 1;
2706
2707	if (run_command(&ip)) {
2708	ret = -1;
2709	goto cleanup;
2710	}
2711
2712	cleanup:
2713	close(tmpfile_fd);
2714	unlink(preq->tmpfile.buf);
2715	return ret;
2716	}
2717
2718	void http_install_packfile(struct packed_git *p,
2719	struct packfile_list *list_to_remove_from)
2720	{
2721	struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources);
2722	packfile_list_remove(list_to_remove_from, p);
2723	packfile_store_add_pack(files->packed, p);
2724	}
2725
2726	struct http_pack_request *new_http_pack_request(
2727	const unsigned char packed_git_hash, const char base_url) {
2728
2729	struct strbuf buf = STRBUF_INIT;
2730
2731	end_url_with_slash(&buf, base_url);
2732	strbuf_addf(&buf, "objects/pack/pack-%s.pack",
2733	hash_to_hex(packed_git_hash));
2734	return new_direct_http_pack_request(packed_git_hash,
2735	strbuf_detach(&buf, NULL));
2736	}
2737
2738	struct http_pack_request *new_direct_http_pack_request(
2739	const unsigned char packed_git_hash, char url)
2740	{
2741	off_t prev_posn = 0;
2742	struct http_pack_request *preq;
2743
2744	CALLOC_ARRAY(preq, 1);
2745	strbuf_init(&preq->tmpfile, 0);
2746
2747	preq->url = url;
2748
2749	odb_pack_name(the_repository, &preq->tmpfile, packed_git_hash, "pack");
2750	strbuf_addstr(&preq->tmpfile, ".temp");
2751	preq->packfile = fopen(preq->tmpfile.buf, "a");
2752	if (!preq->packfile) {
2753	error("Unable to open local file %s for pack",
2754	preq->tmpfile.buf);
2755	goto abort;
2756	}
2757
2758	preq->slot = get_active_slot();
2759	preq->headers = object_request_headers();
2760	curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEDATA, preq->packfile);
2761	curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
2762	curl_easy_setopt(preq->slot->curl, CURLOPT_URL, preq->url);
2763	curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER, preq->headers);
2764
2765	/*
2766	* If there is data present from a previous transfer attempt,
2767	* resume where it left off
2768	*/
2769	prev_posn = ftello(preq->packfile);
2770	if (prev_posn>0) {
2771	if (http_is_verbose)
2772	fprintf(stderr,
2773	"Resuming fetch of pack %s at byte %"PRIuMAX"\n",
2774	hash_to_hex(packed_git_hash),
2775	(uintmax_t)prev_posn);
2776	http_opt_request_remainder(preq->slot->curl, prev_posn);
2777	}
2778
2779	return preq;
2780
2781	abort:
2782	strbuf_release(&preq->tmpfile);
2783	free(preq->url);
2784	free(preq);
2785	return NULL;
2786	}
2787
2788	/* Helpers for fetching objects (loose) */
2789	static size_t fwrite_sha1_file(char *ptr, size_t eltsize, size_t nmemb,
2790	void *data)
2791	{
2792	unsigned char expn[4096];
2793	size_t size = eltsize * nmemb;
2794	int posn = 0;
2795	struct http_object_request *freq = data;
2796	struct active_request_slot *slot = freq->slot;
2797
2798	if (slot) {
2799	CURLcode c = curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE,
2800	&slot->http_code);
2801	if (c != CURLE_OK)
2802	BUG("curl_easy_getinfo for HTTP code failed: %s",
2803	curl_easy_strerror(c));
2804	if (slot->http_code >= 300)
2805	return nmemb;
2806	}
2807
2808	do {
2809	ssize_t retval = xwrite(freq->localfile,
2810	(char *) ptr + posn, size - posn);
2811	if (retval < 0)
2812	return posn / eltsize;
2813	posn += retval;
2814	} while (posn < size);
2815
2816	freq->stream.avail_in = size;
2817	freq->stream.next_in = (void *)ptr;
2818	do {
2819	freq->stream.next_out = expn;
2820	freq->stream.avail_out = sizeof(expn);
2821	freq->zret = git_inflate(&freq->stream, Z_SYNC_FLUSH);
2822	git_hash_update(&freq->c, expn,
2823	sizeof(expn) - freq->stream.avail_out);
2824	} while (freq->stream.avail_in && freq->zret == Z_OK);
2825	return nmemb;
2826	}
2827
2828	struct http_object_request new_http_object_request(const char base_url,
2829	const struct object_id *oid)
2830	{
2831	struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources);
2832	char *hex = oid_to_hex(oid);
2833	struct strbuf filename = STRBUF_INIT;
2834	struct strbuf prevfile = STRBUF_INIT;
2835	int prevlocal;
2836	char prev_buf[PREV_BUF_SIZE];
2837	ssize_t prev_read = 0;
2838	off_t prev_posn = 0;
2839	struct http_object_request *freq;
2840
2841	CALLOC_ARRAY(freq, 1);
2842	strbuf_init(&freq->tmpfile, 0);
2843	oidcpy(&freq->oid, oid);
2844	freq->localfile = -1;
2845
2846	odb_loose_path(files->loose, &filename, oid);
2847	strbuf_addf(&freq->tmpfile, "%s.temp", filename.buf);
2848
2849	strbuf_addf(&prevfile, "%s.prev", filename.buf);
2850	unlink_or_warn(prevfile.buf);
2851	rename(freq->tmpfile.buf, prevfile.buf);
2852	unlink_or_warn(freq->tmpfile.buf);
2853	strbuf_release(&filename);
2854
2855	if (freq->localfile != -1)
2856	error("fd leakage in start: %d", freq->localfile);
2857	freq->localfile = open(freq->tmpfile.buf,
2858	O_WRONLY \| O_CREAT \| O_EXCL, 0666);
2859	/*
2860	* This could have failed due to the "lazy directory creation";
2861	* try to mkdir the last path component.
2862	*/
2863	if (freq->localfile < 0 && errno == ENOENT) {
2864	char *dir = strrchr(freq->tmpfile.buf, '/');
2865	if (dir) {
2866	*dir = 0;
2867	mkdir(freq->tmpfile.buf, 0777);
2868	*dir = '/';
2869	}
2870	freq->localfile = open(freq->tmpfile.buf,
2871	O_WRONLY \| O_CREAT \| O_EXCL, 0666);
2872	}
2873
2874	if (freq->localfile < 0) {
2875	error_errno("Couldn't create temporary file %s",
2876	freq->tmpfile.buf);
2877	goto abort;
2878	}
2879
2880	git_inflate_init(&freq->stream);
2881
2882	git_hash_init(&freq->c, the_hash_algo);
2883
2884	freq->url = get_remote_object_url(base_url, hex, 0);
2885
2886	/*
2887	* If a previous temp file is present, process what was already
2888	* fetched.
2889	*/
2890	prevlocal = open(prevfile.buf, O_RDONLY);
2891	if (prevlocal != -1) {
2892	do {
2893	prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
2894	if (prev_read>0) {
2895	if (fwrite_sha1_file(prev_buf,
2896	1,
2897	prev_read,
2898	freq) == prev_read) {
2899	prev_posn += prev_read;
2900	} else {
2901	prev_read = -1;
2902	}
2903	}
2904	} while (prev_read > 0);
2905	close(prevlocal);
2906	}
2907	unlink_or_warn(prevfile.buf);
2908	strbuf_release(&prevfile);
2909
2910	/*
2911	* Reset inflate/SHA1 if there was an error reading the previous temp
2912	* file; also rewind to the beginning of the local file.
2913	*/
2914	if (prev_read == -1) {
2915	git_inflate_end(&freq->stream);
2916	memset(&freq->stream, 0, sizeof(freq->stream));
2917	git_inflate_init(&freq->stream);
2918	git_hash_init(&freq->c, the_hash_algo);
2919	if (prev_posn>0) {
2920	prev_posn = 0;
2921	lseek(freq->localfile, 0, SEEK_SET);
2922	if (ftruncate(freq->localfile, 0) < 0) {
2923	error_errno("Couldn't truncate temporary file %s",
2924	freq->tmpfile.buf);
2925	goto abort;
2926	}
2927	}
2928	}
2929
2930	freq->slot = get_active_slot();
2931	freq->headers = object_request_headers();
2932
2933	curl_easy_setopt(freq->slot->curl, CURLOPT_WRITEDATA, freq);
2934	curl_easy_setopt(freq->slot->curl, CURLOPT_FAILONERROR, 0L);
2935	curl_easy_setopt(freq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
2936	curl_easy_setopt(freq->slot->curl, CURLOPT_ERRORBUFFER, freq->errorstr);
2937	curl_easy_setopt(freq->slot->curl, CURLOPT_URL, freq->url);
2938	curl_easy_setopt(freq->slot->curl, CURLOPT_HTTPHEADER, freq->headers);
2939
2940	/*
2941	* If we have successfully processed data from a previous fetch
2942	* attempt, only fetch the data we don't already have.
2943	*/
2944	if (prev_posn>0) {
2945	if (http_is_verbose)
2946	fprintf(stderr,
2947	"Resuming fetch of object %s at byte %"PRIuMAX"\n",
2948	hex, (uintmax_t)prev_posn);
2949	http_opt_request_remainder(freq->slot->curl, prev_posn);
2950	}
2951
2952	return freq;
2953
2954	abort:
2955	strbuf_release(&prevfile);
2956	free(freq->url);
2957	free(freq);
2958	return NULL;
2959	}
2960
2961	void process_http_object_request(struct http_object_request *freq)
2962	{
2963	if (!freq->slot)
2964	return;
2965	freq->curl_result = freq->slot->curl_result;
2966	freq->http_code = freq->slot->http_code;
2967	freq->slot = NULL;
2968	}
2969
2970	int finish_http_object_request(struct http_object_request *freq)
2971	{
2972	struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources);
2973	struct stat st;
2974	struct strbuf filename = STRBUF_INIT;
2975
2976	close(freq->localfile);
2977	freq->localfile = -1;
2978
2979	process_http_object_request(freq);
2980
2981	if (freq->http_code == 416) {
2982	warning("requested range invalid; we may already have all the data.");
2983	} else if (freq->curl_result != CURLE_OK) {
2984	if (stat(freq->tmpfile.buf, &st) == 0)
2985	if (st.st_size == 0)
2986	unlink_or_warn(freq->tmpfile.buf);
2987	return -1;
2988	}
2989
2990	git_hash_final_oid(&freq->real_oid, &freq->c);
2991	if (freq->zret != Z_STREAM_END) {
2992	unlink_or_warn(freq->tmpfile.buf);
2993	return -1;
2994	}
2995	if (!oideq(&freq->oid, &freq->real_oid)) {
2996	unlink_or_warn(freq->tmpfile.buf);
2997	return -1;
2998	}
2999	odb_loose_path(files->loose, &filename, &freq->oid);
3000	freq->rename = finalize_object_file(the_repository, freq->tmpfile.buf, filename.buf);
3001	strbuf_release(&filename);
3002
3003	return freq->rename;
3004	}
3005
3006	void abort_http_object_request(struct http_object_request **freq_p)
3007	{
3008	struct http_object_request freq = freq_p;
3009	unlink_or_warn(freq->tmpfile.buf);
3010
3011	release_http_object_request(freq_p);
3012	}
3013
3014	void release_http_object_request(struct http_object_request **freq_p)
3015	{
3016	struct http_object_request freq = freq_p;
3017	if (freq->localfile != -1) {
3018	close(freq->localfile);
3019	freq->localfile = -1;
3020	}
3021	FREE_AND_NULL(freq->url);
3022	if (freq->slot) {
3023	freq->slot->callback_func = NULL;
3024	freq->slot->callback_data = NULL;
3025	release_active_slot(freq->slot);
3026	freq->slot = NULL;
3027	}
3028	curl_slist_free_all(freq->headers);
3029	strbuf_release(&freq->tmpfile);
3030	git_inflate_end(&freq->stream);
3031	git_hash_discard(&freq->c);
3032
3033	free(freq);
3034	*freq_p = NULL;
3035	}