http.c · leroysheep/LeeRoySheep · siGit Code & Deploy

bisect

bisect jch maint master next seen test todo

c 3,032 lines 81.3 KB

Raw

1	#define USE_THE_REPOSITORY_VARIABLE
2	#define DISABLE_SIGN_COMPARE_WARNINGS
3
4	#include "git-compat-util.h"
5	#include "git-curl-compat.h"
6	#include "environment.h"
7	#include "hex.h"
8	#include "http.h"
9	#include "config.h"
10	#include "pack.h"
11	#include "run-command.h"
12	#include "url.h"
13	#include "urlmatch.h"
14	#include "credential.h"
15	#include "version.h"
16	#include "pkt-line.h"
17	#include "gettext.h"
18	#include "trace.h"
19	#include "transport.h"
20	#include "packfile.h"
21	#include "string-list.h"
22	#include "object-file.h"
23	#include "odb.h"
24	#include "tempfile.h"
25	#include "date.h"
26	#include "trace2.h"
27
28	static struct trace_key trace_curl = TRACE_KEY_INIT(CURL);
29	static int trace_curl_data = 1;
30	static int trace_curl_redact = 1;
31	long int git_curl_ipresolve = CURL_IPRESOLVE_WHATEVER;
32	int active_requests;
33	int http_is_verbose;
34	ssize_t http_post_buffer = 16 * LARGE_PACKET_MAX;
35
36	static int min_curl_sessions = 1;
37	static int curl_session_count;
38	static int max_requests = -1;
39	static CURLM *curlm;
40	static CURL *curl_default;
41
42	#define PREV_BUF_SIZE 4096
43
44	char curl_errorstr[CURL_ERROR_SIZE];
45
46	static int curl_ssl_verify = -1;
47	static int curl_ssl_try;
48	static char *curl_http_version;
49	static char *ssl_cert;
50	static char *ssl_cert_type;
51	static char *ssl_cipherlist;
52	static char *ssl_version;
53	static struct {
54	const char *name;
55	long ssl_version;
56	} sslversions[] = {
57	{ "sslv2", CURL_SSLVERSION_SSLv2 },
58	{ "sslv3", CURL_SSLVERSION_SSLv3 },
59	{ "tlsv1", CURL_SSLVERSION_TLSv1 },
60	{ "tlsv1.0", CURL_SSLVERSION_TLSv1_0 },
61	{ "tlsv1.1", CURL_SSLVERSION_TLSv1_1 },
62	{ "tlsv1.2", CURL_SSLVERSION_TLSv1_2 },
63	{ "tlsv1.3", CURL_SSLVERSION_TLSv1_3 },
64	};
65	static char *ssl_key;
66	static char *ssl_key_type;
67	static char *ssl_capath;
68	static char *curl_no_proxy;
69	static char *ssl_pinnedkey;
70	static char *ssl_cainfo;
71	static long curl_low_speed_limit = -1;
72	static long curl_low_speed_time = -1;
73	static int curl_ftp_no_epsv;
74	static char *curl_http_proxy;
75	static char *http_proxy_authmethod;
76
77	static char *http_proxy_ssl_cert;
78	static char *http_proxy_ssl_key;
79	static char *http_proxy_ssl_ca_info;
80	static struct credential proxy_cert_auth = CREDENTIAL_INIT;
81	static int proxy_ssl_cert_password_required;
82
83	static struct {
84	const char *name;
85	long curlauth_param;
86	} proxy_authmethods[] = {
87	{ "basic", CURLAUTH_BASIC },
88	{ "digest", CURLAUTH_DIGEST },
89	{ "negotiate", CURLAUTH_GSSNEGOTIATE },
90	{ "ntlm", CURLAUTH_NTLM },
91	{ "anyauth", CURLAUTH_ANY },
92	/*
93	* CURLAUTH_DIGEST_IE has no corresponding command-line option in
94	* curl(1) and is not included in CURLAUTH_ANY, so we leave it out
95	* here, too
96	*/
97	};
98	#ifdef CURLGSSAPI_DELEGATION_FLAG
99	static char *curl_deleg;
100	static struct {
101	const char *name;
102	long curl_deleg_param;
103	} curl_deleg_levels[] = {
104	{ "none", CURLGSSAPI_DELEGATION_NONE },
105	{ "policy", CURLGSSAPI_DELEGATION_POLICY_FLAG },
106	{ "always", CURLGSSAPI_DELEGATION_FLAG },
107	};
108	#endif
109
110	static long curl_tcp_keepidle = -1;
111	static long curl_tcp_keepintvl = -1;
112	static long curl_tcp_keepcnt = -1;
113
114	enum proactive_auth {
115	PROACTIVE_AUTH_NONE = 0,
116	PROACTIVE_AUTH_IF_CREDENTIALS,
117	PROACTIVE_AUTH_AUTO,
118	PROACTIVE_AUTH_BASIC,
119	};
120
121	static struct credential proxy_auth = CREDENTIAL_INIT;
122	static const char *curl_proxyuserpwd;
123	static char *curl_cookie_file;
124	static int curl_save_cookies;
125	struct credential http_auth = CREDENTIAL_INIT;
126	static enum proactive_auth http_proactive_auth;
127	static char *user_agent;
128	static int curl_empty_auth = -1;
129
130	enum http_follow_config http_follow_config = HTTP_FOLLOW_INITIAL;
131
132	static struct credential cert_auth = CREDENTIAL_INIT;
133	static int ssl_cert_password_required;
134	static unsigned long http_auth_methods = CURLAUTH_ANY;
135	static int http_auth_methods_restricted;
136	/* Modes for which empty_auth cannot actually help us. */
137	static unsigned long empty_auth_useless =
138	CURLAUTH_BASIC
139	\| CURLAUTH_DIGEST_IE
140	\| CURLAUTH_DIGEST;
141	static int empty_auth_try_negotiate;
142
143	static struct curl_slist *pragma_header;
144	static struct string_list extra_http_headers = STRING_LIST_INIT_DUP;
145
146	static struct curl_slist *host_resolutions;
147
148	static struct active_request_slot *active_queue_head;
149
150	static char *cached_accept_language;
151
152	static char *http_ssl_backend;
153
154	static int http_schannel_check_revoke = 1;
155
156	static long http_retry_after = 0;
157	static long http_max_retries = 0;
158	static long http_max_retry_time = 300;
159
160	/*
161	* With the backend being set to `schannel`, setting sslCAinfo would override
162	* the Certificate Store in cURL v7.60.0 and later, which is not what we want
163	* by default.
164	*/
165	static int http_schannel_use_ssl_cainfo;
166
167	static int always_auth_proactively(void)
168	{
169	return http_proactive_auth != PROACTIVE_AUTH_NONE &&
170	http_proactive_auth != PROACTIVE_AUTH_IF_CREDENTIALS;
171	}
172
173	size_t fread_buffer(char ptr, size_t eltsize, size_t nmemb, void buffer_)
174	{
175	size_t size = eltsize * nmemb;
176	struct buffer *buffer = buffer_;
177
178	if (size > buffer->buf.len - buffer->posn)
179	size = buffer->buf.len - buffer->posn;
180	memcpy(ptr, buffer->buf.buf + buffer->posn, size);
181	buffer->posn += size;
182
183	return size / eltsize;
184	}
185
186	int seek_buffer(void *clientp, curl_off_t offset, int origin)
187	{
188	struct buffer *buffer = clientp;
189
190	if (origin != SEEK_SET)
191	BUG("seek_buffer only handles SEEK_SET");
192	if (offset < 0 \|\| offset >= buffer->buf.len) {
193	error("curl seek would be outside of buffer");
194	return CURL_SEEKFUNC_FAIL;
195	}
196
197	buffer->posn = offset;
198	return CURL_SEEKFUNC_OK;
199	}
200
201	size_t fwrite_buffer(char ptr, size_t eltsize, size_t nmemb, void buffer_)
202	{
203	size_t size = eltsize * nmemb;
204	struct strbuf *buffer = buffer_;
205
206	strbuf_add(buffer, ptr, size);
207	return nmemb;
208	}
209
210	/*
211	* A folded header continuation line starts with any number of spaces or
212	* horizontal tab characters (SP or HTAB) as per RFC 7230 section 3.2.
213	* It is not a continuation line if the line starts with any other character.
214	*/
215	static inline int is_hdr_continuation(const char *ptr, const size_t size)
216	{
217	return size && (ptr == ' ' \|\| ptr == '\t');
218	}
219
220	static size_t fwrite_wwwauth(char ptr, size_t eltsize, size_t nmemb, void p MAYBE_UNUSED)
221	{
222	size_t size = eltsize * nmemb;
223	struct strvec *values = &http_auth.wwwauth_headers;
224	struct strbuf buf = STRBUF_INIT;
225	const char *val;
226	size_t val_len;
227
228	/*
229	* Header lines may not come NULL-terminated from libcurl so we must
230	* limit all scans to the maximum length of the header line, or leverage
231	* strbufs for all operations.
232	*
233	* In addition, it is possible that header values can be split over
234	* multiple lines as per RFC 7230. 'Line folding' has been deprecated
235	* but older servers may still emit them. A continuation header field
236	* value is identified as starting with a space or horizontal tab.
237	*
238	* The formal definition of a header field as given in RFC 7230 is:
239	*
240	* header-field = field-name ":" OWS field-value OWS
241	*
242	* field-name = token
243	* field-value = *( field-content / obs-fold )
244	* field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ]
245	* field-vchar = VCHAR / obs-text
246	*
247	* obs-fold = CRLF 1*( SP / HTAB )
248	* ; obsolete line folding
249	* ; see Section 3.2.4
250	*/
251
252	/* Start of a new WWW-Authenticate header */
253	if (skip_iprefix_mem(ptr, size, "www-authenticate:", &val, &val_len)) {
254	strbuf_add(&buf, val, val_len);
255
256	/*
257	* Strip the CRLF that should be present at the end of each
258	* field as well as any trailing or leading whitespace from the
259	* value.
260	*/
261	strbuf_trim(&buf);
262
263	strvec_push(values, buf.buf);
264	http_auth.header_is_last_match = 1;
265	goto exit;
266	}
267
268	/*
269	* This line could be a continuation of the previously matched header
270	* field. If this is the case then we should append this value to the
271	* end of the previously consumed value.
272	*/
273	if (http_auth.header_is_last_match && is_hdr_continuation(ptr, size)) {
274	/*
275	* Trim the CRLF and any leading or trailing from this line.
276	*/
277	strbuf_add(&buf, ptr, size);
278	strbuf_trim(&buf);
279
280	/*
281	* At this point we should always have at least one existing
282	* value, even if it is empty. Do not bother appending the new
283	* value if this continuation header is itself empty.
284	*/
285	if (!values->nr) {
286	BUG("should have at least one existing header value");
287	} else if (buf.len) {
288	char *prev = xstrdup(values->v[values->nr - 1]);
289
290	/* Join two non-empty values with a single space. */
291	const char const sp = prev ? " " : "";
292
293	strvec_pop(values);
294	strvec_pushf(values, "%s%s%s", prev, sp, buf.buf);
295	free(prev);
296	}
297
298	goto exit;
299	}
300
301	/* Not a continuation of a previously matched auth header line. */
302	http_auth.header_is_last_match = 0;
303
304	/*
305	* If this is a HTTP status line and not a header field, this signals
306	* a different HTTP response. libcurl writes all the output of all
307	* response headers of all responses, including redirects.
308	* We only care about the last HTTP request response's headers so clear
309	* the existing array.
310	*/
311	if (skip_iprefix_mem(ptr, size, "http/", &val, &val_len))
312	strvec_clear(values);
313
314	exit:
315	strbuf_release(&buf);
316	return size;
317	}
318
319	size_t fwrite_null(char *ptr UNUSED, size_t eltsize UNUSED, size_t nmemb,
320	void *data UNUSED)
321	{
322	return nmemb;
323	}
324
325	static struct curl_slist *object_request_headers(void)
326	{
327	return curl_slist_append(http_copy_default_headers(), "Pragma:");
328	}
329
330	static void closedown_active_slot(struct active_request_slot *slot)
331	{
332	active_requests--;
333	slot->in_use = 0;
334	}
335
336	static void finish_active_slot(struct active_request_slot *slot)
337	{
338	closedown_active_slot(slot);
339	curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code);
340
341	if (slot->finished)
342	(*slot->finished) = 1;
343
344	/* Store slot results so they can be read after the slot is reused */
345	if (slot->results) {
346	slot->results->curl_result = slot->curl_result;
347	slot->results->http_code = slot->http_code;
348	curl_easy_getinfo(slot->curl, CURLINFO_HTTPAUTH_AVAIL,
349	&slot->results->auth_avail);
350
351	curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CONNECTCODE,
352	&slot->results->http_connectcode);
353	}
354
355	/* Run callback if appropriate */
356	if (slot->callback_func)
357	slot->callback_func(slot->callback_data);
358	}
359
360	static void xmulti_remove_handle(struct active_request_slot *slot)
361	{
362	curl_multi_remove_handle(curlm, slot->curl);
363	}
364
365	static void process_curl_messages(void)
366	{
367	int num_messages;
368	struct active_request_slot *slot;
369	CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
370
371	while (curl_message != NULL) {
372	if (curl_message->msg == CURLMSG_DONE) {
373	int curl_result = curl_message->data.result;
374	slot = active_queue_head;
375	while (slot != NULL &&
376	slot->curl != curl_message->easy_handle)
377	slot = slot->next;
378	if (slot) {
379	xmulti_remove_handle(slot);
380	slot->curl_result = curl_result;
381	finish_active_slot(slot);
382	} else {
383	fprintf(stderr, "Received DONE message for unknown request!\n");
384	}
385	} else {
386	fprintf(stderr, "Unknown CURL message received: %d\n",
387	(int)curl_message->msg);
388	}
389	curl_message = curl_multi_info_read(curlm, &num_messages);
390	}
391	}
392
393	static int http_options(const char var, const char value,
394	const struct config_context ctx, void data)
395	{
396	if (!strcmp("http.version", var)) {
397	return git_config_string(&curl_http_version, var, value);
398	}
399	if (!strcmp("http.sslverify", var)) {
400	curl_ssl_verify = git_config_bool(var, value);
401	return 0;
402	}
403	if (!strcmp("http.sslcipherlist", var))
404	return git_config_string(&ssl_cipherlist, var, value);
405	if (!strcmp("http.sslversion", var))
406	return git_config_string(&ssl_version, var, value);
407	if (!strcmp("http.sslcert", var))
408	return git_config_pathname(&ssl_cert, var, value);
409	if (!strcmp("http.sslcerttype", var))
410	return git_config_string(&ssl_cert_type, var, value);
411	if (!strcmp("http.sslkey", var))
412	return git_config_pathname(&ssl_key, var, value);
413	if (!strcmp("http.sslkeytype", var))
414	return git_config_string(&ssl_key_type, var, value);
415	if (!strcmp("http.sslcapath", var))
416	return git_config_pathname(&ssl_capath, var, value);
417	if (!strcmp("http.sslcainfo", var))
418	return git_config_pathname(&ssl_cainfo, var, value);
419	if (!strcmp("http.sslcertpasswordprotected", var)) {
420	ssl_cert_password_required = git_config_bool(var, value);
421	return 0;
422	}
423	if (!strcmp("http.ssltry", var)) {
424	curl_ssl_try = git_config_bool(var, value);
425	return 0;
426	}
427	if (!strcmp("http.sslbackend", var)) {
428	free(http_ssl_backend);
429	http_ssl_backend = xstrdup_or_null(value);
430	return 0;
431	}
432
433	if (!strcmp("http.schannelcheckrevoke", var)) {
434	http_schannel_check_revoke = git_config_bool(var, value);
435	return 0;
436	}
437
438	if (!strcmp("http.schannelusesslcainfo", var)) {
439	http_schannel_use_ssl_cainfo = git_config_bool(var, value);
440	return 0;
441	}
442
443	if (!strcmp("http.minsessions", var)) {
444	min_curl_sessions = git_config_int(var, value, ctx->kvi);
445	if (min_curl_sessions > 1)
446	min_curl_sessions = 1;
447	return 0;
448	}
449	if (!strcmp("http.maxrequests", var)) {
450	max_requests = git_config_int(var, value, ctx->kvi);
451	return 0;
452	}
453	if (!strcmp("http.lowspeedlimit", var)) {
454	curl_low_speed_limit = git_config_int(var, value, ctx->kvi);
455	return 0;
456	}
457	if (!strcmp("http.lowspeedtime", var)) {
458	curl_low_speed_time = git_config_int(var, value, ctx->kvi);
459	return 0;
460	}
461
462	if (!strcmp("http.noepsv", var)) {
463	curl_ftp_no_epsv = git_config_bool(var, value);
464	return 0;
465	}
466	if (!strcmp("http.proxy", var))
467	return git_config_string(&curl_http_proxy, var, value);
468
469	if (!strcmp("http.proxyauthmethod", var))
470	return git_config_string(&http_proxy_authmethod, var, value);
471
472	if (!strcmp("http.proxysslcert", var))
473	return git_config_string(&http_proxy_ssl_cert, var, value);
474
475	if (!strcmp("http.proxysslkey", var))
476	return git_config_string(&http_proxy_ssl_key, var, value);
477
478	if (!strcmp("http.proxysslcainfo", var))
479	return git_config_string(&http_proxy_ssl_ca_info, var, value);
480
481	if (!strcmp("http.proxysslcertpasswordprotected", var)) {
482	proxy_ssl_cert_password_required = git_config_bool(var, value);
483	return 0;
484	}
485
486	if (!strcmp("http.cookiefile", var))
487	return git_config_pathname(&curl_cookie_file, var, value);
488	if (!strcmp("http.savecookies", var)) {
489	curl_save_cookies = git_config_bool(var, value);
490	return 0;
491	}
492
493	if (!strcmp("http.postbuffer", var)) {
494	http_post_buffer = git_config_ssize_t(var, value, ctx->kvi);
495	if (http_post_buffer < 0)
496	warning(_("negative value for http.postBuffer; defaulting to %d"), LARGE_PACKET_MAX);
497	if (http_post_buffer < LARGE_PACKET_MAX)
498	http_post_buffer = LARGE_PACKET_MAX;
499	return 0;
500	}
501
502	if (!strcmp("http.useragent", var))
503	return git_config_string(&user_agent, var, value);
504
505	if (!strcmp("http.emptyauth", var)) {
506	if (value && !strcmp("auto", value))
507	curl_empty_auth = -1;
508	else
509	curl_empty_auth = git_config_bool(var, value);
510	return 0;
511	}
512
513	if (!strcmp("http.delegation", var)) {
514	#ifdef CURLGSSAPI_DELEGATION_FLAG
515	return git_config_string(&curl_deleg, var, value);
516	#else
517	warning(_("Delegation control is not supported with cURL < 7.22.0"));
518	return 0;
519	#endif
520	}
521
522	if (!strcmp("http.pinnedpubkey", var)) {
523	return git_config_pathname(&ssl_pinnedkey, var, value);
524	}
525
526	if (!strcmp("http.extraheader", var)) {
527	if (!value) {
528	return config_error_nonbool(var);
529	} else if (!*value) {
530	string_list_clear(&extra_http_headers, 0);
531	} else {
532	string_list_append(&extra_http_headers, value);
533	}
534	return 0;
535	}
536
537	if (!strcmp("http.curloptresolve", var)) {
538	if (!value) {
539	return config_error_nonbool(var);
540	} else if (!*value) {
541	curl_slist_free_all(host_resolutions);
542	host_resolutions = NULL;
543	} else {
544	host_resolutions = curl_slist_append(host_resolutions, value);
545	}
546	return 0;
547	}
548
549	if (!strcmp("http.followredirects", var)) {
550	if (value && !strcmp(value, "initial"))
551	http_follow_config = HTTP_FOLLOW_INITIAL;
552	else if (git_config_bool(var, value))
553	http_follow_config = HTTP_FOLLOW_ALWAYS;
554	else
555	http_follow_config = HTTP_FOLLOW_NONE;
556	return 0;
557	}
558
559	if (!strcmp("http.proactiveauth", var)) {
560	if (!value)
561	return config_error_nonbool(var);
562	if (!strcmp(value, "auto"))
563	http_proactive_auth = PROACTIVE_AUTH_AUTO;
564	else if (!strcmp(value, "basic"))
565	http_proactive_auth = PROACTIVE_AUTH_BASIC;
566	else if (!strcmp(value, "none"))
567	http_proactive_auth = PROACTIVE_AUTH_NONE;
568	else
569	warning(_("Unknown value for http.proactiveauth"));
570	return 0;
571	}
572
573	if (!strcmp("http.keepaliveidle", var)) {
574	curl_tcp_keepidle = git_config_int(var, value, ctx->kvi);
575	return 0;
576	}
577	if (!strcmp("http.keepaliveinterval", var)) {
578	curl_tcp_keepintvl = git_config_int(var, value, ctx->kvi);
579	return 0;
580	}
581	if (!strcmp("http.keepalivecount", var)) {
582	curl_tcp_keepcnt = git_config_int(var, value, ctx->kvi);
583	return 0;
584	}
585
586	if (!strcmp("http.retryafter", var)) {
587	http_retry_after = git_config_int(var, value, ctx->kvi);
588	return 0;
589	}
590
591	if (!strcmp("http.maxretries", var)) {
592	http_max_retries = git_config_int(var, value, ctx->kvi);
593	return 0;
594	}
595
596	if (!strcmp("http.maxretrytime", var)) {
597	http_max_retry_time = git_config_int(var, value, ctx->kvi);
598	return 0;
599	}
600
601	/* Fall back on the default ones */
602	return git_default_config(var, value, ctx, data);
603	}
604
605	static int curl_empty_auth_enabled(void)
606	{
607	if (curl_empty_auth >= 0)
608	return curl_empty_auth;
609
610	/*
611	* In the automatic case, kick in the empty-auth
612	* hack as long as we would potentially try some
613	* method more exotic than "Basic" or "Digest".
614	*
615	* But only do this when this is our second or
616	* subsequent request, as by then we know what
617	* methods are available.
618	*/
619	if (http_auth_methods_restricted &&
620	(http_auth_methods & ~empty_auth_useless))
621	return 1;
622	return 0;
623	}
624
625	struct curl_slist http_append_auth_header(const struct credential c,
626	struct curl_slist *headers)
627	{
628	if (c->authtype && c->credential) {
629	struct strbuf auth = STRBUF_INIT;
630	strbuf_addf(&auth, "Authorization: %s %s",
631	c->authtype, c->credential);
632	headers = curl_slist_append(headers, auth.buf);
633	strbuf_release(&auth);
634	}
635	return headers;
636	}
637
638	static void init_curl_http_auth(CURL *result)
639	{
640	if ((!http_auth.username \|\| !*http_auth.username) &&
641	(!http_auth.credential \|\| !*http_auth.credential)) {
642	if (!always_auth_proactively() && curl_empty_auth_enabled()) {
643	curl_easy_setopt(result, CURLOPT_USERPWD, ":");
644	return;
645	} else if (!always_auth_proactively()) {
646	return;
647	} else if (http_proactive_auth == PROACTIVE_AUTH_BASIC) {
648	strvec_push(&http_auth.wwwauth_headers, "Basic");
649	}
650	}
651
652	credential_fill(the_repository, &http_auth, 1);
653
654	if (http_auth.password) {
655	if (always_auth_proactively()) {
656	/*
657	* We got a credential without an authtype and we don't
658	* know what's available. Since our only two options at
659	* the moment are auto (which defaults to basic) and
660	* basic, use basic for now.
661	*/
662	curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
663	}
664	curl_easy_setopt(result, CURLOPT_USERNAME, http_auth.username);
665	curl_easy_setopt(result, CURLOPT_PASSWORD, http_auth.password);
666	}
667	}
668
669	void http_reauth_prepare(int all_capabilities)
670	{
671	/*
672	* If we deferred stripping Negotiate to give empty auth a
673	* chance (auto mode), skip credential_fill on this retry so
674	* that init_curl_http_auth() sends empty credentials and
675	* libcurl can attempt Negotiate with the system ticket cache.
676	*/
677	if (empty_auth_try_negotiate &&
678	!http_auth.password && !http_auth.credential &&
679	(http_auth_methods & CURLAUTH_GSSNEGOTIATE))
680	return;
681
682	credential_fill(the_repository, &http_auth, all_capabilities);
683	}
684
685	/* var must be free-able /
686	static void var_override(char *var, char value)
687	{
688	if (value) {
689	free(*var);
690	*var = xstrdup(value);
691	}
692	}
693
694	static void set_proxyauth_name_password(CURL *result)
695	{
696	if (proxy_auth.password) {
697	curl_easy_setopt(result, CURLOPT_PROXYUSERNAME,
698	proxy_auth.username);
699	curl_easy_setopt(result, CURLOPT_PROXYPASSWORD,
700	proxy_auth.password);
701	} else if (proxy_auth.authtype && proxy_auth.credential) {
702	curl_easy_setopt(result, CURLOPT_PROXYHEADER,
703	http_append_auth_header(&proxy_auth, NULL));
704	}
705	}
706
707	static void init_curl_proxy_auth(CURL *result)
708	{
709	if (proxy_auth.username) {
710	if (!proxy_auth.password && !proxy_auth.credential)
711	credential_fill(the_repository, &proxy_auth, 1);
712	set_proxyauth_name_password(result);
713	}
714
715	var_override(&http_proxy_authmethod, getenv("GIT_HTTP_PROXY_AUTHMETHOD"));
716
717	if (http_proxy_authmethod) {
718	int i;
719	for (i = 0; i < ARRAY_SIZE(proxy_authmethods); i++) {
720	if (!strcmp(http_proxy_authmethod, proxy_authmethods[i].name)) {
721	curl_easy_setopt(result, CURLOPT_PROXYAUTH,
722	proxy_authmethods[i].curlauth_param);
723	break;
724	}
725	}
726	if (i == ARRAY_SIZE(proxy_authmethods)) {
727	warning("unsupported proxy authentication method %s: using anyauth",
728	http_proxy_authmethod);
729	curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
730	}
731	}
732	else
733	curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
734	}
735
736	static int has_cert_password(void)
737	{
738	if (ssl_cert == NULL \|\| ssl_cert_password_required != 1)
739	return 0;
740	if (!cert_auth.password) {
741	cert_auth.protocol = xstrdup("cert");
742	cert_auth.host = xstrdup("");
743	cert_auth.username = xstrdup("");
744	cert_auth.path = xstrdup(ssl_cert);
745	credential_fill(the_repository, &cert_auth, 0);
746	}
747	return 1;
748	}
749
750	static int has_proxy_cert_password(void)
751	{
752	if (http_proxy_ssl_cert == NULL \|\| proxy_ssl_cert_password_required != 1)
753	return 0;
754	if (!proxy_cert_auth.password) {
755	proxy_cert_auth.protocol = xstrdup("cert");
756	proxy_cert_auth.host = xstrdup("");
757	proxy_cert_auth.username = xstrdup("");
758	proxy_cert_auth.path = xstrdup(http_proxy_ssl_cert);
759	credential_fill(the_repository, &proxy_cert_auth, 0);
760	}
761	return 1;
762	}
763
764	static const struct socks_proxy_type {
765	const char *name;
766	long curlsym;
767	} socks_proxy_types[] = {
768	{ "socks", CURLPROXY_SOCKS4 },
769	{ "socks4", CURLPROXY_SOCKS4 },
770	{ "socks4a", CURLPROXY_SOCKS4A },
771	{ "socks5", CURLPROXY_SOCKS5 },
772	{ "socks5h", CURLPROXY_SOCKS5_HOSTNAME },
773	};
774
775	static const struct socks_proxy_type find_socks_proxy_type(const char protocol)
776	{
777	int i;
778
779	if (!protocol)
780	return NULL;
781
782	for (i = 0; i < ARRAY_SIZE(socks_proxy_types); i++) {
783	if (!strcmp(socks_proxy_types[i].name, protocol))
784	return &socks_proxy_types[i];
785	}
786
787	return NULL;
788	}
789
790	static int is_socks_proxy_protocol(const char *protocol)
791	{
792	return !!find_socks_proxy_type(protocol);
793	}
794
795	static int set_curl_proxy_type(CURL result, const char protocol)
796	{
797	const struct socks_proxy_type *socks_proxy_type;
798
799	if (!protocol \|\| !strcmp(protocol, "http"))
800	return 0;
801
802	socks_proxy_type = find_socks_proxy_type(protocol);
803	if (socks_proxy_type) {
804	curl_easy_setopt(result, CURLOPT_PROXYTYPE, socks_proxy_type->curlsym);
805	return 0;
806	}
807
808	if (!strcmp(protocol, "https")) {
809	curl_easy_setopt(result, CURLOPT_PROXYTYPE, (long)CURLPROXY_HTTPS);
810
811	if (http_proxy_ssl_cert)
812	curl_easy_setopt(result, CURLOPT_PROXY_SSLCERT,
813	http_proxy_ssl_cert);
814
815	if (http_proxy_ssl_key)
816	curl_easy_setopt(result, CURLOPT_PROXY_SSLKEY,
817	http_proxy_ssl_key);
818
819	if (has_proxy_cert_password())
820	curl_easy_setopt(result, CURLOPT_PROXY_KEYPASSWD,
821	proxy_cert_auth.password);
822	}
823
824	return -1;
825	}
826
827	/* Return 1 if redactions have been made, 0 otherwise. */
828	static int redact_sensitive_header(struct strbuf *header, size_t offset)
829	{
830	int ret = 0;
831	char *sensitive_header;
832
833	if (trace_curl_redact &&
834	(skip_iprefix(header->buf + offset, "Authorization:", &sensitive_header) \|\|
835	skip_iprefix(header->buf + offset, "Proxy-Authorization:", &sensitive_header))) {
836	/* The first token is the type, which is OK to log */
837	while (isspace(*sensitive_header))
838	sensitive_header++;
839	while (sensitive_header && !isspace(sensitive_header))
840	sensitive_header++;
841	/* Everything else is opaque and possibly sensitive */
842	strbuf_setlen(header, sensitive_header - header->buf);
843	strbuf_addstr(header, " <redacted>");
844	ret = 1;
845	} else if (trace_curl_redact &&
846	skip_iprefix(header->buf + offset, "Cookie:", &sensitive_header)) {
847	struct strbuf redacted_header = STRBUF_INIT;
848	char *cookie;
849
850	while (isspace(*sensitive_header))
851	sensitive_header++;
852
853	cookie = sensitive_header;
854
855	while (cookie) {
856	char *equals;
857	char *semicolon = strstr(cookie, "; ");
858	if (semicolon)
859	*semicolon = 0;
860	equals = strchrnul(cookie, '=');
861	if (!equals) {
862	/* invalid cookie, just append and continue */
863	strbuf_addstr(&redacted_header, cookie);
864	continue;
865	}
866	strbuf_add(&redacted_header, cookie, equals - cookie);
867	strbuf_addstr(&redacted_header, "=<redacted>");
868	if (semicolon) {
869	/*
870	* There are more cookies. (Or, for some
871	* reason, the input string ends in "; ".)
872	*/
873	strbuf_addstr(&redacted_header, "; ");
874	cookie = semicolon + strlen("; ");
875	} else {
876	cookie = NULL;
877	}
878	}
879
880	strbuf_setlen(header, sensitive_header - header->buf);
881	strbuf_addbuf(header, &redacted_header);
882	strbuf_release(&redacted_header);
883	ret = 1;
884	}
885	return ret;
886	}
887
888	static int match_curl_h2_trace(const char line, const char *out)
889	{
890	const char *p;
891
892	/*
893	* curl prior to 8.1.0 gives us:
894	*
895	* h2h3 [<header-name>: <header-val>]
896	*
897	* Starting in 8.1.0, the first token became just "h2".
898	*/
899	if (skip_iprefix(line, "h2h3 [", out) \|\|
900	skip_iprefix(line, "h2 [", out))
901	return 1;
902
903	/*
904	* curl 8.3.0 uses:
905	* [HTTP/2] [<stream-id>] [<header-name>: <header-val>]
906	* where <stream-id> is numeric.
907	*/
908	if (skip_iprefix(line, "[HTTP/2] [", &p)) {
909	while (isdigit(*p))
910	p++;
911	if (skip_prefix(p, "] [", out))
912	return 1;
913	}
914
915	return 0;
916	}
917
918	/* Redact headers in info */
919	static void redact_sensitive_info_header(struct strbuf *header)
920	{
921	const char *sensitive_header;
922
923	if (trace_curl_redact &&
924	match_curl_h2_trace(header->buf, &sensitive_header)) {
925	if (redact_sensitive_header(header, sensitive_header - header->buf)) {
926	/* redaction ate our closing bracket */
927	strbuf_addch(header, ']');
928	}
929	}
930	}
931
932	static void curl_dump_header(const char text, unsigned char ptr, size_t size, int hide_sensitive_header)
933	{
934	struct strbuf out = STRBUF_INIT;
935	struct strbuf headers, header;
936
937	strbuf_addf(&out, "%s, %10.10ld bytes (0x%8.8lx)\n",
938	text, (long)size, (long)size);
939	trace_strbuf(&trace_curl, &out);
940	strbuf_reset(&out);
941	strbuf_add(&out, ptr, size);
942	headers = strbuf_split_max(&out, '\n', 0);
943
944	for (header = headers; *header; header++) {
945	if (hide_sensitive_header)
946	redact_sensitive_header(*header, 0);
947	strbuf_insertstr((*header), 0, text);
948	strbuf_insertstr((*header), strlen(text), ": ");
949	strbuf_rtrim((*header));
950	strbuf_addch((*header), '\n');
951	trace_strbuf(&trace_curl, (*header));
952	}
953	strbuf_list_free(headers);
954	strbuf_release(&out);
955	}
956
957	static void curl_dump_data(const char text, unsigned char ptr, size_t size)
958	{
959	size_t i;
960	struct strbuf out = STRBUF_INIT;
961	unsigned int width = 60;
962
963	strbuf_addf(&out, "%s, %10.10ld bytes (0x%8.8lx)\n",
964	text, (long)size, (long)size);
965	trace_strbuf(&trace_curl, &out);
966
967	for (i = 0; i < size; i += width) {
968	size_t w;
969
970	strbuf_reset(&out);
971	strbuf_addf(&out, "%s: ", text);
972	for (w = 0; (w < width) && (i + w < size); w++) {
973	unsigned char ch = ptr[i + w];
974
975	strbuf_addch(&out,
976	(ch >= 0x20) && (ch < 0x80)
977	? ch : '.');
978	}
979	strbuf_addch(&out, '\n');
980	trace_strbuf(&trace_curl, &out);
981	}
982	strbuf_release(&out);
983	}
984
985	static void curl_dump_info(char *data, size_t size)
986	{
987	struct strbuf buf = STRBUF_INIT;
988
989	strbuf_add(&buf, data, size);
990
991	redact_sensitive_info_header(&buf);
992	trace_printf_key(&trace_curl, "== Info: %s", buf.buf);
993
994	strbuf_release(&buf);
995	}
996
997	static int curl_trace(CURL *handle UNUSED, curl_infotype type,
998	char *data, size_t size,
999	void *userp UNUSED)
1000	{
1001	const char *text;
1002	enum { NO_FILTER = 0, DO_FILTER = 1 };
1003
1004	switch (type) {
1005	case CURLINFO_TEXT:
1006	curl_dump_info(data, size);
1007	break;
1008	case CURLINFO_HEADER_OUT:
1009	text = "=> Send header";
1010	curl_dump_header(text, (unsigned char *)data, size, DO_FILTER);
1011	break;
1012	case CURLINFO_DATA_OUT:
1013	if (trace_curl_data) {
1014	text = "=> Send data";
1015	curl_dump_data(text, (unsigned char *)data, size);
1016	}
1017	break;
1018	case CURLINFO_SSL_DATA_OUT:
1019	if (trace_curl_data) {
1020	text = "=> Send SSL data";
1021	curl_dump_data(text, (unsigned char *)data, size);
1022	}
1023	break;
1024	case CURLINFO_HEADER_IN:
1025	text = "<= Recv header";
1026	curl_dump_header(text, (unsigned char *)data, size, NO_FILTER);
1027	break;
1028	case CURLINFO_DATA_IN:
1029	if (trace_curl_data) {
1030	text = "<= Recv data";
1031	curl_dump_data(text, (unsigned char *)data, size);
1032	}
1033	break;
1034	case CURLINFO_SSL_DATA_IN:
1035	if (trace_curl_data) {
1036	text = "<= Recv SSL data";
1037	curl_dump_data(text, (unsigned char *)data, size);
1038	}
1039	break;
1040
1041	default: /* we ignore unknown types by default */
1042	return 0;
1043	}
1044	return 0;
1045	}
1046
1047	void http_trace_curl_no_data(void)
1048	{
1049	trace_override_envvar(&trace_curl, "1");
1050	trace_curl_data = 0;
1051	}
1052
1053	void setup_curl_trace(CURL *handle)
1054	{
1055	if (!trace_want(&trace_curl))
1056	return;
1057	curl_easy_setopt(handle, CURLOPT_VERBOSE, 1L);
1058	curl_easy_setopt(handle, CURLOPT_DEBUGFUNCTION, curl_trace);
1059	curl_easy_setopt(handle, CURLOPT_DEBUGDATA, NULL);
1060	}
1061
1062	static void proto_list_append(struct strbuf list, const char proto)
1063	{
1064	if (!list)
1065	return;
1066	if (list->len)
1067	strbuf_addch(list, ',');
1068	strbuf_addstr(list, proto);
1069	}
1070
1071	static long get_curl_allowed_protocols(int from_user, struct strbuf *list)
1072	{
1073	long bits = 0;
1074
1075	if (is_transport_allowed("http", from_user)) {
1076	bits \|= CURLPROTO_HTTP;
1077	proto_list_append(list, "http");
1078	}
1079	if (is_transport_allowed("https", from_user)) {
1080	bits \|= CURLPROTO_HTTPS;
1081	proto_list_append(list, "https");
1082	}
1083	if (is_transport_allowed("ftp", from_user)) {
1084	bits \|= CURLPROTO_FTP;
1085	proto_list_append(list, "ftp");
1086	}
1087	if (is_transport_allowed("ftps", from_user)) {
1088	bits \|= CURLPROTO_FTPS;
1089	proto_list_append(list, "ftps");
1090	}
1091
1092	return bits;
1093	}
1094
1095	static int get_curl_http_version_opt(const char version_string, long opt)
1096	{
1097	int i;
1098	static struct {
1099	const char *name;
1100	long opt_token;
1101	} choice[] = {
1102	{ "HTTP/1.1", CURL_HTTP_VERSION_1_1 },
1103	{ "HTTP/2", CURL_HTTP_VERSION_2 }
1104	};
1105
1106	for (i = 0; i < ARRAY_SIZE(choice); i++) {
1107	if (!strcmp(version_string, choice[i].name)) {
1108	*opt = choice[i].opt_token;
1109	return 0;
1110	}
1111	}
1112
1113	warning("unknown value given to http.version: '%s'", version_string);
1114	return -1; /* not found */
1115	}
1116
1117	static CURL *get_curl_handle(void)
1118	{
1119	CURL *result = curl_easy_init();
1120
1121	if (!result)
1122	die("curl_easy_init failed");
1123
1124	if (!curl_ssl_verify) {
1125	curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 0L);
1126	curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 0L);
1127	} else {
1128	/* Verify authenticity of the peer's certificate */
1129	curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 1L);
1130	/* The name in the cert must match whom we tried to connect */
1131	curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 2L);
1132	}
1133
1134	if (curl_http_version) {
1135	long opt;
1136	if (!get_curl_http_version_opt(curl_http_version, &opt)) {
1137	/* Set request use http version */
1138	curl_easy_setopt(result, CURLOPT_HTTP_VERSION, opt);
1139	}
1140	}
1141
1142	curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
1143	curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
1144
1145	#ifdef CURLGSSAPI_DELEGATION_FLAG
1146	if (curl_deleg) {
1147	int i;
1148	for (i = 0; i < ARRAY_SIZE(curl_deleg_levels); i++) {
1149	if (!strcmp(curl_deleg, curl_deleg_levels[i].name)) {
1150	curl_easy_setopt(result, CURLOPT_GSSAPI_DELEGATION,
1151	curl_deleg_levels[i].curl_deleg_param);
1152	break;
1153	}
1154	}
1155	if (i == ARRAY_SIZE(curl_deleg_levels))
1156	warning("Unknown delegation method '%s': using default",
1157	curl_deleg);
1158	}
1159	#endif
1160
1161	if (http_ssl_backend && !strcmp("schannel", http_ssl_backend) &&
1162	!http_schannel_check_revoke) {
1163	curl_easy_setopt(result, CURLOPT_SSL_OPTIONS, (long)CURLSSLOPT_NO_REVOKE);
1164	}
1165
1166	if (http_proactive_auth != PROACTIVE_AUTH_NONE)
1167	init_curl_http_auth(result);
1168
1169	if (getenv("GIT_SSL_VERSION"))
1170	ssl_version = getenv("GIT_SSL_VERSION");
1171	if (ssl_version && *ssl_version) {
1172	int i;
1173	for (i = 0; i < ARRAY_SIZE(sslversions); i++) {
1174	if (!strcmp(ssl_version, sslversions[i].name)) {
1175	curl_easy_setopt(result, CURLOPT_SSLVERSION,
1176	sslversions[i].ssl_version);
1177	break;
1178	}
1179	}
1180	if (i == ARRAY_SIZE(sslversions))
1181	warning("unsupported ssl version %s: using default",
1182	ssl_version);
1183	}
1184
1185	if (getenv("GIT_SSL_CIPHER_LIST"))
1186	ssl_cipherlist = getenv("GIT_SSL_CIPHER_LIST");
1187	if (ssl_cipherlist != NULL && *ssl_cipherlist)
1188	curl_easy_setopt(result, CURLOPT_SSL_CIPHER_LIST,
1189	ssl_cipherlist);
1190
1191	if (ssl_cert)
1192	curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
1193	if (ssl_cert_type)
1194	curl_easy_setopt(result, CURLOPT_SSLCERTTYPE, ssl_cert_type);
1195	if (has_cert_password())
1196	curl_easy_setopt(result, CURLOPT_KEYPASSWD, cert_auth.password);
1197	if (ssl_key)
1198	curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
1199	if (ssl_key_type)
1200	curl_easy_setopt(result, CURLOPT_SSLKEYTYPE, ssl_key_type);
1201	if (ssl_capath)
1202	curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
1203	if (ssl_pinnedkey)
1204	curl_easy_setopt(result, CURLOPT_PINNEDPUBLICKEY, ssl_pinnedkey);
1205	if (http_ssl_backend && !strcmp("schannel", http_ssl_backend) &&
1206	!http_schannel_use_ssl_cainfo) {
1207	curl_easy_setopt(result, CURLOPT_CAINFO, NULL);
1208	curl_easy_setopt(result, CURLOPT_PROXY_CAINFO, NULL);
1209	} else if (ssl_cainfo != NULL \|\| http_proxy_ssl_ca_info != NULL) {
1210	if (ssl_cainfo)
1211	curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
1212	if (http_proxy_ssl_ca_info)
1213	curl_easy_setopt(result, CURLOPT_PROXY_CAINFO, http_proxy_ssl_ca_info);
1214	}
1215
1216	if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
1217	curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
1218	curl_low_speed_limit);
1219	curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
1220	curl_low_speed_time);
1221	}
1222
1223	curl_easy_setopt(result, CURLOPT_MAXREDIRS, 20L);
1224	curl_easy_setopt(result, CURLOPT_POSTREDIR, (long)CURL_REDIR_POST_ALL);
1225
1226	#ifdef GIT_CURL_HAVE_CURLOPT_PROTOCOLS_STR
1227	{
1228	struct strbuf buf = STRBUF_INIT;
1229
1230	get_curl_allowed_protocols(0, &buf);
1231	curl_easy_setopt(result, CURLOPT_REDIR_PROTOCOLS_STR, buf.buf);
1232	strbuf_reset(&buf);
1233
1234	get_curl_allowed_protocols(-1, &buf);
1235	curl_easy_setopt(result, CURLOPT_PROTOCOLS_STR, buf.buf);
1236	strbuf_release(&buf);
1237	}
1238	#else
1239	curl_easy_setopt(result, CURLOPT_REDIR_PROTOCOLS,
1240	get_curl_allowed_protocols(0, NULL));
1241	curl_easy_setopt(result, CURLOPT_PROTOCOLS,
1242	get_curl_allowed_protocols(-1, NULL));
1243	#endif
1244
1245	if (getenv("GIT_CURL_VERBOSE"))
1246	http_trace_curl_no_data();
1247	setup_curl_trace(result);
1248	if (getenv("GIT_TRACE_CURL_NO_DATA"))
1249	trace_curl_data = 0;
1250	if (!git_env_bool("GIT_TRACE_REDACT", 1))
1251	trace_curl_redact = 0;
1252
1253	curl_easy_setopt(result, CURLOPT_USERAGENT,
1254	user_agent ? user_agent : git_user_agent());
1255
1256	if (curl_ftp_no_epsv)
1257	curl_easy_setopt(result, CURLOPT_FTP_USE_EPSV, 0L);
1258
1259	if (curl_ssl_try)
1260	curl_easy_setopt(result, CURLOPT_USE_SSL, CURLUSESSL_TRY);
1261
1262	/*
1263	* CURL also examines these variables as a fallback; but we need to query
1264	* them here in order to decide whether to prompt for missing password (cf.
1265	* init_curl_proxy_auth()).
1266	*
1267	* Unlike many other common environment variables, these are historically
1268	* lowercase only. It appears that CURL did not know this and implemented
1269	* only uppercase variants, which was later corrected to take both - with
1270	* the exception of http_proxy, which is lowercase only also in CURL. As
1271	* the lowercase versions are the historical quasi-standard, they take
1272	* precedence here, as in CURL.
1273	*/
1274	if (!curl_http_proxy) {
1275	if (http_auth.protocol && !strcmp(http_auth.protocol, "https")) {
1276	var_override(&curl_http_proxy, getenv("HTTPS_PROXY"));
1277	var_override(&curl_http_proxy, getenv("https_proxy"));
1278	} else {
1279	var_override(&curl_http_proxy, getenv("http_proxy"));
1280	}
1281	if (!curl_http_proxy) {
1282	var_override(&curl_http_proxy, getenv("ALL_PROXY"));
1283	var_override(&curl_http_proxy, getenv("all_proxy"));
1284	}
1285	}
1286
1287	if (curl_http_proxy && curl_http_proxy[0] == '\0') {
1288	/*
1289	* Handle case with the empty http.proxy value here to keep
1290	* common code clean.
1291	* NB: empty option disables proxying at all.
1292	*/
1293	curl_easy_setopt(result, CURLOPT_PROXY, "");
1294	} else if (curl_http_proxy) {
1295	struct strbuf proxy = STRBUF_INIT;
1296
1297	if (strstr(curl_http_proxy, "://"))
1298	credential_from_url(&proxy_auth, curl_http_proxy);
1299	else {
1300	struct strbuf url = STRBUF_INIT;
1301	strbuf_addf(&url, "http://%s", curl_http_proxy);
1302	credential_from_url(&proxy_auth, url.buf);
1303	strbuf_release(&url);
1304	}
1305
1306	if (set_curl_proxy_type(result, proxy_auth.protocol) < 0)
1307	die("Invalid proxy URL '%s': unsupported proxy scheme '%s'",
1308	curl_http_proxy, proxy_auth.protocol);
1309
1310	if (!proxy_auth.host)
1311	die("Invalid proxy URL '%s'", curl_http_proxy);
1312
1313	strbuf_addstr(&proxy, proxy_auth.host);
1314	if (proxy_auth.path) {
1315	curl_version_info_data *ver = curl_version_info(CURLVERSION_NOW);
1316
1317	if (ver->version_num < 0x075400)
1318	die("libcurl 7.84 or later is required to support paths in proxy URLs");
1319
1320	if (!is_socks_proxy_protocol(proxy_auth.protocol))
1321	die("Invalid proxy URL '%s': only SOCKS proxies support paths",
1322	curl_http_proxy);
1323
1324	if (strcasecmp(proxy_auth.host, "localhost"))
1325	die("Invalid proxy URL '%s': host must be localhost if a path is present",
1326	curl_http_proxy);
1327
1328	strbuf_addch(&proxy, '/');
1329	strbuf_add_percentencode(&proxy, proxy_auth.path, 0);
1330	}
1331	curl_easy_setopt(result, CURLOPT_PROXY, proxy.buf);
1332	strbuf_release(&proxy);
1333
1334	var_override(&curl_no_proxy, getenv("NO_PROXY"));
1335	var_override(&curl_no_proxy, getenv("no_proxy"));
1336	curl_easy_setopt(result, CURLOPT_NOPROXY, curl_no_proxy);
1337	}
1338	init_curl_proxy_auth(result);
1339
1340	curl_easy_setopt(result, CURLOPT_TCP_KEEPALIVE, 1L);
1341
1342	if (curl_tcp_keepidle > -1)
1343	curl_easy_setopt(result, CURLOPT_TCP_KEEPIDLE,
1344	curl_tcp_keepidle);
1345	if (curl_tcp_keepintvl > -1)
1346	curl_easy_setopt(result, CURLOPT_TCP_KEEPINTVL,
1347	curl_tcp_keepintvl);
1348	#ifdef GIT_CURL_HAVE_CURLOPT_TCP_KEEPCNT
1349	if (curl_tcp_keepcnt > -1)
1350	curl_easy_setopt(result, CURLOPT_TCP_KEEPCNT, curl_tcp_keepcnt);
1351	#endif
1352
1353	return result;
1354	}
1355
1356	static void set_from_env(char *var, const char envname)
1357	{
1358	const char *val = getenv(envname);
1359	if (val) {
1360	FREE_AND_NULL(*var);
1361	*var = xstrdup(val);
1362	}
1363	}
1364
1365	static void set_long_from_env(long var, const char envname)
1366	{
1367	const char *val = getenv(envname);
1368	if (val) {
1369	long tmp;
1370	char *endp;
1371	int saved_errno = errno;
1372
1373	errno = 0;
1374	tmp = strtol(val, &endp, 10);
1375
1376	if (errno)
1377	warning_errno(_("failed to parse %s"), envname);
1378	else if (*endp \|\| endp == val)
1379	warning(_("failed to parse %s"), envname);
1380	else
1381	*var = tmp;
1382
1383	errno = saved_errno;
1384	}
1385	}
1386
1387	void http_init(struct remote remote, const char url, int proactive_auth)
1388	{
1389	char *normalized_url;
1390	struct urlmatch_config config = URLMATCH_CONFIG_INIT;
1391
1392	config.section = "http";
1393	config.key = NULL;
1394	config.collect_fn = http_options;
1395	config.cascade_fn = git_default_config;
1396	config.cb = NULL;
1397
1398	http_is_verbose = 0;
1399	normalized_url = url_normalize(url, &config.url);
1400
1401	repo_config(the_repository, urlmatch_config_entry, &config);
1402	free(normalized_url);
1403	string_list_clear(&config.vars, 1);
1404
1405	if (http_ssl_backend) {
1406	const curl_ssl_backend **backends;
1407	struct strbuf buf = STRBUF_INIT;
1408	int i;
1409
1410	switch (curl_global_sslset(-1, http_ssl_backend, &backends)) {
1411	case CURLSSLSET_UNKNOWN_BACKEND:
1412	strbuf_addf(&buf, _("Unsupported SSL backend '%s'. "
1413	"Supported SSL backends:"),
1414	http_ssl_backend);
1415	for (i = 0; backends[i]; i++)
1416	strbuf_addf(&buf, "\n\t%s", backends[i]->name);
1417	die("%s", buf.buf);
1418	case CURLSSLSET_NO_BACKENDS:
1419	die(_("Could not set SSL backend to '%s': "
1420	"cURL was built without SSL backends"),
1421	http_ssl_backend);
1422	case CURLSSLSET_TOO_LATE:
1423	die(_("Could not set SSL backend to '%s': already set"),
1424	http_ssl_backend);
1425	case CURLSSLSET_OK:
1426	break; /* Okay! */
1427	}
1428	}
1429
1430	if (curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK)
1431	die("curl_global_init failed");
1432
1433	#ifdef GIT_CURL_HAVE_GLOBAL_TRACE
1434	{
1435	const char *comp = getenv("GIT_TRACE_CURL_COMPONENTS");
1436	if (comp)
1437	curl_global_trace(comp);
1438	}
1439	#endif
1440
1441	if (proactive_auth && http_proactive_auth == PROACTIVE_AUTH_NONE)
1442	http_proactive_auth = PROACTIVE_AUTH_IF_CREDENTIALS;
1443
1444	if (remote && remote->http_proxy)
1445	curl_http_proxy = xstrdup(remote->http_proxy);
1446
1447	if (remote)
1448	var_override(&http_proxy_authmethod, remote->http_proxy_authmethod);
1449
1450	pragma_header = curl_slist_append(http_copy_default_headers(),
1451	"Pragma: no-cache");
1452
1453	{
1454	char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
1455	if (http_max_requests)
1456	max_requests = atoi(http_max_requests);
1457	}
1458
1459	curlm = curl_multi_init();
1460	if (!curlm)
1461	die("curl_multi_init failed");
1462
1463	if (getenv("GIT_SSL_NO_VERIFY"))
1464	curl_ssl_verify = 0;
1465
1466	set_from_env(&ssl_cert, "GIT_SSL_CERT");
1467	set_from_env(&ssl_cert_type, "GIT_SSL_CERT_TYPE");
1468	set_from_env(&ssl_key, "GIT_SSL_KEY");
1469	set_from_env(&ssl_key_type, "GIT_SSL_KEY_TYPE");
1470	set_from_env(&ssl_capath, "GIT_SSL_CAPATH");
1471	set_from_env(&ssl_cainfo, "GIT_SSL_CAINFO");
1472
1473	set_from_env(&user_agent, "GIT_HTTP_USER_AGENT");
1474
1475	set_long_from_env(&curl_low_speed_limit, "GIT_HTTP_LOW_SPEED_LIMIT");
1476	set_long_from_env(&curl_low_speed_time, "GIT_HTTP_LOW_SPEED_TIME");
1477
1478	if (curl_ssl_verify == -1)
1479	curl_ssl_verify = 1;
1480
1481	curl_session_count = 0;
1482	if (max_requests < 1)
1483	max_requests = DEFAULT_MAX_REQUESTS;
1484
1485	set_from_env(&http_proxy_ssl_cert, "GIT_PROXY_SSL_CERT");
1486	set_from_env(&http_proxy_ssl_key, "GIT_PROXY_SSL_KEY");
1487	set_from_env(&http_proxy_ssl_ca_info, "GIT_PROXY_SSL_CAINFO");
1488
1489	if (getenv("GIT_PROXY_SSL_CERT_PASSWORD_PROTECTED"))
1490	proxy_ssl_cert_password_required = 1;
1491
1492	if (getenv("GIT_CURL_FTP_NO_EPSV"))
1493	curl_ftp_no_epsv = 1;
1494
1495	if (url) {
1496	credential_from_url(&http_auth, url);
1497	if (!ssl_cert_password_required &&
1498	getenv("GIT_SSL_CERT_PASSWORD_PROTECTED") &&
1499	starts_with(url, "https://"))
1500	ssl_cert_password_required = 1;
1501	}
1502
1503	set_long_from_env(&curl_tcp_keepidle, "GIT_TCP_KEEPIDLE");
1504	set_long_from_env(&curl_tcp_keepintvl, "GIT_TCP_KEEPINTVL");
1505	set_long_from_env(&curl_tcp_keepcnt, "GIT_TCP_KEEPCNT");
1506
1507	set_long_from_env(&http_retry_after, "GIT_HTTP_RETRY_AFTER");
1508	set_long_from_env(&http_max_retries, "GIT_HTTP_MAX_RETRIES");
1509	set_long_from_env(&http_max_retry_time, "GIT_HTTP_MAX_RETRY_TIME");
1510
1511	curl_default = get_curl_handle();
1512	}
1513
1514	void http_cleanup(void)
1515	{
1516	struct active_request_slot *slot = active_queue_head;
1517
1518	while (slot != NULL) {
1519	struct active_request_slot *next = slot->next;
1520	if (slot->curl) {
1521	xmulti_remove_handle(slot);
1522	curl_easy_cleanup(slot->curl);
1523	}
1524	free(slot);
1525	slot = next;
1526	}
1527	active_queue_head = NULL;
1528
1529	curl_easy_cleanup(curl_default);
1530
1531	curl_multi_cleanup(curlm);
1532	curl_global_cleanup();
1533
1534	string_list_clear(&extra_http_headers, 0);
1535
1536	curl_slist_free_all(pragma_header);
1537	pragma_header = NULL;
1538
1539	curl_slist_free_all(host_resolutions);
1540	host_resolutions = NULL;
1541
1542	if (curl_http_proxy) {
1543	free((void *)curl_http_proxy);
1544	curl_http_proxy = NULL;
1545	}
1546
1547	if (proxy_auth.password) {
1548	memset(proxy_auth.password, 0, strlen(proxy_auth.password));
1549	FREE_AND_NULL(proxy_auth.password);
1550	}
1551
1552	free((void *)curl_proxyuserpwd);
1553	curl_proxyuserpwd = NULL;
1554
1555	free((void *)http_proxy_authmethod);
1556	http_proxy_authmethod = NULL;
1557
1558	if (cert_auth.password) {
1559	memset(cert_auth.password, 0, strlen(cert_auth.password));
1560	FREE_AND_NULL(cert_auth.password);
1561	}
1562	ssl_cert_password_required = 0;
1563
1564	if (proxy_cert_auth.password) {
1565	memset(proxy_cert_auth.password, 0, strlen(proxy_cert_auth.password));
1566	FREE_AND_NULL(proxy_cert_auth.password);
1567	}
1568	proxy_ssl_cert_password_required = 0;
1569
1570	FREE_AND_NULL(cached_accept_language);
1571	}
1572
1573	struct active_request_slot *get_active_slot(void)
1574	{
1575	struct active_request_slot *slot = active_queue_head;
1576	struct active_request_slot *newslot;
1577
1578	int num_transfers;
1579
1580	/* Wait for a slot to open up if the queue is full */
1581	while (active_requests >= max_requests) {
1582	curl_multi_perform(curlm, &num_transfers);
1583	if (num_transfers < active_requests)
1584	process_curl_messages();
1585	}
1586
1587	while (slot != NULL && slot->in_use)
1588	slot = slot->next;
1589
1590	if (!slot) {
1591	newslot = xmalloc(sizeof(*newslot));
1592	newslot->curl = NULL;
1593	newslot->in_use = 0;
1594	newslot->next = NULL;
1595
1596	slot = active_queue_head;
1597	if (!slot) {
1598	active_queue_head = newslot;
1599	} else {
1600	while (slot->next != NULL)
1601	slot = slot->next;
1602	slot->next = newslot;
1603	}
1604	slot = newslot;
1605	}
1606
1607	if (!slot->curl) {
1608	slot->curl = curl_easy_duphandle(curl_default);
1609	curl_session_count++;
1610	}
1611
1612	active_requests++;
1613	slot->in_use = 1;
1614	slot->results = NULL;
1615	slot->finished = NULL;
1616	slot->callback_data = NULL;
1617	slot->callback_func = NULL;
1618
1619	if (curl_cookie_file && !strcmp(curl_cookie_file, "-")) {
1620	warning(_("refusing to read cookies from http.cookiefile '-'"));
1621	FREE_AND_NULL(curl_cookie_file);
1622	}
1623	curl_easy_setopt(slot->curl, CURLOPT_COOKIEFILE, curl_cookie_file);
1624	if (curl_save_cookies && (!curl_cookie_file \|\| !curl_cookie_file[0])) {
1625	curl_save_cookies = 0;
1626	warning(_("ignoring http.savecookies for empty http.cookiefile"));
1627	}
1628	if (curl_save_cookies)
1629	curl_easy_setopt(slot->curl, CURLOPT_COOKIEJAR, curl_cookie_file);
1630	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
1631	curl_easy_setopt(slot->curl, CURLOPT_RESOLVE, host_resolutions);
1632	curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
1633	curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, NULL);
1634	curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, NULL);
1635	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, NULL);
1636	curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, NULL);
1637	curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, -1L);
1638	curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 0L);
1639	curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1L);
1640	curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 1L);
1641	curl_easy_setopt(slot->curl, CURLOPT_RANGE, NULL);
1642
1643	/*
1644	* Default following to off unless "ALWAYS" is configured; this gives
1645	* callers a sane starting point, and they can tweak for individual
1646	* HTTP_FOLLOW_* cases themselves.
1647	*/
1648	if (http_follow_config == HTTP_FOLLOW_ALWAYS)
1649	curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 1L);
1650	else
1651	curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 0L);
1652
1653	curl_easy_setopt(slot->curl, CURLOPT_IPRESOLVE, git_curl_ipresolve);
1654	curl_easy_setopt(slot->curl, CURLOPT_HTTPAUTH, http_auth_methods);
1655	if (http_auth.password \|\| http_auth.credential \|\| curl_empty_auth_enabled())
1656	init_curl_http_auth(slot->curl);
1657
1658	return slot;
1659	}
1660
1661	int start_active_slot(struct active_request_slot *slot)
1662	{
1663	CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
1664	int num_transfers;
1665
1666	if (curlm_result != CURLM_OK &&
1667	curlm_result != CURLM_CALL_MULTI_PERFORM) {
1668	warning("curl_multi_add_handle failed: %s",
1669	curl_multi_strerror(curlm_result));
1670	active_requests--;
1671	slot->in_use = 0;
1672	return 0;
1673	}
1674
1675	/*
1676	* We know there must be something to do, since we just added
1677	* something.
1678	*/
1679	curl_multi_perform(curlm, &num_transfers);
1680	return 1;
1681	}
1682
1683	struct fill_chain {
1684	void *data;
1685	int (fill)(void );
1686	struct fill_chain *next;
1687	};
1688
1689	static struct fill_chain *fill_cfg;
1690
1691	void add_fill_function(void data, int (fill)(void *))
1692	{
1693	struct fill_chain new_fill = xmalloc(sizeof(new_fill));
1694	struct fill_chain **linkp = &fill_cfg;
1695	new_fill->data = data;
1696	new_fill->fill = fill;
1697	new_fill->next = NULL;
1698	while (*linkp)
1699	linkp = &(*linkp)->next;
1700	*linkp = new_fill;
1701	}
1702
1703	void fill_active_slots(void)
1704	{
1705	struct active_request_slot *slot = active_queue_head;
1706
1707	while (active_requests < max_requests) {
1708	struct fill_chain *fill;
1709	for (fill = fill_cfg; fill; fill = fill->next)
1710	if (fill->fill(fill->data))
1711	break;
1712
1713	if (!fill)
1714	break;
1715	}
1716
1717	while (slot != NULL) {
1718	if (!slot->in_use && slot->curl != NULL
1719	&& curl_session_count > min_curl_sessions) {
1720	curl_easy_cleanup(slot->curl);
1721	slot->curl = NULL;
1722	curl_session_count--;
1723	}
1724	slot = slot->next;
1725	}
1726	}
1727
1728	void step_active_slots(void)
1729	{
1730	int num_transfers;
1731	CURLMcode curlm_result;
1732
1733	do {
1734	curlm_result = curl_multi_perform(curlm, &num_transfers);
1735	} while (curlm_result == CURLM_CALL_MULTI_PERFORM);
1736	if (num_transfers < active_requests) {
1737	process_curl_messages();
1738	fill_active_slots();
1739	}
1740	}
1741
1742	void run_active_slot(struct active_request_slot *slot)
1743	{
1744	fd_set readfds;
1745	fd_set writefds;
1746	fd_set excfds;
1747	int max_fd;
1748	struct timeval select_timeout;
1749	int finished = 0;
1750
1751	slot->finished = &finished;
1752	while (!finished) {
1753	step_active_slots();
1754
1755	if (slot->in_use) {
1756	long curl_timeout;
1757	curl_multi_timeout(curlm, &curl_timeout);
1758	if (curl_timeout == 0) {
1759	continue;
1760	} else if (curl_timeout == -1) {
1761	select_timeout.tv_sec = 0;
1762	select_timeout.tv_usec = 50000;
1763	} else {
1764	select_timeout.tv_sec = curl_timeout / 1000;
1765	select_timeout.tv_usec = (curl_timeout % 1000) * 1000;
1766	}
1767
1768	max_fd = -1;
1769	FD_ZERO(&readfds);
1770	FD_ZERO(&writefds);
1771	FD_ZERO(&excfds);
1772	curl_multi_fdset(curlm, &readfds, &writefds, &excfds, &max_fd);
1773
1774	/*
1775	* It can happen that curl_multi_timeout returns a pathologically
1776	* long timeout when curl_multi_fdset returns no file descriptors
1777	* to read. See commit message for more details.
1778	*/
1779	if (max_fd < 0 &&
1780	(select_timeout.tv_sec > 0 \|\|
1781	select_timeout.tv_usec > 50000)) {
1782	select_timeout.tv_sec = 0;
1783	select_timeout.tv_usec = 50000;
1784	}
1785
1786	select(max_fd+1, &readfds, &writefds, &excfds, &select_timeout);
1787	}
1788	}
1789
1790	/*
1791	* The value of slot->finished we set before the loop was used
1792	* to set our "finished" variable when our request completed.
1793	*
1794	* 1. The slot may not have been reused for another request
1795	* yet, in which case it still has &finished.
1796	*
1797	* 2. The slot may already be in-use to serve another request,
1798	* which can further be divided into two cases:
1799	*
1800	* (a) If call run_active_slot() hasn't been called for that
1801	* other request, slot->finished would have been cleared
1802	* by get_active_slot() and has NULL.
1803	*
1804	* (b) If the request did call run_active_slot(), then the
1805	* call would have updated slot->finished at the beginning
1806	* of this function, and with the clearing of the member
1807	* below, we would find that slot->finished is now NULL.
1808	*
1809	* In all cases, slot->finished has no useful information to
1810	* anybody at this point. Some compilers warn us for
1811	* attempting to smuggle a pointer that is about to become
1812	* invalid, i.e. &finished. We clear it here to assure them.
1813	*/
1814	slot->finished = NULL;
1815	}
1816
1817	static void release_active_slot(struct active_request_slot *slot)
1818	{
1819	closedown_active_slot(slot);
1820	if (slot->curl) {
1821	xmulti_remove_handle(slot);
1822	if (curl_session_count > min_curl_sessions) {
1823	curl_easy_cleanup(slot->curl);
1824	slot->curl = NULL;
1825	curl_session_count--;
1826	}
1827	}
1828	fill_active_slots();
1829	}
1830
1831	void finish_all_active_slots(void)
1832	{
1833	struct active_request_slot *slot = active_queue_head;
1834
1835	while (slot != NULL)
1836	if (slot->in_use) {
1837	run_active_slot(slot);
1838	slot = active_queue_head;
1839	} else {
1840	slot = slot->next;
1841	}
1842	}
1843
1844	/* Helpers for modifying and creating URLs */
1845	static inline int needs_quote(int ch)
1846	{
1847	if (((ch >= 'A') && (ch <= 'Z'))
1848	\|\| ((ch >= 'a') && (ch <= 'z'))
1849	\|\| ((ch >= '0') && (ch <= '9'))
1850	\|\| (ch == '/')
1851	\|\| (ch == '-')
1852	\|\| (ch == '.'))
1853	return 0;
1854	return 1;
1855	}
1856
1857	static char quote_ref_url(const char base, const char *ref)
1858	{
1859	struct strbuf buf = STRBUF_INIT;
1860	const char *cp;
1861	int ch;
1862
1863	end_url_with_slash(&buf, base);
1864
1865	for (cp = ref; (ch = *cp) != 0; cp++)
1866	if (needs_quote(ch))
1867	strbuf_addf(&buf, "%%%02x", ch);
1868	else
1869	strbuf_addch(&buf, *cp);
1870
1871	return strbuf_detach(&buf, NULL);
1872	}
1873
1874	void append_remote_object_url(struct strbuf buf, const char url,
1875	const char *hex,
1876	int only_two_digit_prefix)
1877	{
1878	end_url_with_slash(buf, url);
1879
1880	strbuf_addf(buf, "objects/%.*s/", 2, hex);
1881	if (!only_two_digit_prefix)
1882	strbuf_addstr(buf, hex + 2);
1883	}
1884
1885	char get_remote_object_url(const char url, const char *hex,
1886	int only_two_digit_prefix)
1887	{
1888	struct strbuf buf = STRBUF_INIT;
1889	append_remote_object_url(&buf, url, hex, only_two_digit_prefix);
1890	return strbuf_detach(&buf, NULL);
1891	}
1892
1893	void normalize_curl_result(CURLcode *result, long http_code,
1894	char *errorstr, size_t errorlen)
1895	{
1896	/*
1897	* If we see a failing http code with CURLE_OK, we have turned off
1898	* FAILONERROR (to keep the server's custom error response), and should
1899	* translate the code into failure here.
1900	*
1901	* Likewise, if we see a redirect (30x code), that means we turned off
1902	* redirect-following, and we should treat the result as an error.
1903	*/
1904	if (*result == CURLE_OK && http_code >= 300) {
1905	*result = CURLE_HTTP_RETURNED_ERROR;
1906	/*
1907	* Normally curl will already have put the "reason phrase"
1908	* from the server into curl_errorstr; unfortunately without
1909	* FAILONERROR it is lost, so we can give only the numeric
1910	* status code.
1911	*/
1912	xsnprintf(errorstr, errorlen,
1913	"The requested URL returned error: %ld",
1914	http_code);
1915	}
1916	}
1917
1918	static int handle_curl_result(struct slot_results *results)
1919	{
1920	normalize_curl_result(&results->curl_result, results->http_code,
1921	curl_errorstr, sizeof(curl_errorstr));
1922
1923	if (results->curl_result == CURLE_OK) {
1924	credential_approve(the_repository, &http_auth);
1925	credential_approve(the_repository, &proxy_auth);
1926	credential_approve(the_repository, &cert_auth);
1927	return HTTP_OK;
1928	} else if (results->curl_result == CURLE_SSL_CERTPROBLEM) {
1929	/*
1930	* We can't tell from here whether it's a bad path, bad
1931	* certificate, bad password, or something else wrong
1932	* with the certificate. So we reject the credential to
1933	* avoid caching or saving a bad password.
1934	*/
1935	credential_reject(the_repository, &cert_auth);
1936	return HTTP_NOAUTH;
1937	} else if (results->curl_result == CURLE_SSL_PINNEDPUBKEYNOTMATCH) {
1938	return HTTP_NOMATCHPUBLICKEY;
1939	} else if (missing_target(results))
1940	return HTTP_MISSING_TARGET;
1941	else if (results->http_code == 401) {
1942	if ((http_auth.username && http_auth.password) \|\|\
1943	(http_auth.authtype && http_auth.credential)) {
1944	if (http_auth.multistage) {
1945	credential_clear_secrets(&http_auth);
1946	return HTTP_REAUTH;
1947	}
1948	credential_reject(the_repository, &http_auth);
1949	if (always_auth_proactively())
1950	http_proactive_auth = PROACTIVE_AUTH_NONE;
1951	return HTTP_NOAUTH;
1952	} else {
1953	if (curl_empty_auth == -1 &&
1954	!empty_auth_try_negotiate &&
1955	(results->auth_avail & CURLAUTH_GSSNEGOTIATE)) {
1956	/*
1957	* In auto mode, give Negotiate a chance via
1958	* empty auth before stripping it. If it fails,
1959	* we will strip it on the next 401.
1960	*/
1961	empty_auth_try_negotiate = 1;
1962	} else {
1963	http_auth_methods &= ~CURLAUTH_GSSNEGOTIATE;
1964	}
1965	if (results->auth_avail) {
1966	http_auth_methods &= results->auth_avail;
1967	http_auth_methods_restricted = 1;
1968	}
1969	return HTTP_REAUTH;
1970	}
1971	} else if (results->http_code == 429) {
1972	trace2_data_intmax("http", the_repository, "http/429-retry-after",
1973	results->retry_after);
1974	return HTTP_RATE_LIMITED;
1975	} else {
1976	if (results->http_connectcode == 407)
1977	credential_reject(the_repository, &proxy_auth);
1978	if (!curl_errorstr[0])
1979	strlcpy(curl_errorstr,
1980	curl_easy_strerror(results->curl_result),
1981	sizeof(curl_errorstr));
1982	return HTTP_ERROR;
1983	}
1984	}
1985
1986	int run_one_slot(struct active_request_slot *slot,
1987	struct slot_results *results)
1988	{
1989	slot->results = results;
1990
1991	if (!start_active_slot(slot)) {
1992	xsnprintf(curl_errorstr, sizeof(curl_errorstr),
1993	"failed to start HTTP request");
1994	return HTTP_START_FAILED;
1995	}
1996
1997	run_active_slot(slot);
1998	return handle_curl_result(results);
1999	}
2000
2001	struct curl_slist *http_copy_default_headers(void)
2002	{
2003	struct curl_slist *headers = NULL;
2004	const struct string_list_item *item;
2005
2006	for_each_string_list_item(item, &extra_http_headers)
2007	headers = curl_slist_append(headers, item->string);
2008
2009	return headers;
2010	}
2011
2012	static CURLcode curlinfo_strbuf(CURL curl, CURLINFO info, struct strbuf buf)
2013	{
2014	char *ptr;
2015	CURLcode ret;
2016
2017	strbuf_reset(buf);
2018	ret = curl_easy_getinfo(curl, info, &ptr);
2019	if (!ret && ptr)
2020	strbuf_addstr(buf, ptr);
2021	return ret;
2022	}
2023
2024	/*
2025	* Check for and extract a content-type parameter. "raw"
2026	* should be positioned at the start of the potential
2027	* parameter, with any whitespace already removed.
2028	*
2029	* "name" is the name of the parameter. The value is appended
2030	* to "out".
2031	*/
2032	static int extract_param(const char raw, const char name,
2033	struct strbuf *out)
2034	{
2035	size_t len = strlen(name);
2036
2037	if (strncasecmp(raw, name, len))
2038	return -1;
2039	raw += len;
2040
2041	if (*raw != '=')
2042	return -1;
2043	raw++;
2044
2045	while (raw && !isspace(raw) && *raw != ';')
2046	strbuf_addch(out, *raw++);
2047	return 0;
2048	}
2049
2050	/*
2051	* Extract a normalized version of the content type, with any
2052	* spaces suppressed, all letters lowercased, and no trailing ";"
2053	* or parameters.
2054	*
2055	* Note that we will silently remove even invalid whitespace. For
2056	* example, "text / plain" is specifically forbidden by RFC 2616,
2057	* but "text/plain" is the only reasonable output, and this keeps
2058	* our code simple.
2059	*
2060	* If the "charset" argument is not NULL, store the value of any
2061	* charset parameter there.
2062	*
2063	* Example:
2064	* "TEXT/PLAIN; charset=utf-8" -> "text/plain", "utf-8"
2065	* "text / plain" -> "text/plain"
2066	*/
2067	static void extract_content_type(struct strbuf raw, struct strbuf type,
2068	struct strbuf *charset)
2069	{
2070	const char *p;
2071
2072	strbuf_reset(type);
2073	strbuf_grow(type, raw->len);
2074	for (p = raw->buf; *p; p++) {
2075	if (isspace(*p))
2076	continue;
2077	if (*p == ';') {
2078	p++;
2079	break;
2080	}
2081	strbuf_addch(type, tolower(*p));
2082	}
2083
2084	if (!charset)
2085	return;
2086
2087	strbuf_reset(charset);
2088	while (*p) {
2089	while (isspace(p) \|\| p == ';')
2090	p++;
2091	if (!extract_param(p, "charset", charset))
2092	return;
2093	while (p && !isspace(p))
2094	p++;
2095	}
2096
2097	if (!charset->len && starts_with(type->buf, "text/"))
2098	strbuf_addstr(charset, "ISO-8859-1");
2099	}
2100
2101	static void write_accept_language(struct strbuf *buf)
2102	{
2103	/*
2104	* MAX_DECIMAL_PLACES must not be larger than 3. If it is larger than
2105	* that, q-value will be smaller than 0.001, the minimum q-value the
2106	* HTTP specification allows. See
2107	* https://datatracker.ietf.org/doc/html/rfc7231#section-5.3.1 for q-value.
2108	*/
2109	const int MAX_DECIMAL_PLACES = 3;
2110	const int MAX_LANGUAGE_TAGS = 1000;
2111	const int MAX_ACCEPT_LANGUAGE_HEADER_SIZE = 4000;
2112	char **language_tags = NULL;
2113	int num_langs = 0;
2114	const char *s = get_preferred_languages();
2115	int i;
2116	struct strbuf tag = STRBUF_INIT;
2117
2118	/* Don't add Accept-Language header if no language is preferred. */
2119	if (!s)
2120	return;
2121
2122	/*
2123	* Split the colon-separated string of preferred languages into
2124	* language_tags array.
2125	*/
2126	do {
2127	/* collect language tag */
2128	for (; s && (isalnum(s) \|\| *s == '_'); s++)
2129	strbuf_addch(&tag, s == '_' ? '-' : s);
2130
2131	/* skip .codeset, @modifier and any other unnecessary parts */
2132	while (s && s != ':')
2133	s++;
2134
2135	if (tag.len) {
2136	num_langs++;
2137	REALLOC_ARRAY(language_tags, num_langs);
2138	language_tags[num_langs - 1] = strbuf_detach(&tag, NULL);
2139	if (num_langs >= MAX_LANGUAGE_TAGS - 1) /* -1 for '' /
2140	break;
2141	}
2142	} while (*s++);
2143
2144	/* write Accept-Language header into buf */
2145	if (num_langs) {
2146	int last_buf_len = 0;
2147	int max_q;
2148	int decimal_places;
2149	char q_format[32];
2150
2151	/* add '' /
2152	REALLOC_ARRAY(language_tags, num_langs + 1);
2153	language_tags[num_langs++] = xstrdup("*");
2154
2155	/* compute decimal_places */
2156	for (max_q = 1, decimal_places = 0;
2157	max_q < num_langs && decimal_places <= MAX_DECIMAL_PLACES;
2158	decimal_places++, max_q *= 10)
2159	;
2160
2161	xsnprintf(q_format, sizeof(q_format), ";q=0.%%0%dd", decimal_places);
2162
2163	strbuf_addstr(buf, "Accept-Language: ");
2164
2165	for (i = 0; i < num_langs; i++) {
2166	if (i > 0)
2167	strbuf_addstr(buf, ", ");
2168
2169	strbuf_addstr(buf, language_tags[i]);
2170
2171	if (i > 0)
2172	strbuf_addf(buf, q_format, max_q - i);
2173
2174	if (buf->len > MAX_ACCEPT_LANGUAGE_HEADER_SIZE) {
2175	strbuf_remove(buf, last_buf_len, buf->len - last_buf_len);
2176	break;
2177	}
2178
2179	last_buf_len = buf->len;
2180	}
2181	}
2182
2183	for (i = 0; i < num_langs; i++)
2184	free(language_tags[i]);
2185	free(language_tags);
2186	}
2187
2188	/*
2189	* Get an Accept-Language header which indicates user's preferred languages.
2190	*
2191	* Examples:
2192	* LANGUAGE= -> ""
2193	* LANGUAGE=ko:en -> "Accept-Language: ko, en; q=0.9, *; q=0.1"
2194	* LANGUAGE=ko_KR.UTF-8:sr@latin -> "Accept-Language: ko-KR, sr; q=0.9, *; q=0.1"
2195	* LANGUAGE=ko LANG=en_US.UTF-8 -> "Accept-Language: ko, *; q=0.1"
2196	* LANGUAGE= LANG=en_US.UTF-8 -> "Accept-Language: en-US, *; q=0.1"
2197	* LANGUAGE= LANG=C -> ""
2198	*/
2199	const char *http_get_accept_language_header(void)
2200	{
2201	if (!cached_accept_language) {
2202	struct strbuf buf = STRBUF_INIT;
2203	write_accept_language(&buf);
2204	if (buf.len > 0)
2205	cached_accept_language = strbuf_detach(&buf, NULL);
2206	}
2207
2208	return cached_accept_language;
2209	}
2210
2211	static void http_opt_request_remainder(CURL *curl, off_t pos)
2212	{
2213	char buf[128];
2214	xsnprintf(buf, sizeof(buf), "%"PRIuMAX"-", (uintmax_t)pos);
2215	curl_easy_setopt(curl, CURLOPT_RANGE, buf);
2216	}
2217
2218	/* http_request() targets */
2219	#define HTTP_REQUEST_STRBUF 0
2220	#define HTTP_REQUEST_FILE 1
2221
2222	static int http_request(const char *url,
2223	void *result, int target,
2224	struct http_get_options *options)
2225	{
2226	struct active_request_slot *slot;
2227	struct slot_results results = { .retry_after = -1 };
2228	struct curl_slist *headers = http_copy_default_headers();
2229	struct strbuf buf = STRBUF_INIT;
2230	const char *accept_language;
2231	int ret;
2232
2233	slot = get_active_slot();
2234	curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1L);
2235
2236	if (!result) {
2237	curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1L);
2238	} else {
2239	curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0L);
2240	curl_easy_setopt(slot->curl, CURLOPT_WRITEDATA, result);
2241
2242	if (target == HTTP_REQUEST_FILE) {
2243	off_t posn = ftello(result);
2244	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
2245	fwrite);
2246	if (posn > 0)
2247	http_opt_request_remainder(slot->curl, posn);
2248	} else
2249	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
2250	fwrite_buffer);
2251	}
2252
2253	curl_easy_setopt(slot->curl, CURLOPT_HEADERFUNCTION, fwrite_wwwauth);
2254
2255	accept_language = http_get_accept_language_header();
2256
2257	if (accept_language)
2258	headers = curl_slist_append(headers, accept_language);
2259
2260	strbuf_addstr(&buf, "Pragma:");
2261	if (options->no_cache)
2262	strbuf_addstr(&buf, " no-cache");
2263	if (options->initial_request &&
2264	http_follow_config == HTTP_FOLLOW_INITIAL)
2265	curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 1L);
2266
2267	headers = curl_slist_append(headers, buf.buf);
2268
2269	/* Add additional headers here */
2270	if (options->extra_headers) {
2271	const struct string_list_item *item;
2272	for_each_string_list_item(item, options->extra_headers)
2273	headers = curl_slist_append(headers, item->string);
2274	}
2275
2276	headers = http_append_auth_header(&http_auth, headers);
2277
2278	curl_easy_setopt(slot->curl, CURLOPT_URL, url);
2279	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers);
2280	curl_easy_setopt(slot->curl, CURLOPT_ENCODING, "");
2281	curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 0L);
2282
2283	ret = run_one_slot(slot, &results);
2284
2285	#ifdef GIT_CURL_HAVE_CURLINFO_RETRY_AFTER
2286	if (ret == HTTP_RATE_LIMITED) {
2287	curl_off_t retry_after;
2288	if (curl_easy_getinfo(slot->curl, CURLINFO_RETRY_AFTER,
2289	&retry_after) == CURLE_OK && retry_after > 0)
2290	results.retry_after = (long)retry_after;
2291	}
2292	#endif
2293
2294	options->retry_after = results.retry_after;
2295
2296	if (options->content_type) {
2297	struct strbuf raw = STRBUF_INIT;
2298	curlinfo_strbuf(slot->curl, CURLINFO_CONTENT_TYPE, &raw);
2299	extract_content_type(&raw, options->content_type,
2300	options->charset);
2301	strbuf_release(&raw);
2302	}
2303
2304	if (options->effective_url)
2305	curlinfo_strbuf(slot->curl, CURLINFO_EFFECTIVE_URL,
2306	options->effective_url);
2307
2308	curl_slist_free_all(headers);
2309	strbuf_release(&buf);
2310
2311	return ret;
2312	}
2313
2314	/*
2315	* Update the "base" url to a more appropriate value, as deduced by
2316	* redirects seen when requesting a URL starting with "url".
2317	*
2318	* The "asked" parameter is a URL that we asked curl to access, and must begin
2319	* with "base".
2320	*
2321	* The "got" parameter is the URL that curl reported to us as where we ended
2322	* up.
2323	*
2324	* Returns 1 if we updated the base url, 0 otherwise.
2325	*
2326	* Our basic strategy is to compare "base" and "asked" to find the bits
2327	* specific to our request. We then strip those bits off of "got" to yield the
2328	* new base. So for example, if our base is "http://example.com/foo.git",
2329	* and we ask for "http://example.com/foo.git/info/refs", we might end up
2330	* with "https://other.example.com/foo.git/info/refs". We would want the
2331	* new URL to become "https://other.example.com/foo.git".
2332	*
2333	* Note that this assumes a sane redirect scheme. It's entirely possible
2334	* in the example above to end up at a URL that does not even end in
2335	* "info/refs". In such a case we die. There's not much we can do, such a
2336	* scheme is unlikely to represent a real git repository, and failing to
2337	* rewrite the base opens options for malicious redirects to do funny things.
2338	*/
2339	static int update_url_from_redirect(struct strbuf *base,
2340	const char *asked,
2341	const struct strbuf *got)
2342	{
2343	const char *tail;
2344	size_t new_len;
2345
2346	if (!strcmp(asked, got->buf))
2347	return 0;
2348
2349	if (!skip_prefix(asked, base->buf, &tail))
2350	BUG("update_url_from_redirect: %s is not a superset of %s",
2351	asked, base->buf);
2352
2353	new_len = got->len;
2354	if (!strip_suffix_mem(got->buf, &new_len, tail))
2355	die(_("unable to update url base from redirection:\n"
2356	" asked for: %s\n"
2357	" redirect: %s"),
2358	asked, got->buf);
2359
2360	strbuf_reset(base);
2361	strbuf_add(base, got->buf, new_len);
2362
2363	return 1;
2364	}
2365
2366	/*
2367	* Compute the retry delay for an HTTP 429 response.
2368	* Returns a negative value if configuration is invalid (delay exceeds
2369	* http.maxRetryTime), otherwise returns the delay in seconds (>= 0).
2370	*/
2371	static long handle_rate_limit_retry(long slot_retry_after)
2372	{
2373	/* Use the slot-specific retry_after value or configured default */
2374	if (slot_retry_after >= 0) {
2375	/* Check if retry delay exceeds maximum allowed */
2376	if (slot_retry_after > http_max_retry_time) {
2377	error(_("response requested a delay greater than http.maxRetryTime (%ld > %ld seconds)"),
2378	slot_retry_after, http_max_retry_time);
2379	trace2_data_string("http", the_repository,
2380	"http/429-error", "exceeds-max-retry-time");
2381	trace2_data_intmax("http", the_repository,
2382	"http/429-requested-delay", slot_retry_after);
2383	return -1;
2384	}
2385	return slot_retry_after;
2386	} else {
2387	/* No Retry-After header provided, use configured default */
2388	if (http_retry_after > http_max_retry_time) {
2389	error(_("configured http.retryAfter exceeds http.maxRetryTime (%ld > %ld seconds)"),
2390	http_retry_after, http_max_retry_time);
2391	trace2_data_string("http", the_repository,
2392	"http/429-error", "config-exceeds-max-retry-time");
2393	return -1;
2394	}
2395	trace2_data_string("http", the_repository,
2396	"http/429-retry-source", "config-default");
2397	return http_retry_after;
2398	}
2399	}
2400
2401	static int http_request_recoverable(const char *url,
2402	void *result, int target,
2403	struct http_get_options *options)
2404	{
2405	static struct http_get_options empty_opts;
2406	int i = 3;
2407	int ret;
2408	int rate_limit_retries = http_max_retries;
2409
2410	if (!options)
2411	options = &empty_opts;
2412
2413	if (always_auth_proactively())
2414	credential_fill(the_repository, &http_auth, 1);
2415
2416	ret = http_request(url, result, target, options);
2417
2418	if (ret != HTTP_OK && ret != HTTP_REAUTH && ret != HTTP_RATE_LIMITED)
2419	return ret;
2420
2421	/* If retries are disabled and we got a 429, fail immediately */
2422	if (ret == HTTP_RATE_LIMITED && !http_max_retries)
2423	return HTTP_ERROR;
2424
2425	if (options->effective_url && options->base_url) {
2426	if (update_url_from_redirect(options->base_url,
2427	url, options->effective_url)) {
2428	credential_from_url(&http_auth, options->base_url->buf);
2429	url = options->effective_url->buf;
2430	}
2431	}
2432
2433	while ((ret == HTTP_REAUTH && --i) \|\|
2434	(ret == HTTP_RATE_LIMITED && --rate_limit_retries)) {
2435	long retry_delay = -1;
2436	/*
2437	* The previous request may have put cruft into our output stream; we
2438	* should clear it out before making our next request.
2439	*/
2440	switch (target) {
2441	case HTTP_REQUEST_STRBUF:
2442	strbuf_reset(result);
2443	break;
2444	case HTTP_REQUEST_FILE: {
2445	FILE *f = result;
2446	if (fflush(f)) {
2447	error_errno("unable to flush a file");
2448	return HTTP_START_FAILED;
2449	}
2450	rewind(f);
2451	if (ftruncate(fileno(f), 0) < 0) {
2452	error_errno("unable to truncate a file");
2453	return HTTP_START_FAILED;
2454	}
2455	break;
2456	}
2457	default:
2458	BUG("Unknown http_request target");
2459	}
2460	if (ret == HTTP_RATE_LIMITED) {
2461	retry_delay = handle_rate_limit_retry(options->retry_after);
2462	if (retry_delay < 0)
2463	return HTTP_ERROR;
2464
2465	if (retry_delay > 0) {
2466	warning(_("rate limited, waiting %ld seconds before retry"), retry_delay);
2467	trace2_data_intmax("http", the_repository,
2468	"http/retry-sleep-seconds", retry_delay);
2469	sleep(retry_delay);
2470	}
2471	} else if (ret == HTTP_REAUTH) {
2472	http_reauth_prepare(1);
2473	}
2474
2475	ret = http_request(url, result, target, options);
2476	}
2477	if (ret == HTTP_RATE_LIMITED) {
2478	trace2_data_string("http", the_repository,
2479	"http/429-error", "retries-exhausted");
2480	return HTTP_RATE_LIMITED;
2481	}
2482	return ret;
2483	}
2484
2485	int http_get_strbuf(const char *url,
2486	struct strbuf *result,
2487	struct http_get_options *options)
2488	{
2489	return http_request_recoverable(url, result, HTTP_REQUEST_STRBUF, options);
2490	}
2491
2492	/*
2493	* Downloads a URL and stores the result in the given file.
2494	*
2495	* If a previous interrupted download is detected (i.e. a previous temporary
2496	* file is still around) the download is resumed.
2497	*/
2498	int http_get_file(const char url, const char filename,
2499	struct http_get_options *options)
2500	{
2501	int ret;
2502	struct strbuf tmpfile = STRBUF_INIT;
2503	FILE *result;
2504
2505	strbuf_addf(&tmpfile, "%s.temp", filename);
2506	result = fopen(tmpfile.buf, "a");
2507	if (!result) {
2508	error("Unable to open local file %s", tmpfile.buf);
2509	ret = HTTP_ERROR;
2510	goto cleanup;
2511	}
2512
2513	ret = http_request_recoverable(url, result, HTTP_REQUEST_FILE, options);
2514	fclose(result);
2515
2516	if (ret == HTTP_OK && finalize_object_file(the_repository, tmpfile.buf, filename))
2517	ret = HTTP_ERROR;
2518	cleanup:
2519	strbuf_release(&tmpfile);
2520	return ret;
2521	}
2522
2523	int http_fetch_ref(const char base, struct ref ref)
2524	{
2525	struct http_get_options options = {0};
2526	char *url;
2527	struct strbuf buffer = STRBUF_INIT;
2528	int ret = -1;
2529
2530	options.no_cache = 1;
2531
2532	url = quote_ref_url(base, ref->name);
2533	if (http_get_strbuf(url, &buffer, &options) == HTTP_OK) {
2534	strbuf_rtrim(&buffer);
2535	if (buffer.len == the_hash_algo->hexsz)
2536	ret = get_oid_hex(buffer.buf, &ref->old_oid);
2537	else if (starts_with(buffer.buf, "ref: ")) {
2538	ref->symref = xstrdup(buffer.buf + 5);
2539	ret = 0;
2540	}
2541	}
2542
2543	strbuf_release(&buffer);
2544	free(url);
2545	return ret;
2546	}
2547
2548	/* Helpers for fetching packs */
2549	static char fetch_pack_index(unsigned char hash, const char *base_url)
2550	{
2551	char url, tmp;
2552	struct strbuf buf = STRBUF_INIT;
2553
2554	if (http_is_verbose)
2555	fprintf(stderr, "Getting index for pack %s\n", hash_to_hex(hash));
2556
2557	end_url_with_slash(&buf, base_url);
2558	strbuf_addf(&buf, "objects/pack/pack-%s.idx", hash_to_hex(hash));
2559	url = strbuf_detach(&buf, NULL);
2560
2561	/*
2562	* Don't put this into packs/, since it's just temporary and we don't
2563	* want to confuse it with our local .idx files. We'll generate our
2564	* own index if we choose to download the matching packfile.
2565	*
2566	* It's tempting to use xmks_tempfile() here, but it's important that
2567	* the file not exist, otherwise http_get_file() complains. So we
2568	* create a filename that should be unique, and then just register it
2569	* as a tempfile so that it will get cleaned up on exit.
2570	*
2571	* In theory we could hold on to the tempfile and delete these as soon
2572	* as we download the matching pack, but it would take a bit of
2573	* refactoring. Leaving them until the process ends is probably OK.
2574	*/
2575	tmp = xstrfmt("%s/tmp_pack_%s.idx",
2576	repo_get_object_directory(the_repository),
2577	hash_to_hex(hash));
2578	register_tempfile(tmp);
2579
2580	if (http_get_file(url, tmp, NULL) != HTTP_OK) {
2581	error("Unable to get pack index %s", url);
2582	FREE_AND_NULL(tmp);
2583	}
2584
2585	free(url);
2586	return tmp;
2587	}
2588
2589	static int fetch_and_setup_pack_index(struct packfile_list *packs,
2590	unsigned char *sha1,
2591	const char *base_url)
2592	{
2593	struct packed_git new_pack, p;
2594	char *tmp_idx = NULL;
2595	int ret;
2596
2597	/*
2598	* If we already have the pack locally, no need to fetch its index or
2599	* even add it to list; we already have all of its objects.
2600	*/
2601	repo_for_each_pack(the_repository, p) {
2602	if (hasheq(p->hash, sha1, the_repository->hash_algo))
2603	return 0;
2604	}
2605
2606	tmp_idx = fetch_pack_index(sha1, base_url);
2607	if (!tmp_idx)
2608	return -1;
2609
2610	new_pack = parse_pack_index(the_repository, sha1, tmp_idx);
2611	if (!new_pack) {
2612	free(tmp_idx);
2613	return -1; /* parse_pack_index() already issued error message */
2614	}
2615
2616	ret = verify_pack_index(new_pack);
2617
2618	close_pack_index(new_pack);
2619	free(tmp_idx);
2620	if (ret) {
2621	free(new_pack);
2622	return -1;
2623	}
2624
2625	packfile_list_prepend(packs, new_pack);
2626	return 0;
2627	}
2628
2629	int http_get_info_packs(const char base_url, struct packfile_list packs)
2630	{
2631	struct http_get_options options = {0};
2632	int ret = 0;
2633	char *url;
2634	const char *data;
2635	struct strbuf buf = STRBUF_INIT;
2636	struct object_id oid;
2637
2638	end_url_with_slash(&buf, base_url);
2639	strbuf_addstr(&buf, "objects/info/packs");
2640	url = strbuf_detach(&buf, NULL);
2641
2642	options.no_cache = 1;
2643	ret = http_get_strbuf(url, &buf, &options);
2644	if (ret != HTTP_OK)
2645	goto cleanup;
2646
2647	data = buf.buf;
2648	while (*data) {
2649	if (skip_prefix(data, "P pack-", &data) &&
2650	!parse_oid_hex(data, &oid, &data) &&
2651	skip_prefix(data, ".pack", &data) &&
2652	(data == '\n' \|\| data == '\0')) {
2653	fetch_and_setup_pack_index(packs, oid.hash, base_url);
2654	} else {
2655	data = strchrnul(data, '\n');
2656	}
2657	if (*data)
2658	data++; /* skip past newline */
2659	}
2660
2661	cleanup:
2662	free(url);
2663	strbuf_release(&buf);
2664	return ret;
2665	}
2666
2667	void release_http_pack_request(struct http_pack_request *preq)
2668	{
2669	if (preq->packfile) {
2670	fclose(preq->packfile);
2671	preq->packfile = NULL;
2672	}
2673	preq->slot = NULL;
2674	strbuf_release(&preq->tmpfile);
2675	curl_slist_free_all(preq->headers);
2676	free(preq->url);
2677	free(preq);
2678	}
2679
2680	static const char *default_index_pack_args[] =
2681	{"index-pack", "--stdin", NULL};
2682
2683	int finish_http_pack_request(struct http_pack_request *preq)
2684	{
2685	struct child_process ip = CHILD_PROCESS_INIT;
2686	int tmpfile_fd;
2687	int ret = 0;
2688
2689	fclose(preq->packfile);
2690	preq->packfile = NULL;
2691
2692	tmpfile_fd = xopen(preq->tmpfile.buf, O_RDONLY);
2693
2694	ip.git_cmd = 1;
2695	ip.in = tmpfile_fd;
2696	strvec_pushv(&ip.args, preq->index_pack_args ?
2697	preq->index_pack_args :
2698	default_index_pack_args);
2699
2700	if (preq->preserve_index_pack_stdout)
2701	ip.out = 0;
2702	else
2703	ip.no_stdout = 1;
2704
2705	if (run_command(&ip)) {
2706	ret = -1;
2707	goto cleanup;
2708	}
2709
2710	cleanup:
2711	close(tmpfile_fd);
2712	unlink(preq->tmpfile.buf);
2713	return ret;
2714	}
2715
2716	void http_install_packfile(struct packed_git *p,
2717	struct packfile_list *list_to_remove_from)
2718	{
2719	struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources);
2720	packfile_list_remove(list_to_remove_from, p);
2721	packfile_store_add_pack(files->packed, p);
2722	}
2723
2724	struct http_pack_request *new_http_pack_request(
2725	const unsigned char packed_git_hash, const char base_url) {
2726
2727	struct strbuf buf = STRBUF_INIT;
2728
2729	end_url_with_slash(&buf, base_url);
2730	strbuf_addf(&buf, "objects/pack/pack-%s.pack",
2731	hash_to_hex(packed_git_hash));
2732	return new_direct_http_pack_request(packed_git_hash,
2733	strbuf_detach(&buf, NULL));
2734	}
2735
2736	struct http_pack_request *new_direct_http_pack_request(
2737	const unsigned char packed_git_hash, char url)
2738	{
2739	off_t prev_posn = 0;
2740	struct http_pack_request *preq;
2741
2742	CALLOC_ARRAY(preq, 1);
2743	strbuf_init(&preq->tmpfile, 0);
2744
2745	preq->url = url;
2746
2747	odb_pack_name(the_repository, &preq->tmpfile, packed_git_hash, "pack");
2748	strbuf_addstr(&preq->tmpfile, ".temp");
2749	preq->packfile = fopen(preq->tmpfile.buf, "a");
2750	if (!preq->packfile) {
2751	error("Unable to open local file %s for pack",
2752	preq->tmpfile.buf);
2753	goto abort;
2754	}
2755
2756	preq->slot = get_active_slot();
2757	preq->headers = object_request_headers();
2758	curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEDATA, preq->packfile);
2759	curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
2760	curl_easy_setopt(preq->slot->curl, CURLOPT_URL, preq->url);
2761	curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER, preq->headers);
2762
2763	/*
2764	* If there is data present from a previous transfer attempt,
2765	* resume where it left off
2766	*/
2767	prev_posn = ftello(preq->packfile);
2768	if (prev_posn>0) {
2769	if (http_is_verbose)
2770	fprintf(stderr,
2771	"Resuming fetch of pack %s at byte %"PRIuMAX"\n",
2772	hash_to_hex(packed_git_hash),
2773	(uintmax_t)prev_posn);
2774	http_opt_request_remainder(preq->slot->curl, prev_posn);
2775	}
2776
2777	return preq;
2778
2779	abort:
2780	strbuf_release(&preq->tmpfile);
2781	free(preq->url);
2782	free(preq);
2783	return NULL;
2784	}
2785
2786	/* Helpers for fetching objects (loose) */
2787	static size_t fwrite_sha1_file(char *ptr, size_t eltsize, size_t nmemb,
2788	void *data)
2789	{
2790	unsigned char expn[4096];
2791	size_t size = eltsize * nmemb;
2792	int posn = 0;
2793	struct http_object_request *freq = data;
2794	struct active_request_slot *slot = freq->slot;
2795
2796	if (slot) {
2797	CURLcode c = curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE,
2798	&slot->http_code);
2799	if (c != CURLE_OK)
2800	BUG("curl_easy_getinfo for HTTP code failed: %s",
2801	curl_easy_strerror(c));
2802	if (slot->http_code >= 300)
2803	return nmemb;
2804	}
2805
2806	do {
2807	ssize_t retval = xwrite(freq->localfile,
2808	(char *) ptr + posn, size - posn);
2809	if (retval < 0)
2810	return posn / eltsize;
2811	posn += retval;
2812	} while (posn < size);
2813
2814	freq->stream.avail_in = size;
2815	freq->stream.next_in = (void *)ptr;
2816	do {
2817	freq->stream.next_out = expn;
2818	freq->stream.avail_out = sizeof(expn);
2819	freq->zret = git_inflate(&freq->stream, Z_SYNC_FLUSH);
2820	git_hash_update(&freq->c, expn,
2821	sizeof(expn) - freq->stream.avail_out);
2822	} while (freq->stream.avail_in && freq->zret == Z_OK);
2823	return nmemb;
2824	}
2825
2826	struct http_object_request new_http_object_request(const char base_url,
2827	const struct object_id *oid)
2828	{
2829	struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources);
2830	char *hex = oid_to_hex(oid);
2831	struct strbuf filename = STRBUF_INIT;
2832	struct strbuf prevfile = STRBUF_INIT;
2833	int prevlocal;
2834	char prev_buf[PREV_BUF_SIZE];
2835	ssize_t prev_read = 0;
2836	off_t prev_posn = 0;
2837	struct http_object_request *freq;
2838
2839	CALLOC_ARRAY(freq, 1);
2840	strbuf_init(&freq->tmpfile, 0);
2841	oidcpy(&freq->oid, oid);
2842	freq->localfile = -1;
2843
2844	odb_loose_path(files->loose, &filename, oid);
2845	strbuf_addf(&freq->tmpfile, "%s.temp", filename.buf);
2846
2847	strbuf_addf(&prevfile, "%s.prev", filename.buf);
2848	unlink_or_warn(prevfile.buf);
2849	rename(freq->tmpfile.buf, prevfile.buf);
2850	unlink_or_warn(freq->tmpfile.buf);
2851	strbuf_release(&filename);
2852
2853	if (freq->localfile != -1)
2854	error("fd leakage in start: %d", freq->localfile);
2855	freq->localfile = open(freq->tmpfile.buf,
2856	O_WRONLY \| O_CREAT \| O_EXCL, 0666);
2857	/*
2858	* This could have failed due to the "lazy directory creation";
2859	* try to mkdir the last path component.
2860	*/
2861	if (freq->localfile < 0 && errno == ENOENT) {
2862	char *dir = strrchr(freq->tmpfile.buf, '/');
2863	if (dir) {
2864	*dir = 0;
2865	mkdir(freq->tmpfile.buf, 0777);
2866	*dir = '/';
2867	}
2868	freq->localfile = open(freq->tmpfile.buf,
2869	O_WRONLY \| O_CREAT \| O_EXCL, 0666);
2870	}
2871
2872	if (freq->localfile < 0) {
2873	error_errno("Couldn't create temporary file %s",
2874	freq->tmpfile.buf);
2875	goto abort;
2876	}
2877
2878	git_inflate_init(&freq->stream);
2879
2880	the_hash_algo->init_fn(&freq->c);
2881
2882	freq->url = get_remote_object_url(base_url, hex, 0);
2883
2884	/*
2885	* If a previous temp file is present, process what was already
2886	* fetched.
2887	*/
2888	prevlocal = open(prevfile.buf, O_RDONLY);
2889	if (prevlocal != -1) {
2890	do {
2891	prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
2892	if (prev_read>0) {
2893	if (fwrite_sha1_file(prev_buf,
2894	1,
2895	prev_read,
2896	freq) == prev_read) {
2897	prev_posn += prev_read;
2898	} else {
2899	prev_read = -1;
2900	}
2901	}
2902	} while (prev_read > 0);
2903	close(prevlocal);
2904	}
2905	unlink_or_warn(prevfile.buf);
2906	strbuf_release(&prevfile);
2907
2908	/*
2909	* Reset inflate/SHA1 if there was an error reading the previous temp
2910	* file; also rewind to the beginning of the local file.
2911	*/
2912	if (prev_read == -1) {
2913	git_inflate_end(&freq->stream);
2914	memset(&freq->stream, 0, sizeof(freq->stream));
2915	git_inflate_init(&freq->stream);
2916	the_hash_algo->init_fn(&freq->c);
2917	if (prev_posn>0) {
2918	prev_posn = 0;
2919	lseek(freq->localfile, 0, SEEK_SET);
2920	if (ftruncate(freq->localfile, 0) < 0) {
2921	error_errno("Couldn't truncate temporary file %s",
2922	freq->tmpfile.buf);
2923	goto abort;
2924	}
2925	}
2926	}
2927
2928	freq->slot = get_active_slot();
2929	freq->headers = object_request_headers();
2930
2931	curl_easy_setopt(freq->slot->curl, CURLOPT_WRITEDATA, freq);
2932	curl_easy_setopt(freq->slot->curl, CURLOPT_FAILONERROR, 0L);
2933	curl_easy_setopt(freq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
2934	curl_easy_setopt(freq->slot->curl, CURLOPT_ERRORBUFFER, freq->errorstr);
2935	curl_easy_setopt(freq->slot->curl, CURLOPT_URL, freq->url);
2936	curl_easy_setopt(freq->slot->curl, CURLOPT_HTTPHEADER, freq->headers);
2937
2938	/*
2939	* If we have successfully processed data from a previous fetch
2940	* attempt, only fetch the data we don't already have.
2941	*/
2942	if (prev_posn>0) {
2943	if (http_is_verbose)
2944	fprintf(stderr,
2945	"Resuming fetch of object %s at byte %"PRIuMAX"\n",
2946	hex, (uintmax_t)prev_posn);
2947	http_opt_request_remainder(freq->slot->curl, prev_posn);
2948	}
2949
2950	return freq;
2951
2952	abort:
2953	strbuf_release(&prevfile);
2954	free(freq->url);
2955	free(freq);
2956	return NULL;
2957	}
2958
2959	void process_http_object_request(struct http_object_request *freq)
2960	{
2961	if (!freq->slot)
2962	return;
2963	freq->curl_result = freq->slot->curl_result;
2964	freq->http_code = freq->slot->http_code;
2965	freq->slot = NULL;
2966	}
2967
2968	int finish_http_object_request(struct http_object_request *freq)
2969	{
2970	struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources);
2971	struct stat st;
2972	struct strbuf filename = STRBUF_INIT;
2973
2974	close(freq->localfile);
2975	freq->localfile = -1;
2976
2977	process_http_object_request(freq);
2978
2979	if (freq->http_code == 416) {
2980	warning("requested range invalid; we may already have all the data.");
2981	} else if (freq->curl_result != CURLE_OK) {
2982	if (stat(freq->tmpfile.buf, &st) == 0)
2983	if (st.st_size == 0)
2984	unlink_or_warn(freq->tmpfile.buf);
2985	return -1;
2986	}
2987
2988	git_hash_final_oid(&freq->real_oid, &freq->c);
2989	if (freq->zret != Z_STREAM_END) {
2990	unlink_or_warn(freq->tmpfile.buf);
2991	return -1;
2992	}
2993	if (!oideq(&freq->oid, &freq->real_oid)) {
2994	unlink_or_warn(freq->tmpfile.buf);
2995	return -1;
2996	}
2997	odb_loose_path(files->loose, &filename, &freq->oid);
2998	freq->rename = finalize_object_file(the_repository, freq->tmpfile.buf, filename.buf);
2999	strbuf_release(&filename);
3000
3001	return freq->rename;
3002	}
3003
3004	void abort_http_object_request(struct http_object_request **freq_p)
3005	{
3006	struct http_object_request freq = freq_p;
3007	unlink_or_warn(freq->tmpfile.buf);
3008
3009	release_http_object_request(freq_p);
3010	}
3011
3012	void release_http_object_request(struct http_object_request **freq_p)
3013	{
3014	struct http_object_request freq = freq_p;
3015	if (freq->localfile != -1) {
3016	close(freq->localfile);
3017	freq->localfile = -1;
3018	}
3019	FREE_AND_NULL(freq->url);
3020	if (freq->slot) {
3021	freq->slot->callback_func = NULL;
3022	freq->slot->callback_data = NULL;
3023	release_active_slot(freq->slot);
3024	freq->slot = NULL;
3025	}
3026	curl_slist_free_all(freq->headers);
3027	strbuf_release(&freq->tmpfile);
3028	git_inflate_end(&freq->stream);
3029
3030	free(freq);
3031	*freq_p = NULL;
3032	}