@matheusbsbs / matheus-bagliano / commits / 640657ffd0

http: add support for HTTP 429 rate limit retries

Add retry logic for HTTP 429 (Too Many Requests) responses to handle server-side rate limiting gracefully. When Git's HTTP client receives a 429 response, it can now automatically retry the request after an appropriate delay, respecting the server's rate limits. The implementation supports the RFC-compliant Retry-After header in both delay-seconds (integer) and HTTP-date (RFC 2822) formats. If a past date is provided, Git retries immediately without waiting. Retry behavior is controlled by three new configuration options (http.maxRetries, http.retryAfter, and http.maxRetryTime) which are documented in git-config(1). The retry logic implements a fail-fast approach: if any delay (whether from server header or configuration) exceeds maxRetryTime, Git fails immediately with a clear error message rather than capping the delay. This provides better visibility into rate limiting issues. The implementation includes extensive test coverage for basic retry behavior, Retry-After header formats (integer and HTTP-date), configuration combinations, maxRetryTime limits, invalid header handling, environment variable overrides, and edge cases. Signed-off-by: Vaidas Pilkauskas <vaidas.pilkauskas@shopify.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Vaidas Pilkauskas committed Mar 17, 2026 at 13:00 UTC 640657ffd06999ec1ec3b1d030b7f5aac6b7f57b

10 files changed +551 -21

Documentation/config/http.adoc

+26

@@ -315,6 +315,32 @@ http.keepAliveCount::

315 unset, curl's default value is used. Can be overridden by the

316 `GIT_HTTP_KEEPALIVE_COUNT` environment variable.

317

318 +http.retryAfter::

319 + Default wait time in seconds before retrying when a server returns

320 + HTTP 429 (Too Many Requests) without a Retry-After header.

321 + Defaults to 0 (retry immediately). When a Retry-After header is

322 + present, its value takes precedence over this setting; however,

323 + automatic use of the server-provided `Retry-After` header requires

324 + libcurl 7.66.0 or later. On older versions, configure this setting

325 + manually to control the retry delay. Can be overridden by the

326 + `GIT_HTTP_RETRY_AFTER` environment variable.

327 + See also `http.maxRetries` and `http.maxRetryTime`.

328 +

329 +http.maxRetries::

330 + Maximum number of times to retry after receiving HTTP 429 (Too Many

331 + Requests) responses. Set to 0 (the default) to disable retries.

332 + Can be overridden by the `GIT_HTTP_MAX_RETRIES` environment variable.

333 + See also `http.retryAfter` and `http.maxRetryTime`.

334 +

335 +http.maxRetryTime::

336 + Maximum time in seconds to wait for a single retry attempt when

337 + handling HTTP 429 (Too Many Requests) responses. If the server

338 + requests a delay (via Retry-After header) or if `http.retryAfter`

339 + is configured with a value that exceeds this maximum, Git will fail

340 + immediately rather than waiting. Default is 300 seconds (5 minutes).

341 + Can be overridden by the `GIT_HTTP_MAX_RETRY_TIME` environment

342 + variable. See also `http.retryAfter` and `http.maxRetries`.

343 +

344 http.noEPSV::

345 A boolean which disables using of EPSV ftp command by curl.

346 This can be helpful with some "poor" ftp servers which don't

git-curl-compat.h

@@ -37,6 +37,14 @@

37 #define GIT_CURL_NEED_TRANSFER_ENCODING_HEADER

38 #endif

40 +/**

41 + * CURLINFO_RETRY_AFTER was added in 7.66.0, released in September 2019.

42 + * It allows curl to automatically parse Retry-After headers.

43 + */

44 +#if LIBCURL_VERSION_NUM >= 0x074200

45 +#define GIT_CURL_HAVE_CURLINFO_RETRY_AFTER 1

46 +#endif

47 +

48 /**

49 * CURLOPT_PROTOCOLS_STR and CURLOPT_REDIR_PROTOCOLS_STR were added in 7.85.0,

50 * released in August 2022.

http.c

+123 -21

@@ -22,6 +22,8 @@

22 #include "object-file.h"

23 #include "odb.h"

24 #include "tempfile.h"

25 +#include "date.h"

26 +#include "trace2.h"

28 static struct trace_key trace_curl = TRACE_KEY_INIT(CURL);

29 static int trace_curl_data = 1;

@@ -149,6 +151,11 @@ static char *cached_accept_language;

151 static char *http_ssl_backend;

152

153 static int http_schannel_check_revoke = 1;

154 +

155 +static long http_retry_after = 0;

156 +static long http_max_retries = 0;

157 +static long http_max_retry_time = 300;

158 +

159 /*

160 * With the backend being set to `schannel`, setting sslCAinfo would override

161 * the Certificate Store in cURL v7.60.0 and later, which is not what we want

@@ -209,7 +216,7 @@ static inline int is_hdr_continuation(const char *ptr, const size_t size)

216 return size && (*ptr == ' ' || *ptr == '\t');

217 }

218

212 -static size_t fwrite_wwwauth(char *ptr, size_t eltsize, size_t nmemb, void *p UNUSED)

219 +static size_t fwrite_wwwauth(char *ptr, size_t eltsize, size_t nmemb, void *p MAYBE_UNUSED)

220 {

221 size_t size = eltsize * nmemb;

222 struct strvec *values = &http_auth.wwwauth_headers;

@@ -575,6 +582,21 @@ static int http_options(const char *var, const char *value,

582 return 0;

583 }

584

585 + if (!strcmp("http.retryafter", var)) {

586 + http_retry_after = git_config_int(var, value, ctx->kvi);

587 + return 0;

588 + }

589 +

590 + if (!strcmp("http.maxretries", var)) {

591 + http_max_retries = git_config_int(var, value, ctx->kvi);

592 + return 0;

593 + }

594 +

595 + if (!strcmp("http.maxretrytime", var)) {

596 + http_max_retry_time = git_config_int(var, value, ctx->kvi);

597 + return 0;

598 + }

599 +

600 /* Fall back on the default ones */

601 return git_default_config(var, value, ctx, data);

602 }

@@ -1422,6 +1444,10 @@ void http_init(struct remote *remote, const char *url, int proactive_auth)

1444 set_long_from_env(&curl_tcp_keepintvl, "GIT_TCP_KEEPINTVL");

1445 set_long_from_env(&curl_tcp_keepcnt, "GIT_TCP_KEEPCNT");

1446

1447 + set_long_from_env(&http_retry_after, "GIT_HTTP_RETRY_AFTER");

1448 + set_long_from_env(&http_max_retries, "GIT_HTTP_MAX_RETRIES");

1449 + set_long_from_env(&http_max_retry_time, "GIT_HTTP_MAX_RETRY_TIME");

1450 +

1451 curl_default = get_curl_handle();

1452 }

1453

@@ -1871,6 +1897,10 @@ static int handle_curl_result(struct slot_results *results)

1897 }

1898 return HTTP_REAUTH;

1899 }

1900 + } else if (results->http_code == 429) {

1901 + trace2_data_intmax("http", the_repository, "http/429-retry-after",

1902 + results->retry_after);

1903 + return HTTP_RATE_LIMITED;

1904 } else {

1905 if (results->http_connectcode == 407)

1906 credential_reject(the_repository, &proxy_auth);

@@ -1886,6 +1916,7 @@ int run_one_slot(struct active_request_slot *slot,

1916 struct slot_results *results)

1917 {

1918 slot->results = results;

1919 +

1920 if (!start_active_slot(slot)) {

1921 xsnprintf(curl_errorstr, sizeof(curl_errorstr),

1922 "failed to start HTTP request");

@@ -2119,10 +2150,10 @@ static void http_opt_request_remainder(CURL *curl, off_t pos)

2150

2151 static int http_request(const char *url,

2152 void *result, int target,

2122 - const struct http_get_options *options)

2153 + struct http_get_options *options)

2154 {

2155 struct active_request_slot *slot;

2125 - struct slot_results results;

2156 + struct slot_results results = { .retry_after = -1 };

2157 struct curl_slist *headers = http_copy_default_headers();

2158 struct strbuf buf = STRBUF_INIT;

2159 const char *accept_language;

@@ -2156,22 +2187,19 @@ static int http_request(const char *url,

2187 headers = curl_slist_append(headers, accept_language);

2188

2189 strbuf_addstr(&buf, "Pragma:");

2159 - if (options && options->no_cache)

2190 + if (options->no_cache)

2191 strbuf_addstr(&buf, " no-cache");

2161 - if (options && options->initial_request &&

2192 + if (options->initial_request &&

2193 http_follow_config == HTTP_FOLLOW_INITIAL)

2194 curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 1L);

2195

2196 headers = curl_slist_append(headers, buf.buf);

2197

2198 /* Add additional headers here */

2168 - if (options && options->extra_headers) {

2199 + if (options->extra_headers) {

2200 const struct string_list_item *item;

2170 - if (options && options->extra_headers) {

2171 - for_each_string_list_item(item, options->extra_headers) {

2172 - headers = curl_slist_append(headers, item->string);

2173 - }

2174 - }

2201 + for_each_string_list_item(item, options->extra_headers)

2202 + headers = curl_slist_append(headers, item->string);

2203 }

2204

2205 headers = http_append_auth_header(&http_auth, headers);

@@ -2183,7 +2211,18 @@ static int http_request(const char *url,

2211

2212 ret = run_one_slot(slot, &results);

2213

2186 - if (options && options->content_type) {

2214 +#ifdef GIT_CURL_HAVE_CURLINFO_RETRY_AFTER

2215 + if (ret == HTTP_RATE_LIMITED) {

2216 + curl_off_t retry_after;

2217 + if (curl_easy_getinfo(slot->curl, CURLINFO_RETRY_AFTER,

2218 + &retry_after) == CURLE_OK && retry_after > 0)

2219 + results.retry_after = (long)retry_after;

2220 + }

2221 +#endif

2222 +

2223 + options->retry_after = results.retry_after;

2224 +

2225 + if (options->content_type) {

2226 struct strbuf raw = STRBUF_INIT;

2227 curlinfo_strbuf(slot->curl, CURLINFO_CONTENT_TYPE, &raw);

2228 extract_content_type(&raw, options->content_type,

@@ -2191,7 +2230,7 @@ static int http_request(const char *url,

2230 strbuf_release(&raw);

2231 }

2232

2194 - if (options && options->effective_url)

2233 + if (options->effective_url)

2234 curlinfo_strbuf(slot->curl, CURLINFO_EFFECTIVE_URL,

2235 options->effective_url);

2236

@@ -2253,22 +2292,66 @@ static int update_url_from_redirect(struct strbuf *base,

2292 return 1;

2293 }

2294

2256 -static int http_request_reauth(const char *url,

2295 +/*

2296 + * Compute the retry delay for an HTTP 429 response.

2297 + * Returns a negative value if configuration is invalid (delay exceeds

2298 + * http.maxRetryTime), otherwise returns the delay in seconds (>= 0).

2299 + */

2300 +static long handle_rate_limit_retry(long slot_retry_after)

2301 +{

2302 + /* Use the slot-specific retry_after value or configured default */

2303 + if (slot_retry_after >= 0) {

2304 + /* Check if retry delay exceeds maximum allowed */

2305 + if (slot_retry_after > http_max_retry_time) {

2306 + error(_("response requested a delay greater than http.maxRetryTime (%ld > %ld seconds)"),

2307 + slot_retry_after, http_max_retry_time);

2308 + trace2_data_string("http", the_repository,

2309 + "http/429-error", "exceeds-max-retry-time");

2310 + trace2_data_intmax("http", the_repository,

2311 + "http/429-requested-delay", slot_retry_after);

2312 + return -1;

2313 + }

2314 + return slot_retry_after;

2315 + } else {

2316 + /* No Retry-After header provided, use configured default */

2317 + if (http_retry_after > http_max_retry_time) {

2318 + error(_("configured http.retryAfter exceeds http.maxRetryTime (%ld > %ld seconds)"),

2319 + http_retry_after, http_max_retry_time);

2320 + trace2_data_string("http", the_repository,

2321 + "http/429-error", "config-exceeds-max-retry-time");

2322 + return -1;

2323 + }

2324 + trace2_data_string("http", the_repository,

2325 + "http/429-retry-source", "config-default");

2326 + return http_retry_after;

2327 + }

2328 +}

2329 +

2330 +static int http_request_recoverable(const char *url,

2331 void *result, int target,

2332 struct http_get_options *options)

2333 {

2334 + static struct http_get_options empty_opts;

2335 int i = 3;

2336 int ret;

2337 + int rate_limit_retries = http_max_retries;

2338 +

2339 + if (!options)

2340 + options = &empty_opts;

2341

2342 if (always_auth_proactively())

2343 credential_fill(the_repository, &http_auth, 1);

2344

2345 ret = http_request(url, result, target, options);

2346

2268 - if (ret != HTTP_OK && ret != HTTP_REAUTH)

2347 + if (ret != HTTP_OK && ret != HTTP_REAUTH && ret != HTTP_RATE_LIMITED)

2348 return ret;

2349

2271 - if (options && options->effective_url && options->base_url) {

2350 + /* If retries are disabled and we got a 429, fail immediately */

2351 + if (ret == HTTP_RATE_LIMITED && !http_max_retries)

2352 + return HTTP_ERROR;

2353 +

2354 + if (options->effective_url && options->base_url) {

2355 if (update_url_from_redirect(options->base_url,

2356 url, options->effective_url)) {

2357 credential_from_url(&http_auth, options->base_url->buf);

@@ -2276,7 +2359,9 @@ static int http_request_reauth(const char *url,

2359 }

2360 }

2361

2279 - while (ret == HTTP_REAUTH && --i) {

2362 + while ((ret == HTTP_REAUTH && --i) ||

2363 + (ret == HTTP_RATE_LIMITED && --rate_limit_retries)) {

2364 + long retry_delay = -1;

2365 /*

2366 * The previous request may have put cruft into our output stream; we

2367 * should clear it out before making our next request.

@@ -2301,11 +2386,28 @@ static int http_request_reauth(const char *url,

2386 default:

2387 BUG("Unknown http_request target");

2388 }

2304 -

2305 - credential_fill(the_repository, &http_auth, 1);

2389 + if (ret == HTTP_RATE_LIMITED) {

2390 + retry_delay = handle_rate_limit_retry(options->retry_after);

2391 + if (retry_delay < 0)

2392 + return HTTP_ERROR;

2393 +

2394 + if (retry_delay > 0) {

2395 + warning(_("rate limited, waiting %ld seconds before retry"), retry_delay);

2396 + trace2_data_intmax("http", the_repository,

2397 + "http/retry-sleep-seconds", retry_delay);

2398 + sleep(retry_delay);

2399 + }

2400 + } else if (ret == HTTP_REAUTH) {

2401 + credential_fill(the_repository, &http_auth, 1);

2402 + }

2403

2404 ret = http_request(url, result, target, options);

2405 }

2406 + if (ret == HTTP_RATE_LIMITED) {

2407 + trace2_data_string("http", the_repository,

2408 + "http/429-error", "retries-exhausted");

2409 + return HTTP_RATE_LIMITED;

2410 + }

2411 return ret;

2412 }

2413

@@ -2313,7 +2415,7 @@ int http_get_strbuf(const char *url,

2415 struct strbuf *result,

2416 struct http_get_options *options)

2417 {

2316 - return http_request_reauth(url, result, HTTP_REQUEST_STRBUF, options);

2418 + return http_request_recoverable(url, result, HTTP_REQUEST_STRBUF, options);

2419 }

2420

2421 /*

@@ -2337,7 +2439,7 @@ int http_get_file(const char *url, const char *filename,

2439 goto cleanup;

2440 }

2441

2340 - ret = http_request_reauth(url, result, HTTP_REQUEST_FILE, options);

2442 + ret = http_request_recoverable(url, result, HTTP_REQUEST_FILE, options);

2443 fclose(result);

2444

2445 if (ret == HTTP_OK && finalize_object_file(the_repository, tmpfile.buf, filename))

http.h

@@ -20,6 +20,7 @@ struct slot_results {

20 long http_code;

21 long auth_avail;

22 long http_connectcode;

23 + long retry_after;

24 };

26 struct active_request_slot {

@@ -157,6 +158,13 @@ struct http_get_options {

158 * request has completed.

159 */

160 struct string_list *extra_headers;

161 +

162 + /*

163 + * After a request completes, contains the Retry-After delay in seconds

164 + * if the server returned HTTP 429 with a Retry-After header (requires

165 + * libcurl 7.66.0 or later), or -1 if no such header was present.

166 + */

167 + long retry_after;

168 };

169

170 /* Return values for http_get_*() */

@@ -167,6 +175,7 @@ struct http_get_options {

175 #define HTTP_REAUTH 4

176 #define HTTP_NOAUTH 5

177 #define HTTP_NOMATCHPUBLICKEY 6

178 +#define HTTP_RATE_LIMITED 7

179

180 /*

181 * Requests a URL and stores the result in a strbuf.

remote-curl.c

+11

@@ -529,6 +529,17 @@ static struct discovery *discover_refs(const char *service, int for_push)

529 show_http_message(&type, &charset, &buffer);

530 die(_("unable to access '%s' with http.pinnedPubkey configuration: %s"),

531 transport_anonymize_url(url.buf), curl_errorstr);

532 + case HTTP_RATE_LIMITED:

533 + if (http_options.retry_after > 0) {

534 + show_http_message(&type, &charset, &buffer);

535 + die(_("rate limited by '%s', please try again in %ld seconds"),

536 + transport_anonymize_url(url.buf),

537 + http_options.retry_after);

538 + } else {

539 + show_http_message(&type, &charset, &buffer);

540 + die(_("rate limited by '%s', please try again later"),

541 + transport_anonymize_url(url.buf));

542 + }

543 default:

544 show_http_message(&type, &charset, &buffer);

545 die(_("unable to access '%s': %s"),

t/lib-httpd.sh

@@ -167,6 +167,7 @@ prepare_httpd() {

167 install_script error.sh

168 install_script apply-one-time-script.sh

169 install_script nph-custom-auth.sh

170 + install_script http-429.sh

171

172 ln -s "$LIB_HTTPD_MODULE_PATH" "$HTTPD_ROOT_PATH/modules"

173

t/lib-httpd/apache.conf

@@ -139,6 +139,10 @@ SetEnv PERL_PATH ${PERL_PATH}

139 SetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH}

140 SetEnv GIT_HTTP_EXPORT_ALL

141 </LocationMatch>

142 +<LocationMatch /http_429/>

143 + SetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH}

144 + SetEnv GIT_HTTP_EXPORT_ALL

145 +</LocationMatch>

146 <LocationMatch /smart_v0/>

147 SetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH}

148 SetEnv GIT_HTTP_EXPORT_ALL

@@ -160,6 +164,7 @@ ScriptAlias /broken_smart/ broken-smart-http.sh/

164 ScriptAlias /error_smart/ error-smart-http.sh/

165 ScriptAlias /error/ error.sh/

166 ScriptAliasMatch /one_time_script/(.*) apply-one-time-script.sh/$1

167 +ScriptAliasMatch /http_429/(.*) http-429.sh/$1

168 ScriptAliasMatch /custom_auth/(.*) nph-custom-auth.sh/$1

169 <Directory ${GIT_EXEC_PATH}>

170 Options FollowSymlinks

@@ -185,6 +190,9 @@ ScriptAliasMatch /custom_auth/(.*) nph-custom-auth.sh/$1

190 <Files apply-one-time-script.sh>

191 Options ExecCGI

192 </Files>

193 +<Files http-429.sh>

194 + Options ExecCGI

195 +</Files>

196 <Files ${GIT_EXEC_PATH}/git-http-backend>

197 Options ExecCGI

198 </Files>

t/lib-httpd/http-429.sh new

+98

@@ -0,0 +1,98 @@

1 +#!/bin/sh

2 +

3 +# Script to return HTTP 429 Too Many Requests responses for testing retry logic.

4 +# Usage: /http_429/<test-context>/<retry-after-value>/<repo-path>

5 +#

6 +# The test-context is a unique identifier for each test to isolate state files.

7 +# The retry-after-value can be:

8 +# - A number (e.g., "1", "2", "100") - sets Retry-After header to that many seconds

9 +# - "none" - no Retry-After header

10 +# - "invalid" - invalid Retry-After format

11 +# - "permanent" - always return 429 (never succeed)

12 +# - An HTTP-date string (RFC 2822 format) - sets Retry-After to that date

13 +#

14 +# On first call, returns 429. On subsequent calls (after retry), forwards to git-http-backend

15 +# unless retry-after-value is "permanent".

16 +

17 +# Extract test context, retry-after value and repo path from PATH_INFO

18 +# PATH_INFO format: /<test-context>/<retry-after-value>/<repo-path>

19 +path_info="${PATH_INFO#/}" # Remove leading slash

20 +test_context="${path_info%%/*}" # Get first component (test context)

21 +remaining="${path_info#*/}" # Get rest

22 +retry_after="${remaining%%/*}" # Get second component (retry-after value)

23 +repo_path="${remaining#*/}" # Get rest (repo path)

24 +

25 +# Extract repository name from repo_path (e.g., "repo.git" from "repo.git/info/refs")

26 +# The repo name is the first component before any "/"

27 +repo_name="${repo_path%%/*}"

28 +

29 +# Use current directory (HTTPD_ROOT_PATH) for state file

30 +# Create a safe filename from test_context, retry_after and repo_name

31 +# This ensures all requests for the same test context share the same state file

32 +safe_name=$(echo "${test_context}-${retry_after}-${repo_name}" | tr '/' '_' | tr -cd 'a-zA-Z0-9_-')

33 +state_file="http-429-state-${safe_name}"

34 +

35 +# Check if this is the first call (no state file exists)

36 +if test -f "$state_file"

37 +then

38 + # Already returned 429 once, forward to git-http-backend

39 + # Set PATH_INFO to just the repo path (without retry-after value)

40 + # Set GIT_PROJECT_ROOT so git-http-backend can find the repository

41 + # Use exec to replace this process so git-http-backend gets the updated environment

42 + PATH_INFO="/$repo_path"

43 + export PATH_INFO

44 + # GIT_PROJECT_ROOT points to the document root where repositories are stored

45 + # The script runs from HTTPD_ROOT_PATH, and www/ is the document root

46 + if test -z "$GIT_PROJECT_ROOT"

47 + then

48 + # Construct path: current directory (HTTPD_ROOT_PATH) + /www

49 + GIT_PROJECT_ROOT="$(pwd)/www"

50 + export GIT_PROJECT_ROOT

51 + fi

52 + exec "$GIT_EXEC_PATH/git-http-backend"

53 +fi

54 +

55 +# Mark that we've returned 429

56 +touch "$state_file"

57 +

58 +# Output HTTP 429 response

59 +printf "Status: 429 Too Many Requests\r\n"

60 +

61 +# Set Retry-After header based on retry_after value

62 +case "$retry_after" in

63 + none)

64 + # No Retry-After header

65 + ;;

66 + invalid)

67 + printf "Retry-After: invalid-format-123abc\r\n"

68 + ;;

69 + permanent)

70 + # Always return 429, don't set state file for success

71 + rm -f "$state_file"

72 + printf "Retry-After: 1\r\n"

73 + printf "Content-Type: text/plain\r\n"

74 + printf "\r\n"

75 + printf "Permanently rate limited\n"

76 + exit 0

77 + ;;

78 + *)

79 + # Check if it's a number

80 + case "$retry_after" in

81 + [0-9]*)

82 + # Numeric value

83 + printf "Retry-After: %s\r\n" "$retry_after"

84 + ;;

85 + *)

86 + # Assume it's an HTTP-date format (passed as-is, URL decoded)

87 + # Apache may URL-encode the path, so decode common URL-encoded characters

88 + # %20 = space, %2C = comma, %3A = colon

89 + retry_value=$(echo "$retry_after" | sed -e 's/%20/ /g' -e 's/%2C/,/g' -e 's/%3A/:/g')

90 + printf "Retry-After: %s\r\n" "$retry_value"

91 + ;;

92 + esac

93 + ;;

94 +esac

95 +

96 +printf "Content-Type: text/plain\r\n"

97 +printf "\r\n"

98 +printf "Rate limited\n"

t/meson.build

@@ -700,6 +700,7 @@ integration_tests = [

700 't5581-http-curl-verbose.sh',

701 't5582-fetch-negative-refspec.sh',

702 't5583-push-branches.sh',

703 + 't5584-http-429-retry.sh',

704 't5600-clone-fail-cleanup.sh',

705 't5601-clone.sh',

706 't5602-clone-remote-exec.sh',

t/t5584-http-429-retry.sh new

+266

@@ -0,0 +1,266 @@

1 +#!/bin/sh

2 +

3 +test_description='test HTTP 429 Too Many Requests retry logic'

4 +

5 +. ./test-lib.sh

6 +

7 +. "$TEST_DIRECTORY"/lib-httpd.sh

8 +

9 +start_httpd

10 +

11 +test_expect_success 'setup test repository' '

12 + test_commit initial &&

13 + git clone --bare . "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" &&

14 + git --git-dir="$HTTPD_DOCUMENT_ROOT_PATH/repo.git" config http.receivepack true

15 +'

16 +

17 +# This test suite uses a special HTTP 429 endpoint at /http_429/ that simulates

18 +# rate limiting. The endpoint format is:

19 +# /http_429/<test-context>/<retry-after-value>/<repo-path>

20 +# The http-429.sh script (in t/lib-httpd) returns a 429 response with the

21 +# specified Retry-After header on the first request for each test context,

22 +# then forwards subsequent requests to git-http-backend. Each test context

23 +# is isolated, allowing multiple tests to run independently.

24 +

25 +test_expect_success 'HTTP 429 with retries disabled (maxRetries=0) fails immediately' '

26 + # Set maxRetries to 0 (disabled)

27 + test_config http.maxRetries 0 &&

28 + test_config http.retryAfter 1 &&

29 +

30 + # Should fail immediately without any retry attempt

31 + test_must_fail git ls-remote "$HTTPD_URL/http_429/retries-disabled/1/repo.git" 2>err &&

32 +

33 + # Verify no retry happened (no "waiting" message in stderr)

34 + test_grep ! -i "waiting.*retry" err

35 +'

36 +

37 +test_expect_success 'HTTP 429 permanent should fail after max retries' '

38 + # Enable retries with a limit

39 + test_config http.maxRetries 2 &&

40 +

41 + # Git should retry but eventually fail when 429 persists

42 + test_must_fail git ls-remote "$HTTPD_URL/http_429/permanent-fail/permanent/repo.git" 2>err

43 +'

44 +

45 +test_expect_success 'HTTP 429 with Retry-After is retried and succeeds' '

46 + # Enable retries

47 + test_config http.maxRetries 3 &&

48 +

49 + # Git should retry after receiving 429 and eventually succeed

50 + git ls-remote "$HTTPD_URL/http_429/retry-succeeds/1/repo.git" >output 2>err &&

51 + test_grep "refs/heads/" output

52 +'

53 +

54 +test_expect_success 'HTTP 429 without Retry-After uses configured default' '

55 + # Enable retries and configure default delay

56 + test_config http.maxRetries 3 &&

57 + test_config http.retryAfter 1 &&

58 +

59 + # Git should retry using configured default and succeed

60 + git ls-remote "$HTTPD_URL/http_429/no-retry-after-header/none/repo.git" >output 2>err &&

61 + test_grep "refs/heads/" output

62 +'

63 +

64 +test_expect_success 'HTTP 429 retry delays are respected' '

65 + # Enable retries

66 + test_config http.maxRetries 3 &&

67 +

68 + # Time the operation - it should take at least 2 seconds due to retry delay

69 + start=$(test-tool date getnanos) &&

70 + git ls-remote "$HTTPD_URL/http_429/retry-delays-respected/2/repo.git" >output 2>err &&

71 + duration=$(test-tool date getnanos $start) &&

72 +

73 + # Verify it took at least 2 seconds (allowing some tolerance)

74 + duration_int=${duration%.*} &&

75 + test "$duration_int" -ge 1 &&

76 + test_grep "refs/heads/" output

77 +'

78 +

79 +test_expect_success 'HTTP 429 fails immediately if Retry-After exceeds http.maxRetryTime' '

80 + # Configure max retry time to 3 seconds (much less than requested 100)

81 + test_config http.maxRetries 3 &&

82 + test_config http.maxRetryTime 3 &&

83 +

84 + # Should fail immediately without waiting

85 + start=$(test-tool date getnanos) &&

86 + test_must_fail git ls-remote "$HTTPD_URL/http_429/retry-after-exceeds-max-time/100/repo.git" 2>err &&

87 + duration=$(test-tool date getnanos $start) &&

88 +

89 + # Should fail quickly (no 100 second wait)

90 + duration_int=${duration%.*} &&

91 + test "$duration_int" -lt 99 &&

92 + test_grep "greater than http.maxRetryTime" err

93 +'

94 +

95 +test_expect_success 'HTTP 429 fails if configured http.retryAfter exceeds http.maxRetryTime' '

96 + # Test misconfiguration: retryAfter > maxRetryTime

97 + # Configure retryAfter larger than maxRetryTime

98 + test_config http.maxRetries 3 &&

99 + test_config http.retryAfter 100 &&

100 + test_config http.maxRetryTime 5 &&

101 +

102 + # Should fail immediately with configuration error

103 + start=$(test-tool date getnanos) &&

104 + test_must_fail git ls-remote "$HTTPD_URL/http_429/config-retry-after-exceeds-max-time/none/repo.git" 2>err &&

105 + duration=$(test-tool date getnanos $start) &&

106 +

107 + # Should fail quickly (no 100 second wait)

108 + duration_int=${duration%.*} &&

109 + test "$duration_int" -lt 99 &&

110 + test_grep "configured http.retryAfter.*exceeds.*http.maxRetryTime" err

111 +'

112 +

113 +test_expect_success 'HTTP 429 with Retry-After HTTP-date format' '

114 + # Test HTTP-date format (RFC 2822) in Retry-After header

115 + raw=$(test-tool date timestamp now) &&

116 + now="${raw#* -> }" &&

117 + future_time=$((now + 2)) &&

118 + raw=$(test-tool date show:rfc2822 $future_time) &&

119 + future_date="${raw#* -> }" &&

120 + future_date_encoded=$(echo "$future_date" | sed "s/ /%20/g") &&

121 +

122 + # Enable retries

123 + test_config http.maxRetries 3 &&

124 +

125 + # Git should parse the HTTP-date and retry after the delay

126 + start=$(test-tool date getnanos) &&

127 + git ls-remote "$HTTPD_URL/http_429/http-date-format/$future_date_encoded/repo.git" >output 2>err &&

128 + duration=$(test-tool date getnanos $start) &&

129 +

130 + # Should take at least 1 second (allowing tolerance for processing time)

131 + duration_int=${duration%.*} &&

132 + test "$duration_int" -ge 1 &&

133 + test_grep "refs/heads/" output

134 +'

135 +

136 +test_expect_success 'HTTP 429 with HTTP-date exceeding maxRetryTime fails immediately' '

137 + raw=$(test-tool date timestamp now) &&

138 + now="${raw#* -> }" &&

139 + future_time=$((now + 200)) &&

140 + raw=$(test-tool date show:rfc2822 $future_time) &&

141 + future_date="${raw#* -> }" &&

142 + future_date_encoded=$(echo "$future_date" | sed "s/ /%20/g") &&

143 +

144 + # Configure max retry time much less than the 200 second delay

145 + test_config http.maxRetries 3 &&

146 + test_config http.maxRetryTime 10 &&

147 +

148 + # Should fail immediately without waiting 200 seconds

149 + start=$(test-tool date getnanos) &&

150 + test_must_fail git ls-remote "$HTTPD_URL/http_429/http-date-exceeds-max-time/$future_date_encoded/repo.git" 2>err &&

151 + duration=$(test-tool date getnanos $start) &&

152 +

153 + # Should fail quickly (not wait 200 seconds)

154 + duration_int=${duration%.*} &&

155 + test "$duration_int" -lt 199 &&

156 + test_grep "http.maxRetryTime" err

157 +'

158 +

159 +test_expect_success 'HTTP 429 with past HTTP-date should not wait' '

160 + raw=$(test-tool date timestamp now) &&

161 + now="${raw#* -> }" &&

162 + past_time=$((now - 10)) &&

163 + raw=$(test-tool date show:rfc2822 $past_time) &&

164 + past_date="${raw#* -> }" &&

165 + past_date_encoded=$(echo "$past_date" | sed "s/ /%20/g") &&

166 +

167 + # Enable retries

168 + test_config http.maxRetries 3 &&

169 +

170 + # Git should retry immediately without waiting

171 + start=$(test-tool date getnanos) &&

172 + git ls-remote "$HTTPD_URL/http_429/past-http-date/$past_date_encoded/repo.git" >output 2>err &&

173 + duration=$(test-tool date getnanos $start) &&

174 +

175 + # Should complete quickly (no wait for a past-date Retry-After)

176 + duration_int=${duration%.*} &&

177 + test "$duration_int" -lt 5 &&

178 + test_grep "refs/heads/" output

179 +'

180 +

181 +test_expect_success 'HTTP 429 with invalid Retry-After format uses configured default' '

182 + # Configure default retry-after

183 + test_config http.maxRetries 3 &&

184 + test_config http.retryAfter 1 &&

185 +

186 + # Should use configured default (1 second) since header is invalid

187 + start=$(test-tool date getnanos) &&

188 + git ls-remote "$HTTPD_URL/http_429/invalid-retry-after-format/invalid/repo.git" >output 2>err &&

189 + duration=$(test-tool date getnanos $start) &&

190 +

191 + # Should take at least 1 second (the configured default)

192 + duration_int=${duration%.*} &&

193 + test "$duration_int" -ge 1 &&

194 + test_grep "refs/heads/" output &&

195 + test_grep "waiting.*retry" err

196 +'

197 +

198 +test_expect_success 'HTTP 429 will not be retried without config' '

199 + # Default config means http.maxRetries=0 (retries disabled)

200 + # When 429 is received, it should fail immediately without retry

201 + # Do NOT configure anything - use defaults (http.maxRetries defaults to 0)

202 +

203 + # Should fail immediately without retry

204 + test_must_fail git ls-remote "$HTTPD_URL/http_429/no-retry-without-config/1/repo.git" 2>err &&

205 +

206 + # Verify no retry happened (no "waiting" message)

207 + test_grep ! -i "waiting.*retry" err &&

208 +

209 + # Should get 429 error

210 + test_grep "429" err

211 +'

212 +

213 +test_expect_success 'GIT_HTTP_RETRY_AFTER overrides http.retryAfter config' '

214 + # Configure retryAfter to 10 seconds

215 + test_config http.maxRetries 3 &&

216 + test_config http.retryAfter 10 &&

217 +

218 + # Override with environment variable to 1 second

219 + start=$(test-tool date getnanos) &&

220 + GIT_HTTP_RETRY_AFTER=1 git ls-remote "$HTTPD_URL/http_429/env-retry-after-override/none/repo.git" >output 2>err &&

221 + duration=$(test-tool date getnanos $start) &&

222 +

223 + # Should use env var (1 second), not config (10 seconds)

224 + duration_int=${duration%.*} &&

225 + test "$duration_int" -ge 1 &&

226 + test "$duration_int" -lt 5 &&

227 + test_grep "refs/heads/" output &&

228 + test_grep "waiting.*retry" err

229 +'

230 +

231 +test_expect_success 'GIT_HTTP_MAX_RETRIES overrides http.maxRetries config' '

232 + # Configure maxRetries to 0 (disabled)

233 + test_config http.maxRetries 0 &&

234 + test_config http.retryAfter 1 &&

235 +

236 + # Override with environment variable to enable retries

237 + GIT_HTTP_MAX_RETRIES=3 git ls-remote "$HTTPD_URL/http_429/env-max-retries-override/1/repo.git" >output 2>err &&

238 +

239 + # Should retry (env var enables it despite config saying disabled)

240 + test_grep "refs/heads/" output &&

241 + test_grep "waiting.*retry" err

242 +'

243 +

244 +test_expect_success 'GIT_HTTP_MAX_RETRY_TIME overrides http.maxRetryTime config' '

245 + # Configure maxRetryTime to 100 seconds (would accept 50 second delay)

246 + test_config http.maxRetries 3 &&

247 + test_config http.maxRetryTime 100 &&

248 +

249 + # Override with environment variable to 10 seconds (should reject 50 second delay)

250 + start=$(test-tool date getnanos) &&

251 + test_must_fail env GIT_HTTP_MAX_RETRY_TIME=10 \

252 + git ls-remote "$HTTPD_URL/http_429/env-max-retry-time-override/50/repo.git" 2>err &&

253 + duration=$(test-tool date getnanos $start) &&

254 +

255 + # Should fail quickly (not wait 50 seconds) because env var limits to 10

256 + duration_int=${duration%.*} &&

257 + test "$duration_int" -lt 49 &&

258 + test_grep "greater than http.maxRetryTime" err

259 +'

260 +

261 +test_expect_success 'verify normal repository access still works' '

262 + git ls-remote "$HTTPD_URL/smart/repo.git" >output &&

263 + test_grep "refs/heads/" output

264 +'

265 +

266 +test_done