fetch: support hideRefs to speed up connectivity checks
With roughly 800 remotes all fetching into their own refs/remotes/$REMOTE/* island, the connectivity check[1] gets expensive for each fetch on systems which lack sufficient RAM to cache objects. To do a no-op fetch on one $REMOTE out of hundreds, hideRefs now allows the no-op fetch to take ~30 seconds instead of ~20 minutes on a noisy, RAM-constrained machine (localhost, so no network latency): git -c fetch.hideRefs=refs \ -c fetch.hideRefs='!refs/remotes/$REMOTE/' \ fetch $REMOTE [1] `git rev-list --objects --stdin --not --all --quiet --alternate-refs' Signed-off-by: Eric Wong <e@80x24.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Eric Wong committed
Feb 12, 2023 at 09:04 UTC
c6ce27ab08d30dd9d626d7a56cb928bc5792eb27
8 files changed
+26
-12
Documentation/git-rev-parse.txt
+5
-4
@@ -197,10 +197,11 @@ respectively, and they must begin with `refs/` when applied to `--glob`
197
or `--all`. If a trailing '/{asterisk}' is intended, it must be given
198
explicitly.
199
200
---exclude-hidden=[receive|uploadpack]::
201
- Do not include refs that would be hidden by `git-receive-pack` or
202
- `git-upload-pack` by consulting the appropriate `receive.hideRefs` or
203
- `uploadpack.hideRefs` configuration along with `transfer.hideRefs` (see
200
+--exclude-hidden=[fetch|receive|uploadpack]::
201
+ Do not include refs that would be hidden by `git-fetch`,
202
+ `git-receive-pack` or `git-upload-pack` by consulting the appropriate
203
+ `fetch.hideRefs`, `receive.hideRefs` or `uploadpack.hideRefs`
204
+ configuration along with `transfer.hideRefs` (see
205
linkgit:git-config[1]). This option affects the next pseudo-ref option
206
`--all` or `--glob` and is cleared after processing them.
207
Documentation/rev-list-options.txt
+5
-4
@@ -195,10 +195,11 @@ respectively, and they must begin with `refs/` when applied to `--glob`
195
or `--all`. If a trailing '/{asterisk}' is intended, it must be given
196
explicitly.
197
198
---exclude-hidden=[receive|uploadpack]::
199
- Do not include refs that would be hidden by `git-receive-pack` or
200
- `git-upload-pack` by consulting the appropriate `receive.hideRefs` or
201
- `uploadpack.hideRefs` configuration along with `transfer.hideRefs` (see
198
+--exclude-hidden=[fetch|receive|uploadpack]::
199
+ Do not include refs that would be hidden by `git-fetch`,
200
+ `git-receive-pack` or `git-upload-pack` by consulting the appropriate
201
+ `fetch.hideRefs`, `receive.hideRefs` or `uploadpack.hideRefs`
202
+ configuration along with `transfer.hideRefs` (see
203
linkgit:git-config[1]). This option affects the next pseudo-ref option
204
`--all` or `--glob` and is cleared after processing them.
205
builtin/fetch.c
+2
@@ -1132,6 +1132,7 @@ static int store_updated_refs(const char *raw_url, const char *remote_name,
1132
if (!connectivity_checked) {
1133
struct check_connected_options opt = CHECK_CONNECTED_INIT;
1134
1135
+ opt.exclude_hidden_refs_section = "fetch";
1136
rm = ref_map;
1137
if (check_connected(iterate_ref_map, &rm, &opt)) {
1138
rc = error(_("%s did not send all necessary objects\n"), url);
@@ -1325,6 +1326,7 @@ static int check_exist_and_connected(struct ref *ref_map)
1326
}
1327
1328
opt.quiet = 1;
1329
+ opt.exclude_hidden_refs_section = "fetch";
1330
return check_connected(iterate_ref_map, &rm, &opt);
1331
}
1332
builtin/rev-list.c
+1
-1
@@ -38,7 +38,7 @@ static const char rev_list_usage[] =
38
" --tags\n"
39
" --remotes\n"
40
" --stdin\n"
41
-" --exclude-hidden=[receive|uploadpack]\n"
41
+" --exclude-hidden=[fetch|receive|uploadpack]\n"
42
" --quiet\n"
43
" ordering output:\n"
44
" --topo-order\n"
revision.c
+2
-1
@@ -1574,7 +1574,8 @@ void exclude_hidden_refs(struct ref_exclusions *exclusions, const char *section)
1574
{
1575
struct exclude_hidden_refs_cb cb;
1576
1577
- if (strcmp(section, "receive") && strcmp(section, "uploadpack"))
1577
+ if (strcmp(section, "fetch") && strcmp(section, "receive") &&
1578
+ strcmp(section, "uploadpack"))
1579
die(_("unsupported section for hidden refs: %s"), section);
1580
1581
if (exclusions->hidden_refs_configured)
t/t5510-fetch.sh
+9
@@ -1171,6 +1171,15 @@ test_expect_success '--no-show-forced-updates' '
1171
)
1172
'
1173
1174
+for section in fetch transfer
1175
+do
1176
+ test_expect_success "$section.hideRefs affects connectivity check" '
1177
+ GIT_TRACE="$PWD"/trace git -c $section.hideRefs=refs -c \
1178
+ $section.hideRefs="!refs/tags/" fetch &&
1179
+ grep "git rev-list .*--exclude-hidden=fetch" trace
1180
+ '
1181
+done
1182
+
1183
setup_negotiation_tip () {
1184
SERVER="$1"
1185
URL="$2"
t/t6018-rev-list-glob.sh
+1
-1
@@ -187,7 +187,7 @@ test_expect_success 'rev-parse --exclude=ref with --remotes=glob' '
187
compare rev-parse "--exclude=upstream/x --remotes=upstream/*" "upstream/one upstream/two"
188
'
189
190
-for section in receive uploadpack
190
+for section in fetch receive uploadpack
191
do
192
test_expect_success "rev-parse --exclude-hidden=$section with --all" '
193
compare "-c transfer.hideRefs=refs/remotes/ rev-parse" "--branches --tags" "--exclude-hidden=$section --all"
t/t6021-rev-list-exclude-hidden.sh
+1
-1
@@ -22,7 +22,7 @@ test_expect_success 'invalid section' '
22
test_cmp expected err
23
'
24
25
-for section in receive uploadpack
25
+for section in fetch receive uploadpack
26
do
27
test_expect_success "$section: passed multiple times" '
28
echo "fatal: --exclude-hidden= passed more than once" >expected &&