promisor-remote: lazy-fetch objects in subprocess
Teach Git to lazy-fetch missing objects in a subprocess instead of doing it in-process. This allows any fatal errors that occur during the fetch to be isolated and converted into an error return value, instead of causing the current command being run to terminate. Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jonathan Tan committed
Aug 17, 2020 at 21:01 UTC
7ca3c0ac37bfb803f11db14b7223f6d8f0cd142e
6 files changed
+31
-42
Documentation/technical/partial-clone.txt
+3
-10
@@ -171,20 +171,13 @@ additional flag.
171
Fetching Missing Objects
172
------------------------
173
174
-- Fetching of objects is done using the existing transport mechanism using
175
- transport_fetch_refs(), setting a new transport option
176
- TRANS_OPT_NO_DEPENDENTS to indicate that only the objects themselves are
177
- desired, not any object that they refer to.
178
-+
179
-Because some transports invoke fetch_pack() in the same process, fetch_pack()
180
-has been updated to not use any object flags when the corresponding argument
181
-(no_dependents) is set.
174
+- Fetching of objects is done by invoking a "git fetch" subprocess.
175
176
- The local repository sends a request with the hashes of all requested
184
- objects as "want" lines, and does not perform any packfile negotiation.
177
+ objects, and does not perform any packfile negotiation.
178
It then receives a packfile.
179
187
-- Because we are reusing the existing fetch-pack mechanism, fetching
180
+- Because we are reusing the existing fetch mechanism, fetching
181
currently fetches all objects referred to by the requested objects, even
182
though they are not necessary.
183
promisor-remote.c
+21
-25
@@ -3,6 +3,7 @@
3
#include "promisor-remote.h"
4
#include "config.h"
5
#include "transport.h"
6
+#include "strvec.h"
7
8
static char *repository_format_partial_clone;
9
static const char *core_partial_clone_filter_default;
@@ -12,39 +13,34 @@ void set_repository_format_partial_clone(char *partial_clone)
13
repository_format_partial_clone = xstrdup_or_null(partial_clone);
14
}
15
15
-static int fetch_refs(const char *remote_name, struct ref *ref)
16
-{
17
- struct remote *remote;
18
- struct transport *transport;
19
- int res;
20
-
21
- remote = remote_get(remote_name);
22
- if (!remote->url[0])
23
- die(_("Remote with no URL"));
24
- transport = transport_get(remote, remote->url[0]);
25
-
26
- transport_set_option(transport, TRANS_OPT_FROM_PROMISOR, "1");
27
- transport_set_option(transport, TRANS_OPT_NO_DEPENDENTS, "1");
28
- res = transport_fetch_refs(transport, ref);
29
-
30
- return res;
31
-}
32
-
16
static int fetch_objects(const char *remote_name,
17
const struct object_id *oids,
18
int oid_nr)
19
{
37
- struct ref *ref = NULL;
20
+ struct child_process child = CHILD_PROCESS_INIT;
21
int i;
22
+ FILE *child_in;
23
+
24
+ child.git_cmd = 1;
25
+ child.in = -1;
26
+ strvec_pushl(&child.args, "-c", "fetch.negotiationAlgorithm=noop",
27
+ "fetch", remote_name, "--no-tags",
28
+ "--no-write-fetch-head", "--recurse-submodules=no",
29
+ "--filter=blob:none", "--stdin", NULL);
30
+ if (start_command(&child))
31
+ die(_("promisor-remote: unable to fork off fetch subprocess"));
32
+ child_in = xfdopen(child.in, "w");
33
34
for (i = 0; i < oid_nr; i++) {
41
- struct ref *new_ref = alloc_ref(oid_to_hex(&oids[i]));
42
- oidcpy(&new_ref->old_oid, &oids[i]);
43
- new_ref->exact_oid = 1;
44
- new_ref->next = ref;
45
- ref = new_ref;
35
+ if (fputs(oid_to_hex(&oids[i]), child_in) < 0)
36
+ die_errno(_("promisor-remote: could not write to fetch subprocess"));
37
+ if (fputc('\n', child_in) < 0)
38
+ die_errno(_("promisor-remote: could not write to fetch subprocess"));
39
}
47
- return fetch_refs(remote_name, ref);
40
+
41
+ if (fclose(child_in) < 0)
42
+ die_errno(_("promisor-remote: could not close stdin to fetch subprocess"));
43
+ return finish_command(&child) ? -1 : 0;
44
}
45
46
static struct promisor_remote *promisors;
t/t0410-partial-clone.sh
+1
-1
@@ -214,7 +214,7 @@ test_expect_success 'fetching of missing objects works with ref-in-want enabled'
214
rm -rf repo/.git/objects/* &&
215
rm -f trace &&
216
GIT_TRACE_PACKET="$(pwd)/trace" git -C repo cat-file -p "$HASH" &&
217
- grep "git< fetch=.*ref-in-want" trace
217
+ grep "fetch< fetch=.*ref-in-want" trace
218
'
219
220
test_expect_success 'fetching of missing objects from another promisor remote' '
t/t4067-diff-partial-clone.sh
+4
-4
@@ -20,7 +20,7 @@ test_expect_success 'git show batches blobs' '
20
# Ensure that there is exactly 1 negotiation by checking that there is
21
# only 1 "done" line sent. ("done" marks the end of negotiation.)
22
GIT_TRACE_PACKET="$(pwd)/trace" git -C client show HEAD &&
23
- grep "git> done" trace >done_lines &&
23
+ grep "fetch> done" trace >done_lines &&
24
test_line_count = 1 done_lines
25
'
26
@@ -44,7 +44,7 @@ test_expect_success 'diff batches blobs' '
44
# Ensure that there is exactly 1 negotiation by checking that there is
45
# only 1 "done" line sent. ("done" marks the end of negotiation.)
46
GIT_TRACE_PACKET="$(pwd)/trace" git -C client diff HEAD^ HEAD &&
47
- grep "git> done" trace >done_lines &&
47
+ grep "fetch> done" trace >done_lines &&
48
test_line_count = 1 done_lines
49
'
50
@@ -127,7 +127,7 @@ test_expect_success 'diff with rename detection batches blobs' '
127
# only 1 "done" line sent. ("done" marks the end of negotiation.)
128
GIT_TRACE_PACKET="$(pwd)/trace" git -C client diff --raw -M HEAD^ HEAD >out &&
129
grep ":100644 100644.*R[0-9][0-9][0-9].*b.*c" out &&
130
- grep "git> done" trace >done_lines &&
130
+ grep "fetch> done" trace >done_lines &&
131
test_line_count = 1 done_lines
132
'
133
@@ -175,7 +175,7 @@ test_expect_success 'diff --break-rewrites fetches only if necessary, and batche
175
# by checking that there is only 1 "done" line sent. ("done" marks the
176
# end of negotiation.)
177
GIT_TRACE_PACKET="$(pwd)/trace" git -C client diff --break-rewrites --raw -M HEAD^ HEAD &&
178
- grep "git> done" trace >done_lines &&
178
+ grep "fetch> done" trace >done_lines &&
179
test_line_count = 1 done_lines
180
'
181
t/t5300-pack-object.sh
+1
-1
@@ -528,7 +528,7 @@ test_expect_success 'prefetch objects' '
528
TWO=$(git -C server rev-parse three_branch^) &&
529
git -C client fetch --filter=blob:none origin "$TWO" &&
530
GIT_TRACE_PACKET=$(pwd)/trace git -C client push origin "$TWO":refs/heads/two_branch &&
531
- grep "git> done" trace >donelines &&
531
+ grep "fetch> done" trace >donelines &&
532
test_line_count = 1 donelines
533
'
534
t/t5601-clone.sh
+1
-1
@@ -704,7 +704,7 @@ test_expect_success 'batch missing blob request during checkout' '
704
# Ensure that there is only one negotiation by checking that there is
705
# only "done" line sent. ("done" marks the end of negotiation.)
706
GIT_TRACE_PACKET="$(pwd)/trace" git -C client checkout HEAD^ &&
707
- grep "git> done" trace >done_lines &&
707
+ grep "fetch> done" trace >done_lines &&
708
test_line_count = 1 done_lines
709
'
710