diff: batch fetching of missing blobs
When running a command like "git show" or "git diff" in a partial clone, batch all missing blobs to be fetched as one request. This is similar to c0c578b33c ("unpack-trees: batch fetching of missing blobs", 2017-12-08), but for another command. Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jonathan Tan committed
Apr 5, 2019 at 10:09 UTC
7fbbcb21b162883615d5542e06fb2eb8685fd496
2 files changed
+137
diff.c
+34
@@ -25,6 +25,7 @@
25
#include "packfile.h"
26
#include "parse-options.h"
27
#include "help.h"
28
+#include "fetch-object.h"
29
30
#ifdef NO_FAST_WORKING_DIRECTORY
31
#define FAST_WORKING_DIRECTORY 0
@@ -6366,8 +6367,41 @@ void diffcore_fix_diff_index(void)
6367
QSORT(q->queue, q->nr, diffnamecmp);
6368
}
6369
6370
+static void add_if_missing(struct repository *r,
6371
+ struct oid_array *to_fetch,
6372
+ const struct diff_filespec *filespec)
6373
+{
6374
+ if (filespec && filespec->oid_valid &&
6375
+ oid_object_info_extended(r, &filespec->oid, NULL,
6376
+ OBJECT_INFO_FOR_PREFETCH))
6377
+ oid_array_append(to_fetch, &filespec->oid);
6378
+}
6379
+
6380
void diffcore_std(struct diff_options *options)
6381
{
6382
+ if (options->repo == the_repository &&
6383
+ repository_format_partial_clone) {
6384
+ /*
6385
+ * Prefetch the diff pairs that are about to be flushed.
6386
+ */
6387
+ int i;
6388
+ struct diff_queue_struct *q = &diff_queued_diff;
6389
+ struct oid_array to_fetch = OID_ARRAY_INIT;
6390
+
6391
+ for (i = 0; i < q->nr; i++) {
6392
+ struct diff_filepair *p = q->queue[i];
6393
+ add_if_missing(options->repo, &to_fetch, p->one);
6394
+ add_if_missing(options->repo, &to_fetch, p->two);
6395
+ }
6396
+ if (to_fetch.nr)
6397
+ /*
6398
+ * NEEDSWORK: Consider deduplicating the OIDs sent.
6399
+ */
6400
+ fetch_objects(repository_format_partial_clone,
6401
+ to_fetch.oid, to_fetch.nr);
6402
+ oid_array_clear(&to_fetch);
6403
+ }
6404
+
6405
/* NOTE please keep the following in sync with diff_tree_combined() */
6406
if (options->skip_stat_unmatch)
6407
diffcore_skip_stat_unmatch(options);
t/t4067-diff-partial-clone.sh
new
+103
@@ -0,0 +1,103 @@
1
+#!/bin/sh
2
+
3
+test_description='behavior of diff when reading objects in a partial clone'
4
+
5
+. ./test-lib.sh
6
+
7
+test_expect_success 'git show batches blobs' '
8
+ test_when_finished "rm -rf server client trace" &&
9
+
10
+ test_create_repo server &&
11
+ echo a >server/a &&
12
+ echo b >server/b &&
13
+ git -C server add a b &&
14
+ git -C server commit -m x &&
15
+
16
+ test_config -C server uploadpack.allowfilter 1 &&
17
+ test_config -C server uploadpack.allowanysha1inwant 1 &&
18
+ git clone --bare --filter=blob:limit=0 "file://$(pwd)/server" client &&
19
+
20
+ # Ensure that there is exactly 1 negotiation by checking that there is
21
+ # only 1 "done" line sent. ("done" marks the end of negotiation.)
22
+ GIT_TRACE_PACKET="$(pwd)/trace" git -C client show HEAD &&
23
+ grep "git> done" trace >done_lines &&
24
+ test_line_count = 1 done_lines
25
+'
26
+
27
+test_expect_success 'diff batches blobs' '
28
+ test_when_finished "rm -rf server client trace" &&
29
+
30
+ test_create_repo server &&
31
+ echo a >server/a &&
32
+ echo b >server/b &&
33
+ git -C server add a b &&
34
+ git -C server commit -m x &&
35
+ echo c >server/c &&
36
+ echo d >server/d &&
37
+ git -C server add c d &&
38
+ git -C server commit -m x &&
39
+
40
+ test_config -C server uploadpack.allowfilter 1 &&
41
+ test_config -C server uploadpack.allowanysha1inwant 1 &&
42
+ git clone --bare --filter=blob:limit=0 "file://$(pwd)/server" client &&
43
+
44
+ # Ensure that there is exactly 1 negotiation by checking that there is
45
+ # only 1 "done" line sent. ("done" marks the end of negotiation.)
46
+ GIT_TRACE_PACKET="$(pwd)/trace" git -C client diff HEAD^ HEAD &&
47
+ grep "git> done" trace >done_lines &&
48
+ test_line_count = 1 done_lines
49
+'
50
+
51
+test_expect_success 'diff skips same-OID blobs' '
52
+ test_when_finished "rm -rf server client trace" &&
53
+
54
+ test_create_repo server &&
55
+ echo a >server/a &&
56
+ echo b >server/b &&
57
+ git -C server add a b &&
58
+ git -C server commit -m x &&
59
+ echo another-a >server/a &&
60
+ git -C server add a &&
61
+ git -C server commit -m x &&
62
+
63
+ test_config -C server uploadpack.allowfilter 1 &&
64
+ test_config -C server uploadpack.allowanysha1inwant 1 &&
65
+ git clone --bare --filter=blob:limit=0 "file://$(pwd)/server" client &&
66
+
67
+ echo a | git hash-object --stdin >hash-old-a &&
68
+ echo another-a | git hash-object --stdin >hash-new-a &&
69
+ echo b | git hash-object --stdin >hash-b &&
70
+
71
+ # Ensure that only a and another-a are fetched.
72
+ GIT_TRACE_PACKET="$(pwd)/trace" git -C client diff HEAD^ HEAD &&
73
+ grep "want $(cat hash-old-a)" trace &&
74
+ grep "want $(cat hash-new-a)" trace &&
75
+ ! grep "want $(cat hash-b)" trace
76
+'
77
+
78
+test_expect_success 'diff with rename detection batches blobs' '
79
+ test_when_finished "rm -rf server client trace" &&
80
+
81
+ test_create_repo server &&
82
+ echo a >server/a &&
83
+ printf "b\nb\nb\nb\nb\n" >server/b &&
84
+ git -C server add a b &&
85
+ git -C server commit -m x &&
86
+ rm server/b &&
87
+ printf "b\nb\nb\nb\nbX\n" >server/c &&
88
+ git -C server add c &&
89
+ git -C server commit -a -m x &&
90
+
91
+ test_config -C server uploadpack.allowfilter 1 &&
92
+ test_config -C server uploadpack.allowanysha1inwant 1 &&
93
+ git clone --bare --filter=blob:limit=0 "file://$(pwd)/server" client &&
94
+
95
+ # Ensure that there is exactly 1 negotiation by checking that there is
96
+ # only 1 "done" line sent. ("done" marks the end of negotiation.)
97
+ GIT_TRACE_PACKET="$(pwd)/trace" git -C client diff -M HEAD^ HEAD >out &&
98
+ grep "similarity index" out &&
99
+ grep "git> done" trace >done_lines &&
100
+ test_line_count = 1 done_lines
101
+'
102
+
103
+test_done