pack-objects: prefetch objects to be packed
When an object to be packed is noticed to be missing, prefetch all to-be-packed objects in one batch. Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jonathan Tan committed
Jul 20, 2020 at 17:21 UTC
e00549aa9b19cc85c8bf3c3efd0a466aaf5e08f3
2 files changed
+68
-4
builtin/pack-objects.c
+32
-4
@@ -35,6 +35,7 @@
35
#include "midx.h"
36
#include "trace2.h"
37
#include "shallow.h"
38
+#include "promisor-remote.h"
39
40
#define IN_PACK(obj) oe_in_pack(&to_pack, obj)
41
#define SIZE(obj) oe_size(&to_pack, obj)
@@ -1704,7 +1705,26 @@ static int can_reuse_delta(const struct object_id *base_oid,
1705
return 0;
1706
}
1707
1707
-static void check_object(struct object_entry *entry)
1708
+static void prefetch_to_pack(uint32_t object_index_start) {
1709
+ struct oid_array to_fetch = OID_ARRAY_INIT;
1710
+ uint32_t i;
1711
+
1712
+ for (i = object_index_start; i < to_pack.nr_objects; i++) {
1713
+ struct object_entry *entry = to_pack.objects + i;
1714
+
1715
+ if (!oid_object_info_extended(the_repository,
1716
+ &entry->idx.oid,
1717
+ NULL,
1718
+ OBJECT_INFO_FOR_PREFETCH))
1719
+ continue;
1720
+ oid_array_append(&to_fetch, &entry->idx.oid);
1721
+ }
1722
+ promisor_remote_get_direct(the_repository,
1723
+ to_fetch.oid, to_fetch.nr);
1724
+ oid_array_clear(&to_fetch);
1725
+}
1726
+
1727
+static void check_object(struct object_entry *entry, uint32_t object_index)
1728
{
1729
unsigned long canonical_size;
1730
enum object_type type;
@@ -1843,8 +1863,16 @@ static void check_object(struct object_entry *entry)
1863
}
1864
1865
if (oid_object_info_extended(the_repository, &entry->idx.oid, &oi,
1846
- OBJECT_INFO_LOOKUP_REPLACE) < 0)
1847
- type = -1;
1866
+ OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_LOOKUP_REPLACE) < 0) {
1867
+ if (has_promisor_remote()) {
1868
+ prefetch_to_pack(object_index);
1869
+ if (oid_object_info_extended(the_repository, &entry->idx.oid, &oi,
1870
+ OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_LOOKUP_REPLACE) < 0)
1871
+ type = -1;
1872
+ } else {
1873
+ type = -1;
1874
+ }
1875
+ }
1876
oe_set_type(entry, type);
1877
if (entry->type_valid) {
1878
SET_SIZE(entry, canonical_size);
@@ -2065,7 +2093,7 @@ static void get_object_details(void)
2093
2094
for (i = 0; i < to_pack.nr_objects; i++) {
2095
struct object_entry *entry = sorted_by_offset[i];
2068
- check_object(entry);
2096
+ check_object(entry, i);
2097
if (entry->type_valid &&
2098
oe_size_greater_than(&to_pack, entry, big_file_threshold))
2099
entry->no_try_delta = 1;
t/t5300-pack-object.sh
+36
@@ -497,4 +497,40 @@ test_expect_success 'make sure index-pack detects the SHA1 collision (large blob
497
)
498
'
499
500
+test_expect_success 'prefetch objects' '
501
+ rm -rf server client &&
502
+
503
+ git init server &&
504
+ test_config -C server uploadpack.allowanysha1inwant 1 &&
505
+ test_config -C server uploadpack.allowfilter 1 &&
506
+ test_config -C server protocol.version 2 &&
507
+
508
+ echo one >server/one &&
509
+ git -C server add one &&
510
+ git -C server commit -m one &&
511
+ git -C server branch one_branch &&
512
+
513
+ echo two_a >server/two_a &&
514
+ echo two_b >server/two_b &&
515
+ git -C server add two_a two_b &&
516
+ git -C server commit -m two &&
517
+
518
+ echo three >server/three &&
519
+ git -C server add three &&
520
+ git -C server commit -m three &&
521
+ git -C server branch three_branch &&
522
+
523
+ # Clone, fetch "two" with blobs excluded, and re-push it. This requires
524
+ # the client to have the blobs of "two" - verify that these are
525
+ # prefetched in one batch.
526
+ git clone --filter=blob:none --single-branch -b one_branch \
527
+ "file://$(pwd)/server" client &&
528
+ test_config -C client protocol.version 2 &&
529
+ TWO=$(git -C server rev-parse three_branch^) &&
530
+ git -C client fetch --filter=blob:none origin "$TWO" &&
531
+ GIT_TRACE_PACKET=$(pwd)/trace git -C client push origin "$TWO":refs/heads/two_branch &&
532
+ grep "git> done" trace >donelines &&
533
+ test_line_count = 1 donelines
534
+'
535
+
536
test_done