fast-import: implement unpack limit
With many incremental imports, small packs become highly inefficient due to the need to readdir scan and load many indices to locate even a single object. Frequent repacking and consolidation may be prohibitively expensive in terms of disk I/O, especially in large repositories where the initial packs were aggressively optimized and marked with .keep files. In those cases, users may be better served with loose objects and relying on "git gc --auto". This changes the default behavior of fast-import for small imports found in test cases, so adjustments to t9300 were necessary. Signed-off-by: Eric Wong <normalperson@yhbt.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Eric Wong committed
Apr 25, 2016 at 21:17 UTC
d9545c7f465ed103df44cd93caddfdd265757779
5 files changed
+93
Documentation/config.txt
+9
@@ -1153,6 +1153,15 @@ difftool.<tool>.cmd::
1153
difftool.prompt::
1154
Prompt before each invocation of the diff tool.
1155
1156
+fastimport.unpackLimit::
1157
+ If the number of objects imported by linkgit:git-fast-import[1]
1158
+ is below this limit, then the objects will be unpacked into
1159
+ loose object files. However if the number of imported objects
1160
+ equals or exceeds this limit then the pack will be stored as a
1161
+ pack. Storing the pack from a fast-import can make the import
1162
+ operation complete faster, especially on slow filesystems. If
1163
+ not set, the value of `transfer.unpackLimit` is used instead.
1164
+
1165
fetch.recurseSubmodules::
1166
This option can be either set to a boolean value or to 'on-demand'.
1167
Setting it to a boolean changes the behavior of fetch and pull to
Documentation/git-fast-import.txt
+2
@@ -136,6 +136,8 @@ Performance and Compression Tuning
136
Maximum size of each output packfile.
137
The default is unlimited.
138
139
+fastimport.unpackLimit::
140
+ See linkgit:git-config[1]
141
142
Performance
143
-----------
fast-import.c
+32
@@ -166,6 +166,7 @@ Format of STDIN stream:
166
#include "quote.h"
167
#include "exec_cmd.h"
168
#include "dir.h"
169
+#include "run-command.h"
170
171
#define PACK_ID_BITS 16
172
#define MAX_PACK_ID ((1<<PACK_ID_BITS)-1)
@@ -282,6 +283,7 @@ struct recent_command {
283
/* Configured limits on output */
284
static unsigned long max_depth = 10;
285
static off_t max_packsize;
286
+static int unpack_limit = 100;
287
static int force_update;
288
static int pack_compression_level = Z_DEFAULT_COMPRESSION;
289
static int pack_compression_seen;
@@ -950,6 +952,23 @@ static void unkeep_all_packs(void)
952
}
953
}
954
955
+static int loosen_small_pack(const struct packed_git *p)
956
+{
957
+ struct child_process unpack = CHILD_PROCESS_INIT;
958
+
959
+ if (lseek(p->pack_fd, 0, SEEK_SET) < 0)
960
+ die_errno("Failed seeking to start of '%s'", p->pack_name);
961
+
962
+ unpack.in = p->pack_fd;
963
+ unpack.git_cmd = 1;
964
+ unpack.stdout_to_stderr = 1;
965
+ argv_array_push(&unpack.args, "unpack-objects");
966
+ if (!show_stats)
967
+ argv_array_push(&unpack.args, "-q");
968
+
969
+ return run_command(&unpack);
970
+}
971
+
972
static void end_packfile(void)
973
{
974
static int running;
@@ -972,6 +991,12 @@ static void end_packfile(void)
991
fixup_pack_header_footer(pack_data->pack_fd, pack_data->sha1,
992
pack_data->pack_name, object_count,
993
cur_pack_sha1, pack_size);
994
+
995
+ if (object_count <= unpack_limit) {
996
+ if (!loosen_small_pack(pack_data))
997
+ goto discard_pack;
998
+ }
999
+
1000
close(pack_data->pack_fd);
1001
idx_name = keep_pack(create_index());
1002
@@ -1002,6 +1027,7 @@ static void end_packfile(void)
1027
pack_id++;
1028
}
1029
else {
1030
+discard_pack:
1031
close(pack_data->pack_fd);
1032
unlink_or_warn(pack_data->pack_name);
1033
}
@@ -3317,6 +3343,7 @@ static void parse_option(const char *option)
3343
static void git_pack_config(void)
3344
{
3345
int indexversion_value;
3346
+ int limit;
3347
unsigned long packsizelimit_value;
3348
3349
if (!git_config_get_ulong("pack.depth", &max_depth)) {
@@ -3341,6 +3368,11 @@ static void git_pack_config(void)
3368
if (!git_config_get_ulong("pack.packsizelimit", &packsizelimit_value))
3369
max_packsize = packsizelimit_value;
3370
3371
+ if (!git_config_get_int("fastimport.unpacklimit", &limit))
3372
+ unpack_limit = limit;
3373
+ else if (!git_config_get_int("transfer.unpacklimit", &limit))
3374
+ unpack_limit = limit;
3375
+
3376
git_config(git_default_config, NULL);
3377
}
3378
t/t9300-fast-import.sh
+2
@@ -52,6 +52,7 @@ echo "$@"'
52
###
53
54
test_expect_success 'empty stream succeeds' '
55
+ git config fastimport.unpackLimit 0 &&
56
git fast-import </dev/null
57
'
58
@@ -2675,6 +2676,7 @@ test_expect_success 'R: blob bigger than threshold' '
2676
echo >>input &&
2677
2678
test_create_repo R &&
2679
+ git --git-dir=R/.git config fastimport.unpackLimit 0 &&
2680
git --git-dir=R/.git fast-import --big-file-threshold=1 <input
2681
'
2682
t/t9302-fast-import-unpack-limit.sh
new
+48
@@ -0,0 +1,48 @@
1
+#!/bin/sh
2
+test_description='test git fast-import unpack limit'
3
+. ./test-lib.sh
4
+
5
+test_expect_success 'create loose objects on import' '
6
+ test_tick &&
7
+ cat >input <<-INPUT_END &&
8
+ commit refs/heads/master
9
+ committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
10
+ data <<COMMIT
11
+ initial
12
+ COMMIT
13
+
14
+ done
15
+ INPUT_END
16
+
17
+ git -c fastimport.unpackLimit=2 fast-import --done <input &&
18
+ git fsck --no-progress &&
19
+ test $(find .git/objects/?? -type f | wc -l) -eq 2 &&
20
+ test $(find .git/objects/pack -type f | wc -l) -eq 0
21
+'
22
+
23
+test_expect_success 'bigger packs are preserved' '
24
+ test_tick &&
25
+ cat >input <<-INPUT_END &&
26
+ commit refs/heads/master
27
+ committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
28
+ data <<COMMIT
29
+ incremental should create a pack
30
+ COMMIT
31
+ from refs/heads/master^0
32
+
33
+ commit refs/heads/branch
34
+ committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
35
+ data <<COMMIT
36
+ branch
37
+ COMMIT
38
+
39
+ done
40
+ INPUT_END
41
+
42
+ git -c fastimport.unpackLimit=2 fast-import --done <input &&
43
+ git fsck --no-progress &&
44
+ test $(find .git/objects/?? -type f | wc -l) -eq 2 &&
45
+ test $(find .git/objects/pack -type f | wc -l) -eq 2
46
+'
47
+
48
+test_done