gc: add --keep-largest-pack option
This adds a new repack mode that combines everything into a secondary pack, leaving the largest pack alone. This could help reduce memory pressure. On linux-2.6.git, valgrind massif reports 1.6GB heap in "pack all" case, and 535MB in "pack all except the base pack" case. We save roughly 1GB memory by excluding the base pack. This should also lower I/O because we don't have to rewrite a giant pack every time (e.g. for linux-2.6.git that's a 1.4GB pack file).. PS. The use of string_list here seems overkill, but we'll need it in the next patch... Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Nguyễn Thái Ngọc Duy committed
Apr 15, 2018 at 17:36 UTC
ae4e89e549b76b561a1c384dd7314c9b671c22bc
3 files changed
+71
-5
Documentation/git-gc.txt
+5
-1
@@ -9,7 +9,7 @@ git-gc - Cleanup unnecessary files and optimize the local repository
9
SYNOPSIS
10
--------
11
[verse]
12
-'git gc' [--aggressive] [--auto] [--quiet] [--prune=<date> | --no-prune] [--force]
12
+'git gc' [--aggressive] [--auto] [--quiet] [--prune=<date> | --no-prune] [--force] [--keep-largest-pack]
13
14
DESCRIPTION
15
-----------
@@ -84,6 +84,10 @@ be performed as well.
84
Force `git gc` to run even if there may be another `git gc`
85
instance running on this repository.
86
87
+--keep-largest-pack::
88
+ All packs except the largest pack and those marked with a
89
+ `.keep` files are consolidated into a single pack.
90
+
91
Configuration
92
-------------
93
builtin/gc.c
+41
-4
@@ -166,6 +166,22 @@ static int too_many_loose_objects(void)
166
return needed;
167
}
168
169
+static void find_base_packs(struct string_list *packs)
170
+{
171
+ struct packed_git *p, *base = NULL;
172
+
173
+ for (p = get_packed_git(the_repository); p; p = p->next) {
174
+ if (!p->pack_local)
175
+ continue;
176
+ if (!base || base->pack_size < p->pack_size) {
177
+ base = p;
178
+ }
179
+ }
180
+
181
+ if (base)
182
+ string_list_append(packs, base->pack_name);
183
+}
184
+
185
static int too_many_packs(void)
186
{
187
struct packed_git *p;
@@ -188,7 +204,13 @@ static int too_many_packs(void)
204
return gc_auto_pack_limit < cnt;
205
}
206
191
-static void add_repack_all_option(void)
207
+static int keep_one_pack(struct string_list_item *item, void *data)
208
+{
209
+ argv_array_pushf(&repack, "--keep-pack=%s", basename(item->string));
210
+ return 0;
211
+}
212
+
213
+static void add_repack_all_option(struct string_list *keep_pack)
214
{
215
if (prune_expire && !strcmp(prune_expire, "now"))
216
argv_array_push(&repack, "-a");
@@ -197,6 +219,9 @@ static void add_repack_all_option(void)
219
if (prune_expire)
220
argv_array_pushf(&repack, "--unpack-unreachable=%s", prune_expire);
221
}
222
+
223
+ if (keep_pack)
224
+ for_each_string_list(keep_pack, keep_one_pack, NULL);
225
}
226
227
static void add_repack_incremental_option(void)
@@ -220,7 +245,7 @@ static int need_to_gc(void)
245
* there is no need.
246
*/
247
if (too_many_packs())
223
- add_repack_all_option();
248
+ add_repack_all_option(NULL);
249
else if (too_many_loose_objects())
250
add_repack_incremental_option();
251
else
@@ -354,6 +379,7 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
379
const char *name;
380
pid_t pid;
381
int daemonized = 0;
382
+ int keep_base_pack = -1;
383
384
struct option builtin_gc_options[] = {
385
OPT__QUIET(&quiet, N_("suppress progress reporting")),
@@ -366,6 +392,8 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
392
OPT_BOOL_F(0, "force", &force,
393
N_("force running gc even if there may be another gc running"),
394
PARSE_OPT_NOCOMPLETE),
395
+ OPT_BOOL(0, "keep-largest-pack", &keep_base_pack,
396
+ N_("repack all other packs except the largest pack")),
397
OPT_END()
398
};
399
@@ -431,8 +459,17 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
459
*/
460
daemonized = !daemonize();
461
}
434
- } else
435
- add_repack_all_option();
462
+ } else {
463
+ struct string_list keep_pack = STRING_LIST_INIT_NODUP;
464
+
465
+ if (keep_base_pack != -1) {
466
+ if (keep_base_pack)
467
+ find_base_packs(&keep_pack);
468
+ }
469
+
470
+ add_repack_all_option(&keep_pack);
471
+ string_list_clear(&keep_pack, 0);
472
+ }
473
474
name = lock_repo_for_gc(force, &pid);
475
if (name) {
t/t6500-gc.sh
+25
@@ -43,6 +43,31 @@ test_expect_success 'gc is not aborted due to a stale symref' '
43
)
44
'
45
46
+test_expect_success 'gc --keep-largest-pack' '
47
+ test_create_repo keep-pack &&
48
+ (
49
+ cd keep-pack &&
50
+ test_commit one &&
51
+ test_commit two &&
52
+ test_commit three &&
53
+ git gc &&
54
+ ( cd .git/objects/pack && ls *.pack ) >pack-list &&
55
+ test_line_count = 1 pack-list &&
56
+ BASE_PACK=.git/objects/pack/pack-*.pack &&
57
+ test_commit four &&
58
+ git repack -d &&
59
+ test_commit five &&
60
+ git repack -d &&
61
+ ( cd .git/objects/pack && ls *.pack ) >pack-list &&
62
+ test_line_count = 3 pack-list &&
63
+ git gc --keep-largest-pack &&
64
+ ( cd .git/objects/pack && ls *.pack ) >pack-list &&
65
+ test_line_count = 2 pack-list &&
66
+ test_path_is_file $BASE_PACK &&
67
+ git fsck
68
+ )
69
+'
70
+
71
test_expect_success 'auto gc with too many loose objects does not attempt to create bitmaps' '
72
test_config gc.auto 3 &&
73
test_config gc.autodetach false &&