gc --auto: exclude base pack if not enough mem to "repack -ad"
pack-objects could be a big memory hog especially on large repos, everybody knows that. The suggestion to stick a .keep file on the giant base pack to avoid this problem is also known for a long time. Recent patches add an option to do just this, but it has to be either configured or activated manually. This patch lets `git gc --auto` activate this mode automatically when it thinks `repack -ad` will use a lot of memory and start affecting the system due to swapping or flushing OS cache. gc --auto decides to do this based on an estimation of pack-objects memory usage, which is quite accurate at least for the heap part, and whether that fits in half of system memory (the assumption here is for desktop environment where there are many other applications running). This mechanism only kicks in if gc.bigBasePackThreshold is not configured. If it is, it is assumed that the user already knows what they want. Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Nguyễn Thái Ngọc Duy committed
Apr 15, 2018 at 17:36 UTC
9806f5a7bf3d02247c2c500ef74f56213cd7b07a
7 files changed
+119
-4
Documentation/git-gc.txt
+7
-2
@@ -59,8 +59,13 @@ If the number of packs exceeds the value of `gc.autoPackLimit`,
59
then existing packs (except those marked with a `.keep` file
60
or over `gc.bigPackThreshold` limit)
61
are consolidated into a single pack by using the `-A` option of
62
-'git repack'. Setting `gc.autoPackLimit` to 0 disables
63
-automatic consolidation of packs.
62
+'git repack'.
63
+If the amount of memory is estimated not enough for `git repack` to
64
+run smoothly and `gc.bigPackThreshold` is not set, the largest
65
+pack will also be excluded (this is the equivalent of running `git gc`
66
+with `--keep-base-pack`).
67
+Setting `gc.autoPackLimit` to 0 disables automatic consolidation of
68
+packs.
69
+
70
If houskeeping is required due to many loose objects or packs, all
71
other housekeeping tasks (e.g. rerere, working trees, reflog...) will
builtin/gc.c
+97
-1
@@ -22,6 +22,10 @@
22
#include "commit.h"
23
#include "packfile.h"
24
#include "object-store.h"
25
+#include "pack.h"
26
+#include "pack-objects.h"
27
+#include "blob.h"
28
+#include "tree.h"
29
30
#define FAILED_RUN "failed to run %s"
31
@@ -42,6 +46,7 @@ static const char *gc_log_expire = "1.day.ago";
46
static const char *prune_expire = "2.weeks.ago";
47
static const char *prune_worktrees_expire = "3.months.ago";
48
static unsigned long big_pack_threshold;
49
+static unsigned long max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE;
50
51
static struct argv_array pack_refs_cmd = ARGV_ARRAY_INIT;
52
static struct argv_array reflog = ARGV_ARRAY_INIT;
@@ -130,6 +135,7 @@ static void gc_config(void)
135
git_config_get_expiry("gc.logexpiry", &gc_log_expire);
136
137
git_config_get_ulong("gc.bigpackthreshold", &big_pack_threshold);
138
+ git_config_get_ulong("pack.deltacachesize", &max_delta_cache_size);
139
140
git_config(git_default_config, NULL);
141
}
@@ -169,7 +175,8 @@ static int too_many_loose_objects(void)
175
return needed;
176
}
177
172
-static void find_base_packs(struct string_list *packs, unsigned long limit)
178
+static struct packed_git *find_base_packs(struct string_list *packs,
179
+ unsigned long limit)
180
{
181
struct packed_git *p, *base = NULL;
182
@@ -186,6 +193,8 @@ static void find_base_packs(struct string_list *packs, unsigned long limit)
193
194
if (base)
195
string_list_append(packs, base->pack_name);
196
+
197
+ return base;
198
}
199
200
static int too_many_packs(void)
@@ -210,6 +219,79 @@ static int too_many_packs(void)
219
return gc_auto_pack_limit < cnt;
220
}
221
222
+static uint64_t total_ram(void)
223
+{
224
+#if defined(HAVE_SYSINFO)
225
+ struct sysinfo si;
226
+
227
+ if (!sysinfo(&si))
228
+ return si.totalram;
229
+#elif defined(HAVE_BSD_SYSCTL) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM))
230
+ int64_t physical_memory;
231
+ int mib[2];
232
+ size_t length;
233
+
234
+ mib[0] = CTL_HW;
235
+# if defined(HW_MEMSIZE)
236
+ mib[1] = HW_MEMSIZE;
237
+# else
238
+ mib[1] = HW_PHYSMEM;
239
+# endif
240
+ length = sizeof(int64_t);
241
+ if (!sysctl(mib, 2, &physical_memory, &length, NULL, 0))
242
+ return physical_memory;
243
+#elif defined(GIT_WINDOWS_NATIVE)
244
+ MEMORYSTATUSEX memInfo;
245
+
246
+ memInfo.dwLength = sizeof(MEMORYSTATUSEX);
247
+ if (GlobalMemoryStatusEx(&memInfo))
248
+ return memInfo.ullTotalPhys;
249
+#endif
250
+ return 0;
251
+}
252
+
253
+static uint64_t estimate_repack_memory(struct packed_git *pack)
254
+{
255
+ unsigned long nr_objects = approximate_object_count();
256
+ size_t os_cache, heap;
257
+
258
+ if (!pack || !nr_objects)
259
+ return 0;
260
+
261
+ /*
262
+ * First we have to scan through at least one pack.
263
+ * Assume enough room in OS file cache to keep the entire pack
264
+ * or we may accidentally evict data of other processes from
265
+ * the cache.
266
+ */
267
+ os_cache = pack->pack_size + pack->index_size;
268
+ /* then pack-objects needs lots more for book keeping */
269
+ heap = sizeof(struct object_entry) * nr_objects;
270
+ /*
271
+ * internal rev-list --all --objects takes up some memory too,
272
+ * let's say half of it is for blobs
273
+ */
274
+ heap += sizeof(struct blob) * nr_objects / 2;
275
+ /*
276
+ * and the other half is for trees (commits and tags are
277
+ * usually insignificant)
278
+ */
279
+ heap += sizeof(struct tree) * nr_objects / 2;
280
+ /* and then obj_hash[], underestimated in fact */
281
+ heap += sizeof(struct object *) * nr_objects;
282
+ /* revindex is used also */
283
+ heap += sizeof(struct revindex_entry) * nr_objects;
284
+ /*
285
+ * read_sha1_file() (either at delta calculation phase, or
286
+ * writing phase) also fills up the delta base cache
287
+ */
288
+ heap += delta_base_cache_limit;
289
+ /* and of course pack-objects has its own delta cache */
290
+ heap += max_delta_cache_size;
291
+
292
+ return os_cache + heap;
293
+}
294
+
295
static int keep_one_pack(struct string_list_item *item, void *data)
296
{
297
argv_array_pushf(&repack, "--keep-pack=%s", basename(item->string));
@@ -260,6 +342,20 @@ static int need_to_gc(void)
342
string_list_clear(&keep_pack, 0);
343
find_base_packs(&keep_pack, 0);
344
}
345
+ } else {
346
+ struct packed_git *p = find_base_packs(&keep_pack, 0);
347
+ uint64_t mem_have, mem_want;
348
+
349
+ mem_have = total_ram();
350
+ mem_want = estimate_repack_memory(p);
351
+
352
+ /*
353
+ * Only allow 1/2 of memory for pack-objects, leave
354
+ * the rest for the OS and other processes in the
355
+ * system.
356
+ */
357
+ if (!mem_have || mem_want < mem_have / 2)
358
+ string_list_clear(&keep_pack, 0);
359
}
360
361
add_repack_all_option(&keep_pack);
builtin/pack-objects.c
+1
-1
@@ -82,7 +82,7 @@ static uint16_t write_bitmap_options;
82
static int exclude_promisor_objects;
83
84
static unsigned long delta_cache_size = 0;
85
-static unsigned long max_delta_cache_size = 256 * 1024 * 1024;
85
+static unsigned long max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE;
86
static unsigned long cache_max_small_delta_size = 1000;
87
88
static unsigned long window_memory_limit = 0;
config.mak.uname
+1
@@ -37,6 +37,7 @@ ifeq ($(uname_S),Linux)
37
HAVE_GETDELIM = YesPlease
38
SANE_TEXT_GREP=-a
39
FREAD_READS_DIRECTORIES = UnfortunatelyYes
40
+ BASIC_CFLAGS += -DHAVE_SYSINFO
41
endif
42
ifeq ($(uname_S),GNU/kFreeBSD)
43
HAVE_ALLOCA_H = YesPlease
git-compat-util.h
+4
@@ -284,6 +284,10 @@ extern char *gitdirname(char *);
284
#include <openssl/err.h>
285
#endif
286
287
+#ifdef HAVE_SYSINFO
288
+# include <sys/sysinfo.h>
289
+#endif
290
+
291
/* On most systems <netdb.h> would have given us this, but
292
* not on some systems (e.g. z/OS).
293
*/
pack-objects.h
+2
@@ -1,6 +1,8 @@
1
#ifndef PACK_OBJECTS_H
2
#define PACK_OBJECTS_H
3
4
+#define DEFAULT_DELTA_CACHE_SIZE (256 * 1024 * 1024)
5
+
6
struct object_entry {
7
struct pack_idx_entry idx;
8
unsigned long size; /* uncompressed size */
t/t6500-gc.sh
+7
@@ -5,6 +5,13 @@ test_description='basic git gc tests
5
6
. ./test-lib.sh
7
8
+test_expect_success 'setup' '
9
+ # do not let the amount of physical memory affects gc
10
+ # behavior, make sure we always pack everything to one pack by
11
+ # default
12
+ git config gc.bigPackThreshold 2g
13
+'
14
+
15
test_expect_success 'gc empty repository' '
16
git gc
17
'