gc: ignore old gc.log files

A server can end up in a state where there are lots of unreferenced loose objects (say, because many users are doing a bunch of rebasing and pushing their rebased branches). Running "git gc --auto" in this state would cause a gc.log file to be created, preventing future auto gcs, causing pack files to pile up. Since many git operations are O(n) in the number of pack files, this would lead to poor performance. Git should never get itself into a state where it refuses to do any maintenance, just because at some point some piece of the maintenance didn't make progress. Teach Git to ignore gc.log files which are older than (by default) one day old, which can be tweaked via the gc.logExpiry configuration variable. That way, these pack files will get cleaned up, if necessary, at least once per day. And operators who find a need for more-frequent gcs can adjust gc.logExpiry to meet their needs. There is also some cleanup: a successful manual gc, or a warning-free auto gc with an old log file, will remove any old gc.log files. It might still happen that manual intervention is required (e.g. because the repo is corrupt), but at the very least it won't be because Git is too dumb to try again. Signed-off-by: David Turner <dturner@twosigma.com> Helped-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>

David Turner committed Feb 10, 2017 at 16:28 UTC a831c06a2b639b0f862297672973bef965c7ffcd
3 files changed +71 -7
Documentation/config.txt
+6
@@ -1397,6 +1397,12 @@ gc.autoDetach::
1397 Make `git gc --auto` return immediately and run in background
1398 if the system supports it. Default is true.
1399
1400 +gc.logExpiry::
1401 + If the file gc.log exists, then `git gc --auto` won't run
1402 + unless that file is more than 'gc.logExpiry' old. Default is
1403 + "1.day". See `gc.pruneExpire` for more ways to specify its
1404 + value.
1405 +
1406 gc.packRefs::
1407 Running `git pack-refs` in a repository renders it
1408 unclonable by Git versions prior to 1.5.1.2 over dumb
builtin/gc.c
+50 -7
@@ -33,6 +33,8 @@ static int aggressive_window = 250;
33 static int gc_auto_threshold = 6700;
34 static int gc_auto_pack_limit = 50;
35 static int detach_auto = 1;
36 +static unsigned long gc_log_expire_time;
37 +static const char *gc_log_expire = "1.day.ago";
38 static const char *prune_expire = "2.weeks.ago";
39 static const char *prune_worktrees_expire = "3.months.ago";
40
@@ -76,10 +78,28 @@ static void git_config_date_string(const char *key, const char **output)
78 static void process_log_file(void)
79 {
80 struct stat st;
79 - if (!fstat(get_lock_file_fd(&log_lock), &st) && st.st_size)
81 + if (fstat(get_lock_file_fd(&log_lock), &st)) {
82 + /*
83 + * Perhaps there was an i/o error or another
84 + * unlikely situation. Try to make a note of
85 + * this in gc.log along with any existing
86 + * messages.
87 + */
88 + int saved_errno = errno;
89 + fprintf(stderr, _("Failed to fstat %s: %s"),
90 + get_tempfile_path(&log_lock.tempfile),
91 + strerror(saved_errno));
92 + fflush(stderr);
93 commit_lock_file(&log_lock);
81 - else
94 + errno = saved_errno;
95 + } else if (st.st_size) {
96 + /* There was some error recorded in the lock file */
97 + commit_lock_file(&log_lock);
98 + } else {
99 + /* No error, clean up any old gc.log */
100 + unlink(git_path("gc.log"));
101 rollback_lock_file(&log_lock);
102 + }
103 }
104
105 static void process_log_file_at_exit(void)
@@ -113,6 +133,8 @@ static void gc_config(void)
133 git_config_get_bool("gc.autodetach", &detach_auto);
134 git_config_date_string("gc.pruneexpire", &prune_expire);
135 git_config_date_string("gc.worktreepruneexpire", &prune_worktrees_expire);
136 + git_config_date_string("gc.logexpiry", &gc_log_expire);
137 +
138 git_config(git_default_config, NULL);
139 }
140
@@ -290,19 +312,34 @@ static const char *lock_repo_for_gc(int force, pid_t* ret_pid)
312 static int report_last_gc_error(void)
313 {
314 struct strbuf sb = STRBUF_INIT;
293 - int ret;
315 + int ret = 0;
316 + struct stat st;
317 + char *gc_log_path = git_pathdup("gc.log");
318
295 - ret = strbuf_read_file(&sb, git_path("gc.log"), 0);
319 + if (stat(gc_log_path, &st)) {
320 + if (errno == ENOENT)
321 + goto done;
322 +
323 + ret = error_errno(_("Can't stat %s"), gc_log_path);
324 + goto done;
325 + }
326 +
327 + if (st.st_mtime < gc_log_expire_time)
328 + goto done;
329 +
330 + ret = strbuf_read_file(&sb, gc_log_path, 0);
331 if (ret > 0)
297 - return error(_("The last gc run reported the following. "
332 + ret = error(_("The last gc run reported the following. "
333 "Please correct the root cause\n"
334 "and remove %s.\n"
335 "Automatic cleanup will not be performed "
336 "until the file is removed.\n\n"
337 "%s"),
303 - git_path("gc.log"), sb.buf);
338 + gc_log_path, sb.buf);
339 strbuf_release(&sb);
305 - return 0;
340 +done:
341 + free(gc_log_path);
342 + return ret;
343 }
344
345 static int gc_before_repack(void)
@@ -349,7 +386,10 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
386 argv_array_pushl(&prune_worktrees, "worktree", "prune", "--expire", NULL);
387 argv_array_pushl(&rerere, "rerere", "gc", NULL);
388
389 + /* default expiry time, overwritten in gc_config */
390 gc_config();
391 + if (parse_expiry_date(gc_log_expire, &gc_log_expire_time))
392 + die(_("Failed to parse gc.logexpiry value %s"), gc_log_expire);
393
394 if (pack_refs < 0)
395 pack_refs = !is_bare_repository();
@@ -448,5 +488,8 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
488 warning(_("There are too many unreachable loose objects; "
489 "run 'git prune' to remove them."));
490
491 + if (!daemonized)
492 + unlink(git_path("gc.log"));
493 +
494 return 0;
495 }
t/t6500-gc.sh
+15
@@ -67,5 +67,20 @@ test_expect_success 'auto gc with too many loose objects does not attempt to cre
67 test_line_count = 2 new # There is one new pack and its .idx
68 '
69
70 +test_expect_success 'background auto gc does not run if gc.log is present and recent but does if it is old' '
71 + test_commit foo &&
72 + test_commit bar &&
73 + git repack &&
74 + test_config gc.autopacklimit 1 &&
75 + test_config gc.autodetach true &&
76 + echo fleem >.git/gc.log &&
77 + test_must_fail git gc --auto 2>err &&
78 + test_i18ngrep "^error:" err &&
79 + test_config gc.logexpiry 5.days &&
80 + test-chmtime =-345600 .git/gc.log &&
81 + test_must_fail git gc --auto &&
82 + test_config gc.logexpiry 2.days &&
83 + git gc --auto
84 +'
85
86 test_done