commit-graph: create options for split files
The split commit-graph feature is now fully implemented, but needs some more run-time configurability. Allow direct callers to 'git commit-graph write --split' to specify the values used in the merge strategy and the expire time. Update the documentation to specify these values. Signed-off-by: Derrick Stolee <dstolee@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Derrick Stolee committed
Jun 18, 2019 at 11:14 UTC
c2bc6e6ab0ade70c475a73d06326e677e70840d2
8 files changed
+128
-24
Documentation/git-commit-graph.txt
+20
-1
@@ -26,7 +26,7 @@ OPTIONS
26
Use given directory for the location of packfiles and commit-graph
27
file. This parameter exists to specify the location of an alternate
28
that only has the objects directory, not a full `.git` directory. The
29
- commit-graph file is expected to be at `<dir>/info/commit-graph` and
29
+ commit-graph file is expected to be in the `<dir>/info` directory and
30
the packfiles are expected to be in `<dir>/pack`.
31
32
@@ -51,6 +51,25 @@ or `--stdin-packs`.)
51
+
52
With the `--append` option, include all commits that are present in the
53
existing commit-graph file.
54
++
55
+With the `--split` option, write the commit-graph as a chain of multiple
56
+commit-graph files stored in `<dir>/info/commit-graphs`. The new commits
57
+not already in the commit-graph are added in a new "tip" file. This file
58
+is merged with the existing file if the following merge conditions are
59
+met:
60
++
61
+* If `--size-multiple=<X>` is not specified, let `X` equal 2. If the new
62
+tip file would have `N` commits and the previous tip has `M` commits and
63
+`X` times `N` is greater than `M`, instead merge the two files into a
64
+single file.
65
++
66
+* If `--max-commits=<M>` is specified with `M` a positive integer, and the
67
+new tip file would have more than `M` commits, then instead merge the new
68
+tip with the previous tip.
69
++
70
+Finally, if `--expire-time=<datetime>` is not specified, let `datetime`
71
+be the current time. After writing the split commit-graph, delete all
72
+unused commit-graph whose modified times are older than `datetime`.
73
74
'read'::
75
Documentation/technical/commit-graph.txt
+4
-3
@@ -248,10 +248,11 @@ When writing a set of commits that do not exist in the commit-graph stack of
248
height N, we default to creating a new file at level N + 1. We then decide to
249
merge with the Nth level if one of two conditions hold:
250
251
- 1. The expected file size for level N + 1 is at least half the file size for
252
- level N.
251
+ 1. `--size-multiple=<X>` is specified or X = 2, and the number of commits in
252
+ level N is less than X times the number of commits in level N + 1.
253
254
- 2. Level N + 1 contains more than 64,0000 commits.
254
+ 2. `--max-commits=<C>` is specified with non-zero C and the number of commits
255
+ in level N + 1 is more than C commits.
256
257
This decision cascades down the levels: when we merge a level we create a new
258
set of commits that then compares to the next level.
builtin/commit-graph.c
+20
-5
@@ -10,7 +10,7 @@ static char const * const builtin_commit_graph_usage[] = {
10
N_("git commit-graph [--object-dir <objdir>]"),
11
N_("git commit-graph read [--object-dir <objdir>]"),
12
N_("git commit-graph verify [--object-dir <objdir>]"),
13
- N_("git commit-graph write [--object-dir <objdir>] [--append|--split] [--reachable|--stdin-packs|--stdin-commits]"),
13
+ N_("git commit-graph write [--object-dir <objdir>] [--append|--split] [--reachable|--stdin-packs|--stdin-commits] <split options>"),
14
NULL
15
};
16
@@ -25,7 +25,7 @@ static const char * const builtin_commit_graph_read_usage[] = {
25
};
26
27
static const char * const builtin_commit_graph_write_usage[] = {
28
- N_("git commit-graph write [--object-dir <objdir>] [--append|--split] [--reachable|--stdin-packs|--stdin-commits]"),
28
+ N_("git commit-graph write [--object-dir <objdir>] [--append|--split] [--reachable|--stdin-packs|--stdin-commits] <split options>"),
29
NULL
30
};
31
@@ -135,6 +135,7 @@ static int graph_read(int argc, const char **argv)
135
}
136
137
extern int read_replace_refs;
138
+static struct split_commit_graph_opts split_opts;
139
140
static int graph_write(int argc, const char **argv)
141
{
@@ -158,9 +159,19 @@ static int graph_write(int argc, const char **argv)
159
N_("include all commits already in the commit-graph file")),
160
OPT_BOOL(0, "split", &opts.split,
161
N_("allow writing an incremental commit-graph file")),
162
+ OPT_INTEGER(0, "max-commits", &split_opts.max_commits,
163
+ N_("maximum number of commits in a non-base split commit-graph")),
164
+ OPT_INTEGER(0, "size-multiple", &split_opts.size_multiple,
165
+ N_("maximum ratio between two levels of a split commit-graph")),
166
+ OPT_EXPIRY_DATE(0, "expire-time", &split_opts.expire_time,
167
+ N_("maximum number of commits in a non-base split commit-graph")),
168
OPT_END(),
169
};
170
171
+ split_opts.size_multiple = 2;
172
+ split_opts.max_commits = 0;
173
+ split_opts.expire_time = 0;
174
+
175
argc = parse_options(argc, argv, NULL,
176
builtin_commit_graph_write_options,
177
builtin_commit_graph_write_usage, 0);
@@ -176,8 +187,11 @@ static int graph_write(int argc, const char **argv)
187
188
read_replace_refs = 0;
189
179
- if (opts.reachable)
180
- return write_commit_graph_reachable(opts.obj_dir, flags);
190
+ if (opts.reachable) {
191
+ if (write_commit_graph_reachable(opts.obj_dir, flags, &split_opts))
192
+ return 1;
193
+ return 0;
194
+ }
195
196
string_list_init(&lines, 0);
197
if (opts.stdin_packs || opts.stdin_commits) {
@@ -197,7 +211,8 @@ static int graph_write(int argc, const char **argv)
211
if (write_commit_graph(opts.obj_dir,
212
pack_indexes,
213
commit_hex,
200
- flags))
214
+ flags,
215
+ &split_opts))
216
result = 1;
217
218
UNLEAK(lines);
builtin/commit.c
+1
-1
@@ -1670,7 +1670,7 @@ int cmd_commit(int argc, const char **argv, const char *prefix)
1670
"not exceeded, and then \"git reset HEAD\" to recover."));
1671
1672
if (git_env_bool(GIT_TEST_COMMIT_GRAPH, 0) &&
1673
- write_commit_graph_reachable(get_object_directory(), 0))
1673
+ write_commit_graph_reachable(get_object_directory(), 0, NULL))
1674
return 1;
1675
1676
repo_rerere(the_repository, 0);
builtin/gc.c
+2
-1
@@ -666,7 +666,8 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
666
667
if (gc_write_commit_graph &&
668
write_commit_graph_reachable(get_object_directory(),
669
- !quiet && !daemonized ? COMMIT_GRAPH_PROGRESS : 0))
669
+ !quiet && !daemonized ? COMMIT_GRAPH_PROGRESS : 0,
670
+ NULL))
671
return 1;
672
673
if (auto_gc && too_many_loose_objects())
commit-graph.c
+24
-11
@@ -768,6 +768,8 @@ struct write_commit_graph_context {
768
unsigned append:1,
769
report_progress:1,
770
split:1;
771
+
772
+ const struct split_commit_graph_opts *split_opts;
773
};
774
775
static void write_graph_chunk_fanout(struct hashfile *f,
@@ -1116,14 +1118,15 @@ static int add_ref_to_list(const char *refname,
1118
return 0;
1119
}
1120
1119
-int write_commit_graph_reachable(const char *obj_dir, unsigned int flags)
1121
+int write_commit_graph_reachable(const char *obj_dir, unsigned int flags,
1122
+ const struct split_commit_graph_opts *split_opts)
1123
{
1124
struct string_list list = STRING_LIST_INIT_DUP;
1125
int result;
1126
1127
for_each_ref(add_ref_to_list, &list);
1128
result = write_commit_graph(obj_dir, NULL, &list,
1126
- flags);
1129
+ flags, split_opts);
1130
1131
string_list_clear(&list, 0);
1132
return result;
@@ -1498,20 +1501,25 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx)
1501
return 0;
1502
}
1503
1501
-static int split_strategy_max_commits = 64000;
1502
-static float split_strategy_size_mult = 2.0f;
1503
-
1504
static void split_graph_merge_strategy(struct write_commit_graph_context *ctx)
1505
{
1506
struct commit_graph *g = ctx->r->objects->commit_graph;
1507
uint32_t num_commits = ctx->commits.nr;
1508
uint32_t i;
1509
1510
+ int max_commits = 0;
1511
+ int size_mult = 2;
1512
+
1513
+ if (ctx->split_opts) {
1514
+ max_commits = ctx->split_opts->max_commits;
1515
+ size_mult = ctx->split_opts->size_multiple;
1516
+ }
1517
+
1518
g = ctx->r->objects->commit_graph;
1519
ctx->num_commit_graphs_after = ctx->num_commit_graphs_before + 1;
1520
1513
- while (g && (g->num_commits <= split_strategy_size_mult * num_commits ||
1514
- num_commits > split_strategy_max_commits)) {
1521
+ while (g && (g->num_commits <= size_mult * num_commits ||
1522
+ (max_commits && num_commits > max_commits))) {
1523
if (strcmp(g->obj_dir, ctx->obj_dir))
1524
break;
1525
@@ -1675,7 +1683,10 @@ static void expire_commit_graphs(struct write_commit_graph_context *ctx)
1683
DIR *dir;
1684
struct dirent *de;
1685
size_t dirnamelen;
1678
- time_t expire_time = time(NULL);
1686
+ timestamp_t expire_time = time(NULL);
1687
+
1688
+ if (ctx->split_opts && ctx->split_opts->expire_time)
1689
+ expire_time -= ctx->split_opts->expire_time;
1690
1691
strbuf_addstr(&path, ctx->obj_dir);
1692
strbuf_addstr(&path, "/info/commit-graphs");
@@ -1719,7 +1730,8 @@ static void expire_commit_graphs(struct write_commit_graph_context *ctx)
1730
int write_commit_graph(const char *obj_dir,
1731
struct string_list *pack_indexes,
1732
struct string_list *commit_hex,
1722
- unsigned int flags)
1733
+ unsigned int flags,
1734
+ const struct split_commit_graph_opts *split_opts)
1735
{
1736
struct write_commit_graph_context *ctx;
1737
uint32_t i, count_distinct = 0;
@@ -1734,6 +1746,7 @@ int write_commit_graph(const char *obj_dir,
1746
ctx->append = flags & COMMIT_GRAPH_APPEND ? 1 : 0;
1747
ctx->report_progress = flags & COMMIT_GRAPH_PROGRESS ? 1 : 0;
1748
ctx->split = flags & COMMIT_GRAPH_SPLIT ? 1 : 0;
1749
+ ctx->split_opts = split_opts;
1750
1751
if (ctx->split) {
1752
struct commit_graph *g;
@@ -1761,8 +1774,8 @@ int write_commit_graph(const char *obj_dir,
1774
ctx->approx_nr_objects = approximate_object_count();
1775
ctx->oids.alloc = ctx->approx_nr_objects / 32;
1776
1764
- if (ctx->split && ctx->oids.alloc > split_strategy_max_commits)
1765
- ctx->oids.alloc = split_strategy_max_commits;
1777
+ if (ctx->split && split_opts && ctx->oids.alloc > split_opts->max_commits)
1778
+ ctx->oids.alloc = split_opts->max_commits;
1779
1780
if (ctx->append) {
1781
prepare_commit_graph_one(ctx->r, ctx->obj_dir);
commit-graph.h
+10
-2
@@ -75,17 +75,25 @@ int generation_numbers_enabled(struct repository *r);
75
#define COMMIT_GRAPH_PROGRESS (1 << 1)
76
#define COMMIT_GRAPH_SPLIT (1 << 2)
77
78
+struct split_commit_graph_opts {
79
+ int size_multiple;
80
+ int max_commits;
81
+ timestamp_t expire_time;
82
+};
83
+
84
/*
85
* The write_commit_graph* methods return zero on success
86
* and a negative value on failure. Note that if the repository
87
* is not compatible with the commit-graph feature, then the
88
* methods will return 0 without writing a commit-graph.
89
*/
84
-int write_commit_graph_reachable(const char *obj_dir, unsigned int flags);
90
+int write_commit_graph_reachable(const char *obj_dir, unsigned int flags,
91
+ const struct split_commit_graph_opts *split_opts);
92
int write_commit_graph(const char *obj_dir,
93
struct string_list *pack_indexes,
94
struct string_list *commit_hex,
88
- unsigned int flags);
95
+ unsigned int flags,
96
+ const struct split_commit_graph_opts *split_opts);
97
98
int verify_commit_graph(struct repository *r, struct commit_graph *g);
99
t/t5324-split-commit-graph.sh
+47
@@ -169,4 +169,51 @@ test_expect_success 'create fork and chain across alternate' '
169
170
graph_git_behavior 'alternate: commit 13 vs 6' commits/13 commits/6
171
172
+test_expect_success 'test merge stragety constants' '
173
+ git clone . merge-2 &&
174
+ (
175
+ cd merge-2 &&
176
+ git config core.commitGraph true &&
177
+ test_line_count = 2 $graphdir/commit-graph-chain &&
178
+ test_commit 14 &&
179
+ git commit-graph write --reachable --split --size-multiple=2 &&
180
+ test_line_count = 3 $graphdir/commit-graph-chain
181
+
182
+ ) &&
183
+ git clone . merge-10 &&
184
+ (
185
+ cd merge-10 &&
186
+ git config core.commitGraph true &&
187
+ test_line_count = 2 $graphdir/commit-graph-chain &&
188
+ test_commit 14 &&
189
+ git commit-graph write --reachable --split --size-multiple=10 &&
190
+ test_line_count = 1 $graphdir/commit-graph-chain &&
191
+ ls $graphdir/graph-*.graph >graph-files &&
192
+ test_line_count = 1 graph-files
193
+ ) &&
194
+ git clone . merge-10-expire &&
195
+ (
196
+ cd merge-10-expire &&
197
+ git config core.commitGraph true &&
198
+ test_line_count = 2 $graphdir/commit-graph-chain &&
199
+ test_commit 15 &&
200
+ git commit-graph write --reachable --split --size-multiple=10 --expire-time=1980-01-01 &&
201
+ test_line_count = 1 $graphdir/commit-graph-chain &&
202
+ ls $graphdir/graph-*.graph >graph-files &&
203
+ test_line_count = 3 graph-files
204
+ ) &&
205
+ git clone --no-hardlinks . max-commits &&
206
+ (
207
+ cd max-commits &&
208
+ git config core.commitGraph true &&
209
+ test_line_count = 2 $graphdir/commit-graph-chain &&
210
+ test_commit 16 &&
211
+ test_commit 17 &&
212
+ git commit-graph write --reachable --split --max-commits=1 &&
213
+ test_line_count = 1 $graphdir/commit-graph-chain &&
214
+ ls $graphdir/graph-*.graph >graph-files &&
215
+ test_line_count = 1 graph-files
216
+ )
217
+'
218
+
219
test_done