commit-graph.c · leroysheep/rag-system

jch

bisect jch maint master next seen test todo

c 2,972 lines 79.2 KB

Raw

1	#define DISABLE_SIGN_COMPARE_WARNINGS
2
3	#include "git-compat-util.h"
4	#include "config.h"
5	#include "csum-file.h"
6	#include "environment.h"
7	#include "gettext.h"
8	#include "hex.h"
9	#include "lockfile.h"
10	#include "packfile.h"
11	#include "commit.h"
12	#include "object.h"
13	#include "refs.h"
14	#include "hash-lookup.h"
15	#include "commit-graph.h"
16	#include "odb.h"
17	#include "oid-array.h"
18	#include "path.h"
19	#include "alloc.h"
20	#include "hashmap.h"
21	#include "replace-object.h"
22	#include "progress.h"
23	#include "bloom.h"
24	#include "commit-slab.h"
25	#include "shallow.h"
26	#include "json-writer.h"
27	#include "trace2.h"
28	#include "tree.h"
29	#include "chunk-format.h"
30
31	void git_test_write_commit_graph_or_die(struct odb_source *source)
32	{
33	int flags = 0;
34	if (!git_env_bool(GIT_TEST_COMMIT_GRAPH, 0))
35	return;
36
37	if (git_env_bool(GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS, 0))
38	flags = COMMIT_GRAPH_WRITE_BLOOM_FILTERS;
39
40	if (write_commit_graph_reachable(source, flags, NULL))
41	die("failed to write commit-graph under GIT_TEST_COMMIT_GRAPH");
42	}
43
44	#define GRAPH_SIGNATURE 0x43475048 /* "CGPH" */
45	#define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
46	#define GRAPH_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
47	#define GRAPH_CHUNKID_DATA 0x43444154 /* "CDAT" */
48	#define GRAPH_CHUNKID_GENERATION_DATA 0x47444132 /* "GDA2" */
49	#define GRAPH_CHUNKID_GENERATION_DATA_OVERFLOW 0x47444f32 /* "GDO2" */
50	#define GRAPH_CHUNKID_EXTRAEDGES 0x45444745 /* "EDGE" */
51	#define GRAPH_CHUNKID_BLOOMINDEXES 0x42494458 /* "BIDX" */
52	#define GRAPH_CHUNKID_BLOOMDATA 0x42444154 /* "BDAT" */
53	#define GRAPH_CHUNKID_BASE 0x42415345 /* "BASE" */
54
55	#define GRAPH_VERSION_1 0x1
56	#define GRAPH_VERSION GRAPH_VERSION_1
57
58	#define GRAPH_EXTRA_EDGES_NEEDED 0x80000000
59	#define GRAPH_EDGE_LAST_MASK 0x7fffffff
60	#define GRAPH_PARENT_NONE 0x70000000
61
62	#define GRAPH_LAST_EDGE 0x80000000
63
64	#define GRAPH_HEADER_SIZE 8
65	#define GRAPH_FANOUT_SIZE (4 * 256)
66
67	#define CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW (1ULL << 31)
68
69	/* Remember to update object flag allocation in object.h */
70	#define REACHABLE (1u<<15)
71
72	define_commit_slab(topo_level_slab, uint32_t);
73
74	/* Keep track of the order in which commits are added to our list. */
75	define_commit_slab(commit_pos, int);
76	static struct commit_pos commit_pos = COMMIT_SLAB_INIT(1, commit_pos);
77
78	static size_t graph_data_width(const struct git_hash_algo *algop)
79	{
80	return algop->rawsz + 16;
81	}
82
83	static size_t graph_min_size(const struct git_hash_algo *algop)
84	{
85	return GRAPH_HEADER_SIZE + 4 * CHUNK_TOC_ENTRY_SIZE + GRAPH_FANOUT_SIZE + algop->rawsz;
86	}
87
88	static void set_commit_pos(struct repository r, const struct object_id oid)
89	{
90	static int32_t max_pos;
91	struct commit *commit = lookup_commit(r, oid);
92
93	if (!commit)
94	return; /* should never happen, but be lenient */
95
96	*commit_pos_at(&commit_pos, commit) = max_pos++;
97	}
98
99	static int commit_pos_cmp(const void va, const void vb)
100	{
101	const struct commit a = (const struct commit **)va;
102	const struct commit b = (const struct commit **)vb;
103	return commit_pos_at(&commit_pos, a) -
104	commit_pos_at(&commit_pos, b);
105	}
106
107	define_commit_slab(commit_graph_data_slab, struct commit_graph_data);
108	static struct commit_graph_data_slab commit_graph_data_slab =
109	COMMIT_SLAB_INIT(1, commit_graph_data_slab);
110
111	static int get_configured_generation_version(struct repository *r)
112	{
113	int version = 2;
114	repo_config_get_int(r, "commitgraph.generationversion", &version);
115	return version;
116	}
117
118	uint32_t commit_graph_position(const struct commit *c)
119	{
120	struct commit_graph_data *data =
121	commit_graph_data_slab_peek(&commit_graph_data_slab, c);
122
123	return data ? data->graph_pos : COMMIT_NOT_FROM_GRAPH;
124	}
125
126	timestamp_t commit_graph_generation(const struct commit *c)
127	{
128	struct commit_graph_data *data =
129	commit_graph_data_slab_peek(&commit_graph_data_slab, c);
130
131	if (data && data->generation)
132	return data->generation;
133
134	return GENERATION_NUMBER_INFINITY;
135	}
136
137	static timestamp_t commit_graph_generation_from_graph(const struct commit *c)
138	{
139	struct commit_graph_data *data =
140	commit_graph_data_slab_peek(&commit_graph_data_slab, c);
141
142	if (!data \|\| data->graph_pos == COMMIT_NOT_FROM_GRAPH)
143	return GENERATION_NUMBER_INFINITY;
144	return data->generation;
145	}
146
147	static struct commit_graph_data commit_graph_data_at(const struct commit c)
148	{
149	unsigned int i, nth_slab;
150	struct commit_graph_data *data =
151	commit_graph_data_slab_peek(&commit_graph_data_slab, c);
152
153	if (data)
154	return data;
155
156	nth_slab = c->index / commit_graph_data_slab.slab_size;
157	data = commit_graph_data_slab_at(&commit_graph_data_slab, c);
158
159	/*
160	* commit-slab initializes elements with zero, overwrite this with
161	* COMMIT_NOT_FROM_GRAPH for graph_pos.
162	*
163	* We avoid initializing generation with checking if graph position
164	* is not COMMIT_NOT_FROM_GRAPH.
165	*/
166	for (i = 0; i < commit_graph_data_slab.slab_size; i++) {
167	commit_graph_data_slab.slab[nth_slab][i].graph_pos =
168	COMMIT_NOT_FROM_GRAPH;
169	}
170
171	return data;
172	}
173
174	/*
175	* Should be used only while writing commit-graph as it compares
176	* generation value of commits by directly accessing commit-slab.
177	*/
178	static int commit_gen_cmp(const void va, const void vb)
179	{
180	const struct commit a = (const struct commit **)va;
181	const struct commit b = (const struct commit **)vb;
182
183	const timestamp_t generation_a = commit_graph_data_at(a)->generation;
184	const timestamp_t generation_b = commit_graph_data_at(b)->generation;
185	/* lower generation commits first */
186	if (generation_a < generation_b)
187	return -1;
188	else if (generation_a > generation_b)
189	return 1;
190
191	/* use date as a heuristic when generations are equal */
192	if (a->date < b->date)
193	return -1;
194	else if (a->date > b->date)
195	return 1;
196	return 0;
197	}
198
199	char get_commit_graph_filename(struct odb_source source)
200	{
201	return xstrfmt("%s/info/commit-graph", source->path);
202	}
203
204	static char get_split_graph_filename(struct odb_source source,
205	const char *oid_hex)
206	{
207	return xstrfmt("%s/info/commit-graphs/graph-%s.graph", source->path,
208	oid_hex);
209	}
210
211	char get_commit_graph_chain_filename(struct odb_source source)
212	{
213	return xstrfmt("%s/info/commit-graphs/commit-graph-chain", source->path);
214	}
215
216	static struct commit_graph *alloc_commit_graph(void)
217	{
218	struct commit_graph g = xcalloc(1, sizeof(g));
219
220	return g;
221	}
222
223	static int commit_graph_compatible(struct repository *r)
224	{
225	if (!r->gitdir)
226	return 0;
227
228	if (replace_refs_enabled(r)) {
229	prepare_replace_object(r);
230	if (oidmap_get_size(&r->objects->replace_map))
231	return 0;
232	}
233
234	prepare_commit_graft(r);
235	if (r->parsed_objects &&
236	(r->parsed_objects->grafts_nr \|\| r->parsed_objects->substituted_parent))
237	return 0;
238	if (is_repository_shallow(r))
239	return 0;
240
241	return 1;
242	}
243
244	int open_commit_graph(const char graph_file, int fd, struct stat *st)
245	{
246	*fd = git_open(graph_file);
247	if (*fd < 0)
248	return 0;
249	if (fstat(*fd, st)) {
250	close(*fd);
251	return 0;
252	}
253	return 1;
254	}
255
256	struct commit_graph load_commit_graph_one_fd_st(struct odb_source source,
257	int fd, struct stat *st)
258	{
259	void *graph_map;
260	size_t graph_size;
261	struct commit_graph *ret;
262
263	graph_size = xsize_t(st->st_size);
264
265	if (graph_size < graph_min_size(source->odb->repo->hash_algo)) {
266	close(fd);
267	error(_("commit-graph file is too small"));
268	return NULL;
269	}
270	graph_map = xmmap(NULL, graph_size, PROT_READ, MAP_PRIVATE, fd, 0);
271	close(fd);
272
273	ret = parse_commit_graph(source->odb->repo, graph_map, graph_size);
274	if (ret)
275	ret->odb_source = source;
276	else
277	munmap(graph_map, graph_size);
278
279	return ret;
280	}
281
282	static int graph_read_oid_fanout(const unsigned char *chunk_start,
283	size_t chunk_size, void *data)
284	{
285	struct commit_graph *g = data;
286	int i;
287
288	if (chunk_size != 256 * sizeof(uint32_t))
289	return error(_("commit-graph oid fanout chunk is wrong size"));
290	g->chunk_oid_fanout = (const uint32_t *)chunk_start;
291	g->num_commits = ntohl(g->chunk_oid_fanout[255]);
292
293	for (i = 0; i < 255; i++) {
294	uint32_t oid_fanout1 = ntohl(g->chunk_oid_fanout[i]);
295	uint32_t oid_fanout2 = ntohl(g->chunk_oid_fanout[i + 1]);
296
297	if (oid_fanout1 > oid_fanout2) {
298	error(_("commit-graph fanout values out of order"));
299	return 1;
300	}
301	}
302
303	return 0;
304	}
305
306	static int graph_read_oid_lookup(const unsigned char *chunk_start,
307	size_t chunk_size, void *data)
308	{
309	struct commit_graph *g = data;
310	g->chunk_oid_lookup = chunk_start;
311	if (chunk_size / g->hash_algo->rawsz != g->num_commits)
312	return error(_("commit-graph OID lookup chunk is the wrong size"));
313	return 0;
314	}
315
316	static int graph_read_commit_data(const unsigned char *chunk_start,
317	size_t chunk_size, void *data)
318	{
319	struct commit_graph *g = data;
320	if (chunk_size / graph_data_width(g->hash_algo) != g->num_commits)
321	return error(_("commit-graph commit data chunk is wrong size"));
322	g->chunk_commit_data = chunk_start;
323	return 0;
324	}
325
326	static int graph_read_generation_data(const unsigned char *chunk_start,
327	size_t chunk_size, void *data)
328	{
329	struct commit_graph *g = data;
330	if (chunk_size / sizeof(uint32_t) != g->num_commits)
331	return error(_("commit-graph generations chunk is wrong size"));
332	g->chunk_generation_data = chunk_start;
333	return 0;
334	}
335
336	static int graph_read_bloom_index(const unsigned char *chunk_start,
337	size_t chunk_size, void *data)
338	{
339	struct commit_graph *g = data;
340	if (chunk_size / 4 != g->num_commits) {
341	warning(_("commit-graph changed-path index chunk is too small"));
342	return -1;
343	}
344	g->chunk_bloom_indexes = chunk_start;
345	return 0;
346	}
347
348	static int graph_read_bloom_data(const unsigned char *chunk_start,
349	size_t chunk_size, void *data)
350	{
351	struct commit_graph *g = data;
352
353	if (chunk_size < BLOOMDATA_CHUNK_HEADER_SIZE) {
354	warning(_("ignoring too-small changed-path chunk"
355	" (%"PRIuMAX" < %"PRIuMAX") in commit-graph file"),
356	(uintmax_t)chunk_size,
357	(uintmax_t)BLOOMDATA_CHUNK_HEADER_SIZE);
358	return -1;
359	}
360
361	g->chunk_bloom_data = chunk_start;
362	g->chunk_bloom_data_size = chunk_size;
363
364	g->bloom_filter_settings = xmalloc(sizeof(struct bloom_filter_settings));
365	g->bloom_filter_settings->hash_version = get_be32(chunk_start);
366	g->bloom_filter_settings->num_hashes = get_be32(chunk_start + 4);
367	g->bloom_filter_settings->bits_per_entry = get_be32(chunk_start + 8);
368	g->bloom_filter_settings->max_changed_paths = DEFAULT_BLOOM_MAX_CHANGES;
369
370	return 0;
371	}
372
373	struct commit_graph parse_commit_graph(struct repository r,
374	void *graph_map, size_t graph_size)
375	{
376	const unsigned char *data;
377	struct commit_graph *graph;
378	uint32_t graph_signature;
379	unsigned char graph_version, hash_version;
380	struct chunkfile *cf = NULL;
381
382	if (!graph_map)
383	return NULL;
384
385	if (graph_size < graph_min_size(r->hash_algo))
386	return NULL;
387
388	data = (const unsigned char *)graph_map;
389
390	graph_signature = get_be32(data);
391	if (graph_signature != GRAPH_SIGNATURE) {
392	error(_("commit-graph signature %X does not match signature %X"),
393	graph_signature, GRAPH_SIGNATURE);
394	return NULL;
395	}
396
397	graph_version = (unsigned char)(data + 4);
398	if (graph_version != GRAPH_VERSION) {
399	error(_("commit-graph version %X does not match version %X"),
400	graph_version, GRAPH_VERSION);
401	return NULL;
402	}
403
404	hash_version = (unsigned char)(data + 5);
405	if (hash_version != oid_version(r->hash_algo)) {
406	error(_("commit-graph hash version %X does not match version %X"),
407	hash_version, oid_version(r->hash_algo));
408	return NULL;
409	}
410
411	graph = alloc_commit_graph();
412
413	graph->hash_algo = r->hash_algo;
414	graph->num_chunks = (unsigned char)(data + 6);
415	graph->data = graph_map;
416	graph->data_len = graph_size;
417
418	if (graph_size < GRAPH_HEADER_SIZE +
419	(graph->num_chunks + 1) * CHUNK_TOC_ENTRY_SIZE +
420	GRAPH_FANOUT_SIZE + r->hash_algo->rawsz) {
421	error(_("commit-graph file is too small to hold %u chunks"),
422	graph->num_chunks);
423	free(graph);
424	return NULL;
425	}
426
427	cf = init_chunkfile(NULL);
428
429	if (read_table_of_contents(cf, graph->data, graph_size,
430	GRAPH_HEADER_SIZE, graph->num_chunks, 1))
431	goto free_and_return;
432
433	if (read_chunk(cf, GRAPH_CHUNKID_OIDFANOUT, graph_read_oid_fanout, graph)) {
434	error(_("commit-graph required OID fanout chunk missing or corrupted"));
435	goto free_and_return;
436	}
437	if (read_chunk(cf, GRAPH_CHUNKID_OIDLOOKUP, graph_read_oid_lookup, graph)) {
438	error(_("commit-graph required OID lookup chunk missing or corrupted"));
439	goto free_and_return;
440	}
441	if (read_chunk(cf, GRAPH_CHUNKID_DATA, graph_read_commit_data, graph)) {
442	error(_("commit-graph required commit data chunk missing or corrupted"));
443	goto free_and_return;
444	}
445
446	pair_chunk(cf, GRAPH_CHUNKID_EXTRAEDGES, &graph->chunk_extra_edges,
447	&graph->chunk_extra_edges_size);
448	pair_chunk(cf, GRAPH_CHUNKID_BASE, &graph->chunk_base_graphs,
449	&graph->chunk_base_graphs_size);
450
451	prepare_repo_settings(r);
452
453	if (r->settings.commit_graph_generation_version >= 2) {
454	read_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA,
455	graph_read_generation_data, graph);
456	pair_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA_OVERFLOW,
457	&graph->chunk_generation_data_overflow,
458	&graph->chunk_generation_data_overflow_size);
459
460	if (graph->chunk_generation_data)
461	graph->read_generation_data = 1;
462	}
463
464	if (r->settings.commit_graph_changed_paths_version) {
465	read_chunk(cf, GRAPH_CHUNKID_BLOOMINDEXES,
466	graph_read_bloom_index, graph);
467	read_chunk(cf, GRAPH_CHUNKID_BLOOMDATA,
468	graph_read_bloom_data, graph);
469	}
470
471	if (graph->chunk_bloom_indexes && graph->chunk_bloom_data) {
472	init_bloom_filters();
473	} else {
474	/* We need both the bloom chunks to exist together. Else ignore the data */
475	graph->chunk_bloom_indexes = NULL;
476	graph->chunk_bloom_data = NULL;
477	FREE_AND_NULL(graph->bloom_filter_settings);
478	}
479
480	oidread(&graph->oid, graph->data + graph->data_len - graph->hash_algo->rawsz,
481	r->hash_algo);
482
483	free_chunkfile(cf);
484	return graph;
485
486	free_and_return:
487	free_chunkfile(cf);
488	free(graph->bloom_filter_settings);
489	free(graph);
490	return NULL;
491	}
492
493	static struct commit_graph load_commit_graph_one(struct odb_source source,
494	const char *graph_file)
495	{
496	struct stat st;
497	int fd;
498	struct commit_graph *g;
499	int open_ok = open_commit_graph(graph_file, &fd, &st);
500
501	if (!open_ok)
502	return NULL;
503
504	g = load_commit_graph_one_fd_st(source, fd, &st);
505	if (g)
506	g->filename = xstrdup(graph_file);
507
508	return g;
509	}
510
511	static struct commit_graph load_commit_graph_v1(struct odb_source source)
512	{
513	char *graph_name = get_commit_graph_filename(source);
514	struct commit_graph *g = load_commit_graph_one(source, graph_name);
515	free(graph_name);
516
517	return g;
518	}
519
520	/*
521	* returns 1 if and only if all graphs in the chain have
522	* corrected commit dates stored in the generation_data chunk.
523	*/
524	static int validate_mixed_generation_chain(struct commit_graph *g)
525	{
526	int read_generation_data = 1;
527	struct commit_graph *p = g;
528
529	while (read_generation_data && p) {
530	read_generation_data = p->read_generation_data;
531	p = p->base_graph;
532	}
533
534	if (read_generation_data)
535	return 1;
536
537	while (g) {
538	g->read_generation_data = 0;
539	g = g->base_graph;
540	}
541
542	return 0;
543	}
544
545	static void validate_mixed_bloom_settings(struct commit_graph *g)
546	{
547	struct bloom_filter_settings *settings = NULL;
548	for (; g; g = g->base_graph) {
549	if (!g->bloom_filter_settings)
550	continue;
551	if (!settings) {
552	settings = g->bloom_filter_settings;
553	continue;
554	}
555
556	if (g->bloom_filter_settings->bits_per_entry != settings->bits_per_entry \|\|
557	g->bloom_filter_settings->num_hashes != settings->num_hashes \|\|
558	g->bloom_filter_settings->hash_version != settings->hash_version) {
559	g->chunk_bloom_indexes = NULL;
560	g->chunk_bloom_data = NULL;
561	FREE_AND_NULL(g->bloom_filter_settings);
562
563	warning(_("disabling Bloom filters for commit-graph "
564	"layer '%s' due to incompatible settings"),
565	oid_to_hex(&g->oid));
566	}
567	}
568	}
569
570	static int add_graph_to_chain(struct commit_graph *g,
571	struct commit_graph *chain,
572	struct object_id *oids,
573	int n)
574	{
575	struct commit_graph *cur_g = chain;
576
577	if (n && !g->chunk_base_graphs) {
578	warning(_("commit-graph has no base graphs chunk"));
579	return 0;
580	}
581
582	if (g->chunk_base_graphs_size / g->hash_algo->rawsz < n) {
583	warning(_("commit-graph base graphs chunk is too small"));
584	return 0;
585	}
586
587	while (n) {
588	n--;
589
590	if (!cur_g \|\|
591	!oideq(&oids[n], &cur_g->oid) \|\|
592	!hasheq(oids[n].hash, g->chunk_base_graphs + st_mult(g->hash_algo->rawsz, n),
593	g->hash_algo)) {
594	warning(_("commit-graph chain does not match"));
595	return 0;
596	}
597
598	cur_g = cur_g->base_graph;
599	}
600
601	if (chain) {
602	if (unsigned_add_overflows(chain->num_commits,
603	chain->num_commits_in_base)) {
604	warning(_("commit count in base graph too high: %"PRIuMAX),
605	(uintmax_t)chain->num_commits_in_base);
606	return 0;
607	}
608	g->num_commits_in_base = chain->num_commits + chain->num_commits_in_base;
609	}
610
611	g->base_graph = chain;
612
613	return 1;
614	}
615
616	int open_commit_graph_chain(const char *chain_file,
617	int fd, struct stat st,
618	const struct git_hash_algo *hash_algo)
619	{
620	*fd = git_open(chain_file);
621	if (*fd < 0)
622	return 0;
623	if (fstat(*fd, st)) {
624	close(*fd);
625	return 0;
626	}
627	if (st->st_size < hash_algo->hexsz) {
628	close(*fd);
629	if (!st->st_size) {
630	/* treat empty files the same as missing */
631	errno = ENOENT;
632	} else {
633	warning(_("commit-graph chain file too small"));
634	errno = EINVAL;
635	}
636	return 0;
637	}
638	return 1;
639	}
640
641	struct commit_graph load_commit_graph_chain_fd_st(struct object_database odb,
642	int fd, struct stat *st,
643	int *incomplete_chain)
644	{
645	struct commit_graph *graph_chain = NULL;
646	struct strbuf line = STRBUF_INIT;
647	struct object_id *oids;
648	int i = 0, valid = 1, count;
649	FILE *fp = xfdopen(fd, "r");
650
651	count = st->st_size / (odb->repo->hash_algo->hexsz + 1);
652	CALLOC_ARRAY(oids, count);
653
654	odb_prepare_alternates(odb);
655
656	for (i = 0; i < count; i++) {
657	struct odb_source *source;
658
659	if (strbuf_getline_lf(&line, fp) == EOF)
660	break;
661
662	if (get_oid_hex_algop(line.buf, &oids[i], odb->repo->hash_algo)) {
663	warning(_("invalid commit-graph chain: line '%s' not a hash"),
664	line.buf);
665	valid = 0;
666	break;
667	}
668
669	valid = 0;
670	for (source = odb->sources; source; source = source->next) {
671	char *graph_name = get_split_graph_filename(source, line.buf);
672	struct commit_graph *g = load_commit_graph_one(source, graph_name);
673
674	free(graph_name);
675
676	if (g) {
677	if (add_graph_to_chain(g, graph_chain, oids, i)) {
678	graph_chain = g;
679	valid = 1;
680	} else {
681	free_commit_graph(g);
682	}
683
684	break;
685	}
686	}
687
688	if (!valid) {
689	warning(_("unable to find all commit-graph files"));
690	break;
691	}
692	}
693
694	validate_mixed_generation_chain(graph_chain);
695	validate_mixed_bloom_settings(graph_chain);
696
697	free(oids);
698	fclose(fp);
699	strbuf_release(&line);
700
701	*incomplete_chain = !valid;
702	return graph_chain;
703	}
704
705	static struct commit_graph load_commit_graph_chain(struct odb_source source)
706	{
707	char *chain_file = get_commit_graph_chain_filename(source);
708	struct stat st;
709	int fd;
710	struct commit_graph *g = NULL;
711
712	if (open_commit_graph_chain(chain_file, &fd, &st, source->odb->repo->hash_algo)) {
713	int incomplete;
714	/* ownership of fd is taken over by load function */
715	g = load_commit_graph_chain_fd_st(source->odb, fd, &st, &incomplete);
716	}
717
718	free(chain_file);
719	return g;
720	}
721
722	struct commit_graph read_commit_graph_one(struct odb_source source)
723	{
724	struct commit_graph *g = load_commit_graph_v1(source);
725
726	if (!g)
727	g = load_commit_graph_chain(source);
728
729	return g;
730	}
731
732	/*
733	* Return 1 if commit_graph is non-NULL, and 0 otherwise.
734	*
735	* On the first invocation, this function attempts to load the commit
736	* graph if the repository is configured to have one.
737	*/
738	static struct commit_graph prepare_commit_graph(struct repository r)
739	{
740	struct odb_source *source;
741
742	/*
743	* Early return if there is no object database or if the commit graph is
744	* disabled.
745	*
746	* This must come before the "already attempted?" check below, because
747	* we want to disable even an already-loaded graph file.
748	*/
749	if (!r->objects \|\| r->commit_graph_disabled)
750	return NULL;
751
752	if (r->objects->commit_graph_attempted)
753	return r->objects->commit_graph;
754	r->objects->commit_graph_attempted = 1;
755
756	prepare_repo_settings(r);
757
758	if (!git_env_bool(GIT_TEST_COMMIT_GRAPH, 0) &&
759	r->settings.core_commit_graph != 1)
760	/*
761	* This repository is not configured to use commit graphs, so
762	* do not load one. (But report commit_graph_attempted anyway
763	* so that commit graph loading is not attempted again for this
764	* repository.)
765	*/
766	return NULL;
767
768	if (!commit_graph_compatible(r))
769	return NULL;
770
771	odb_prepare_alternates(r->objects);
772	for (source = r->objects->sources; source; source = source->next) {
773	r->objects->commit_graph = read_commit_graph_one(source);
774	if (r->objects->commit_graph)
775	break;
776	}
777
778	return r->objects->commit_graph;
779	}
780
781	int generation_numbers_enabled(struct repository *r)
782	{
783	uint32_t first_generation;
784	struct commit_graph *g;
785
786	g = prepare_commit_graph(r);
787	if (!g \|\| !g->num_commits)
788	return 0;
789
790	first_generation = get_be32(g->chunk_commit_data +
791	g->hash_algo->rawsz + 8) >> 2;
792
793	return !!first_generation;
794	}
795
796	int corrected_commit_dates_enabled(struct repository *r)
797	{
798	struct commit_graph *g;
799
800	g = prepare_commit_graph(r);
801	if (!g \|\| !g->num_commits)
802	return 0;
803
804	return g->read_generation_data;
805	}
806
807	struct bloom_filter_settings get_bloom_filter_settings(struct repository r)
808	{
809	struct commit_graph *g;
810
811	if (!prepare_commit_graph(r))
812	return NULL;
813
814	g = r->objects->commit_graph;
815	while (g) {
816	if (g->bloom_filter_settings)
817	return g->bloom_filter_settings;
818	g = g->base_graph;
819	}
820	return NULL;
821	}
822
823	void close_commit_graph(struct object_database *o)
824	{
825	if (!o->commit_graph)
826	return;
827
828	clear_commit_graph_data_slab(&commit_graph_data_slab);
829	deinit_bloom_filters();
830	free_commit_graph(o->commit_graph);
831	o->commit_graph = NULL;
832	}
833
834	static int bsearch_graph(struct commit_graph g, const struct object_id oid, uint32_t *pos)
835	{
836	return bsearch_hash(oid->hash, g->chunk_oid_fanout,
837	g->chunk_oid_lookup, g->hash_algo->rawsz, pos);
838	}
839
840	static void load_oid_from_graph(struct commit_graph *g,
841	uint32_t pos,
842	struct object_id *oid)
843	{
844	uint32_t lex_index;
845
846	while (g && pos < g->num_commits_in_base)
847	g = g->base_graph;
848
849	if (!g)
850	BUG("NULL commit-graph");
851
852	if (pos >= g->num_commits + g->num_commits_in_base)
853	die(_("invalid commit position. commit-graph is likely corrupt"));
854
855	lex_index = pos - g->num_commits_in_base;
856
857	oidread(oid, g->chunk_oid_lookup + st_mult(g->hash_algo->rawsz, lex_index),
858	g->hash_algo);
859	}
860
861	static struct commit_list *insert_parent_or_die(struct commit_graph g,
862	uint32_t pos,
863	struct commit_list **pptr)
864	{
865	struct commit *c;
866	struct object_id oid;
867
868	if (pos >= g->num_commits + g->num_commits_in_base)
869	die("invalid parent position %"PRIu32, pos);
870
871	load_oid_from_graph(g, pos, &oid);
872	c = lookup_commit(g->odb_source->odb->repo, &oid);
873	if (!c)
874	die(_("could not find commit %s"), oid_to_hex(&oid));
875	commit_graph_data_at(c)->graph_pos = pos;
876	return &commit_list_insert(c, pptr)->next;
877	}
878
879	static void fill_commit_graph_info(struct commit item, struct commit_graph g, uint32_t pos)
880	{
881	const unsigned char *commit_data;
882	struct commit_graph_data *graph_data;
883	uint32_t lex_index, offset_pos;
884	uint64_t date_high, date_low, offset;
885
886	while (pos < g->num_commits_in_base)
887	g = g->base_graph;
888
889	if (pos >= g->num_commits + g->num_commits_in_base)
890	die(_("invalid commit position. commit-graph is likely corrupt"));
891
892	lex_index = pos - g->num_commits_in_base;
893	commit_data = g->chunk_commit_data + st_mult(graph_data_width(g->hash_algo), lex_index);
894
895	graph_data = commit_graph_data_at(item);
896	graph_data->graph_pos = pos;
897
898	date_high = get_be32(commit_data + g->hash_algo->rawsz + 8) & 0x3;
899	date_low = get_be32(commit_data + g->hash_algo->rawsz + 12);
900	item->date = (timestamp_t)((date_high << 32) \| date_low);
901
902	if (g->read_generation_data) {
903	offset = (timestamp_t)get_be32(g->chunk_generation_data + st_mult(sizeof(uint32_t), lex_index));
904
905	if (offset & CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW) {
906	if (!g->chunk_generation_data_overflow)
907	die(_("commit-graph requires overflow generation data but has none"));
908
909	offset_pos = offset ^ CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW;
910	if (g->chunk_generation_data_overflow_size / sizeof(uint64_t) <= offset_pos)
911	die(_("commit-graph overflow generation data is too small"));
912	graph_data->generation = item->date +
913	get_be64(g->chunk_generation_data_overflow + sizeof(uint64_t) * offset_pos);
914	} else
915	graph_data->generation = item->date + offset;
916	} else
917	graph_data->generation = get_be32(commit_data + g->hash_algo->rawsz + 8) >> 2;
918
919	if (g->topo_levels)
920	*topo_level_slab_at(g->topo_levels, item) = get_be32(commit_data + g->hash_algo->rawsz + 8) >> 2;
921	}
922
923	static inline void set_commit_tree(struct commit c, struct tree t)
924	{
925	c->maybe_tree = t;
926	}
927
928	static int fill_commit_in_graph(struct commit *item,
929	struct commit_graph *g, uint32_t pos)
930	{
931	uint32_t edge_value;
932	uint32_t parent_data_pos;
933	struct commit_list **pptr;
934	const unsigned char *commit_data;
935	uint32_t lex_index;
936
937	while (pos < g->num_commits_in_base)
938	g = g->base_graph;
939
940	fill_commit_graph_info(item, g, pos);
941
942	lex_index = pos - g->num_commits_in_base;
943	commit_data = g->chunk_commit_data + st_mult(g->hash_algo->rawsz + 16, lex_index);
944
945	item->object.parsed = 1;
946
947	set_commit_tree(item, NULL);
948
949	pptr = &item->parents;
950
951	edge_value = get_be32(commit_data + g->hash_algo->rawsz);
952	if (edge_value == GRAPH_PARENT_NONE)
953	return 1;
954	pptr = insert_parent_or_die(g, edge_value, pptr);
955
956	edge_value = get_be32(commit_data + g->hash_algo->rawsz + 4);
957	if (edge_value == GRAPH_PARENT_NONE)
958	return 1;
959	if (!(edge_value & GRAPH_EXTRA_EDGES_NEEDED)) {
960	pptr = insert_parent_or_die(g, edge_value, pptr);
961	return 1;
962	}
963
964	parent_data_pos = edge_value & GRAPH_EDGE_LAST_MASK;
965	do {
966	if (g->chunk_extra_edges_size / sizeof(uint32_t) <= parent_data_pos) {
967	error(_("commit-graph extra-edges pointer out of bounds"));
968	commit_list_free(item->parents);
969	item->parents = NULL;
970	item->object.parsed = 0;
971	return 0;
972	}
973	edge_value = get_be32(g->chunk_extra_edges +
974	sizeof(uint32_t) * parent_data_pos);
975	pptr = insert_parent_or_die(g,
976	edge_value & GRAPH_EDGE_LAST_MASK,
977	pptr);
978	parent_data_pos++;
979	} while (!(edge_value & GRAPH_LAST_EDGE));
980
981	return 1;
982	}
983
984	static int search_commit_pos_in_graph(const struct object_id id, struct commit_graph g, uint32_t *pos)
985	{
986	struct commit_graph *cur_g = g;
987	uint32_t lex_index;
988
989	while (cur_g && !bsearch_graph(cur_g, id, &lex_index))
990	cur_g = cur_g->base_graph;
991
992	if (cur_g) {
993	*pos = lex_index + cur_g->num_commits_in_base;
994	return 1;
995	}
996
997	return 0;
998	}
999
1000	static int find_commit_pos_in_graph(struct commit item, struct commit_graph g, uint32_t *pos)
1001	{
1002	uint32_t graph_pos = commit_graph_position(item);
1003	if (graph_pos != COMMIT_NOT_FROM_GRAPH) {
1004	*pos = graph_pos;
1005	return 1;
1006	} else {
1007	return search_commit_pos_in_graph(&item->object.oid, g, pos);
1008	}
1009	}
1010
1011	struct commit_graph repo_find_commit_pos_in_graph(struct repository r,
1012	struct commit *c,
1013	uint32_t *pos)
1014	{
1015	struct commit_graph *g = prepare_commit_graph(r);
1016	if (!g)
1017	return NULL;
1018	if (!find_commit_pos_in_graph(c, g, pos))
1019	return NULL;
1020	return g;
1021	}
1022
1023	struct commit lookup_commit_in_graph(struct repository repo, const struct object_id *id)
1024	{
1025	static int commit_graph_paranoia = -1;
1026	struct commit_graph *g;
1027	struct commit *commit;
1028	uint32_t pos;
1029
1030	if (commit_graph_paranoia == -1)
1031	commit_graph_paranoia = git_env_bool(GIT_COMMIT_GRAPH_PARANOIA, 0);
1032
1033	g = prepare_commit_graph(repo);
1034	if (!g)
1035	return NULL;
1036	if (!search_commit_pos_in_graph(id, g, &pos))
1037	return NULL;
1038	if (commit_graph_paranoia && !odb_has_object(repo->objects, id, 0))
1039	return NULL;
1040
1041	commit = lookup_commit(repo, id);
1042	if (!commit)
1043	return NULL;
1044	if (commit->object.parsed)
1045	return commit;
1046
1047	if (!fill_commit_in_graph(commit, g, pos))
1048	return NULL;
1049
1050	return commit;
1051	}
1052
1053	static int parse_commit_in_graph_one(struct commit_graph *g,
1054	struct commit *item)
1055	{
1056	uint32_t pos;
1057
1058	if (item->object.parsed)
1059	return 1;
1060
1061	if (find_commit_pos_in_graph(item, g, &pos))
1062	return fill_commit_in_graph(item, g, pos);
1063
1064	return 0;
1065	}
1066
1067	int parse_commit_in_graph(struct repository r, struct commit item)
1068	{
1069	static int checked_env = 0;
1070	struct commit_graph *g;
1071
1072	if (!checked_env &&
1073	git_env_bool(GIT_TEST_COMMIT_GRAPH_DIE_ON_PARSE, 0))
1074	die("dying as requested by the '%s' variable on commit-graph parse!",
1075	GIT_TEST_COMMIT_GRAPH_DIE_ON_PARSE);
1076	checked_env = 1;
1077
1078	g = prepare_commit_graph(r);
1079	if (!g)
1080	return 0;
1081	return parse_commit_in_graph_one(g, item);
1082	}
1083
1084	void load_commit_graph_info(struct repository r, struct commit item)
1085	{
1086	struct commit_graph *g;
1087	uint32_t pos;
1088
1089	g = repo_find_commit_pos_in_graph(r, item, &pos);
1090	if (g)
1091	fill_commit_graph_info(item, g, pos);
1092	}
1093
1094	static struct tree load_tree_for_commit(struct commit_graph g,
1095	struct commit *c)
1096	{
1097	struct object_id oid;
1098	const unsigned char *commit_data;
1099	uint32_t graph_pos = commit_graph_position(c);
1100
1101	while (graph_pos < g->num_commits_in_base)
1102	g = g->base_graph;
1103
1104	commit_data = g->chunk_commit_data +
1105	st_mult(graph_data_width(g->hash_algo),
1106	graph_pos - g->num_commits_in_base);
1107
1108	oidread(&oid, commit_data, g->hash_algo);
1109	set_commit_tree(c, lookup_tree(g->odb_source->odb->repo, &oid));
1110
1111	return c->maybe_tree;
1112	}
1113
1114	static struct tree get_commit_tree_in_graph_one(struct commit_graph g,
1115	const struct commit *c)
1116	{
1117	if (c->maybe_tree)
1118	return c->maybe_tree;
1119	if (commit_graph_position(c) == COMMIT_NOT_FROM_GRAPH)
1120	BUG("get_commit_tree_in_graph_one called from non-commit-graph commit");
1121
1122	return load_tree_for_commit(g, (struct commit *)c);
1123	}
1124
1125	struct tree get_commit_tree_in_graph(struct repository r, const struct commit *c)
1126	{
1127	return get_commit_tree_in_graph_one(r->objects->commit_graph, c);
1128	}
1129
1130	struct write_commit_graph_context {
1131	struct repository *r;
1132	struct odb_source *odb_source;
1133	char *graph_name;
1134	struct oid_array oids;
1135	struct commit_stack commits;
1136	int num_extra_edges;
1137	int num_generation_data_overflows;
1138	unsigned long approx_nr_objects;
1139	struct progress *progress;
1140	int progress_done;
1141	uint64_t progress_cnt;
1142
1143	char *base_graph_name;
1144	int num_commit_graphs_before;
1145	int num_commit_graphs_after;
1146	char **commit_graph_filenames_before;
1147	char **commit_graph_filenames_after;
1148	char **commit_graph_hash_after;
1149	uint32_t new_num_commits_in_base;
1150	struct commit_graph *new_base_graph;
1151
1152	unsigned append:1,
1153	report_progress:1,
1154	split:1,
1155	changed_paths:1,
1156	order_by_pack:1,
1157	write_generation_data:1,
1158	trust_generation_numbers:1;
1159
1160	struct topo_level_slab *topo_levels;
1161	const struct commit_graph_opts *opts;
1162	size_t total_bloom_filter_data_size;
1163	const struct bloom_filter_settings *bloom_settings;
1164
1165	int count_bloom_filter_computed;
1166	int count_bloom_filter_not_computed;
1167	int count_bloom_filter_trunc_empty;
1168	int count_bloom_filter_trunc_large;
1169	int count_bloom_filter_upgraded;
1170	};
1171
1172	static int write_graph_chunk_fanout(struct hashfile *f,
1173	void *data)
1174	{
1175	struct write_commit_graph_context *ctx = data;
1176	int i, count = 0;
1177	struct commit **list = ctx->commits.items;
1178
1179	/*
1180	* Write the first-level table (the list is sorted,
1181	* but we use a 256-entry lookup to be able to avoid
1182	* having to do eight extra binary search iterations).
1183	*/
1184	for (i = 0; i < 256; i++) {
1185	while (count < ctx->commits.nr) {
1186	if ((*list)->object.oid.hash[0] != i)
1187	break;
1188	display_progress(ctx->progress, ++ctx->progress_cnt);
1189	count++;
1190	list++;
1191	}
1192
1193	hashwrite_be32(f, count);
1194	}
1195
1196	return 0;
1197	}
1198
1199	static int write_graph_chunk_oids(struct hashfile *f,
1200	void *data)
1201	{
1202	struct write_commit_graph_context *ctx = data;
1203	struct commit **list = ctx->commits.items;
1204	int count;
1205	for (count = 0; count < ctx->commits.nr; count++, list++) {
1206	display_progress(ctx->progress, ++ctx->progress_cnt);
1207	hashwrite(f, (*list)->object.oid.hash, f->algop->rawsz);
1208	}
1209
1210	return 0;
1211	}
1212
1213	static const struct object_id commit_to_oid(size_t index, const void table)
1214	{
1215	const struct commit * const *commits = table;
1216	return &commits[index]->object.oid;
1217	}
1218
1219	static int write_graph_chunk_data(struct hashfile *f,
1220	void *data)
1221	{
1222	struct write_commit_graph_context *ctx = data;
1223	struct commit **list = ctx->commits.items;
1224	struct commit **last = ctx->commits.items + ctx->commits.nr;
1225	uint32_t num_extra_edges = 0;
1226
1227	while (list < last) {
1228	struct commit_list *parent;
1229	struct object_id *tree;
1230	int edge_value;
1231	uint32_t packedDate[2];
1232	display_progress(ctx->progress, ++ctx->progress_cnt);
1233
1234	if (repo_parse_commit_no_graph(ctx->r, *list))
1235	die(_("unable to parse commit %s"),
1236	oid_to_hex(&(*list)->object.oid));
1237	tree = get_commit_tree_oid(*list);
1238	hashwrite(f, tree->hash, ctx->r->hash_algo->rawsz);
1239
1240	parent = (*list)->parents;
1241
1242	if (!parent)
1243	edge_value = GRAPH_PARENT_NONE;
1244	else {
1245	edge_value = oid_pos(&parent->item->object.oid,
1246	ctx->commits.items,
1247	ctx->commits.nr,
1248	commit_to_oid);
1249
1250	if (edge_value >= 0)
1251	edge_value += ctx->new_num_commits_in_base;
1252	else if (ctx->new_base_graph) {
1253	uint32_t pos;
1254	if (find_commit_pos_in_graph(parent->item,
1255	ctx->new_base_graph,
1256	&pos))
1257	edge_value = pos;
1258	}
1259
1260	if (edge_value < 0)
1261	BUG("missing parent %s for commit %s",
1262	oid_to_hex(&parent->item->object.oid),
1263	oid_to_hex(&(*list)->object.oid));
1264	}
1265
1266	hashwrite_be32(f, edge_value);
1267
1268	if (parent)
1269	parent = parent->next;
1270
1271	if (!parent)
1272	edge_value = GRAPH_PARENT_NONE;
1273	else if (parent->next)
1274	edge_value = GRAPH_EXTRA_EDGES_NEEDED \| num_extra_edges;
1275	else {
1276	edge_value = oid_pos(&parent->item->object.oid,
1277	ctx->commits.items,
1278	ctx->commits.nr,
1279	commit_to_oid);
1280
1281	if (edge_value >= 0)
1282	edge_value += ctx->new_num_commits_in_base;
1283	else if (ctx->new_base_graph) {
1284	uint32_t pos;
1285	if (find_commit_pos_in_graph(parent->item,
1286	ctx->new_base_graph,
1287	&pos))
1288	edge_value = pos;
1289	}
1290
1291	if (edge_value < 0)
1292	BUG("missing parent %s for commit %s",
1293	oid_to_hex(&parent->item->object.oid),
1294	oid_to_hex(&(*list)->object.oid));
1295	}
1296
1297	hashwrite_be32(f, edge_value);
1298
1299	if (edge_value & GRAPH_EXTRA_EDGES_NEEDED) {
1300	do {
1301	num_extra_edges++;
1302	parent = parent->next;
1303	} while (parent);
1304	}
1305
1306	if (sizeof((*list)->date) > 4)
1307	packedDate[0] = htonl(((*list)->date >> 32) & 0x3);
1308	else
1309	packedDate[0] = 0;
1310
1311	packedDate[0] \|= htonl(topo_level_slab_at(ctx->topo_levels, list) << 2);
1312
1313	packedDate[1] = htonl((*list)->date);
1314	hashwrite(f, packedDate, 8);
1315
1316	list++;
1317	}
1318
1319	return 0;
1320	}
1321
1322	/*
1323	* Compute the generation offset between the commit date and its generation.
1324	* This is what's ultimately stored as generation number in the commit graph.
1325	*
1326	* Note that the computation of the commit date is more involved than you might
1327	* think. Instead of using the full commit date, we're in fact masking bits so
1328	* that only the 34 lowest bits are considered. This results from the fact that
1329	* commit graphs themselves only ever store 34 bits of the commit date
1330	* themselves.
1331	*
1332	* This means that if we have a commit date that exceeds 34 bits we'll end up
1333	* in situations where depending on whether the commit has been parsed from the
1334	* object database or the commit graph we'll have different dates, where the
1335	* ones parsed from the object database would have full 64 bit precision.
1336	*
1337	* But ultimately, we only ever want the offset to be relative to what we
1338	* actually end up storing on disk, and hence we have to mask all the other
1339	* bits.
1340	*/
1341	static timestamp_t compute_generation_offset(struct commit *c)
1342	{
1343	timestamp_t masked_date;
1344
1345	if (sizeof(timestamp_t) > 4)
1346	masked_date = c->date & (((timestamp_t) 1 << 34) - 1);
1347	else
1348	masked_date = c->date;
1349
1350	return commit_graph_data_at(c)->generation - masked_date;
1351	}
1352
1353	static int write_graph_chunk_generation_data(struct hashfile *f,
1354	void *data)
1355	{
1356	struct write_commit_graph_context *ctx = data;
1357	int i, num_generation_data_overflows = 0;
1358
1359	for (i = 0; i < ctx->commits.nr; i++) {
1360	struct commit *c = ctx->commits.items[i];
1361	timestamp_t offset;
1362	repo_parse_commit(ctx->r, c);
1363	offset = compute_generation_offset(c);
1364	display_progress(ctx->progress, ++ctx->progress_cnt);
1365
1366	if (offset > GENERATION_NUMBER_V2_OFFSET_MAX) {
1367	offset = CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW \| num_generation_data_overflows;
1368	num_generation_data_overflows++;
1369	}
1370
1371	hashwrite_be32(f, offset);
1372	}
1373
1374	return 0;
1375	}
1376
1377	static int write_graph_chunk_generation_data_overflow(struct hashfile *f,
1378	void *data)
1379	{
1380	struct write_commit_graph_context *ctx = data;
1381	int i;
1382	for (i = 0; i < ctx->commits.nr; i++) {
1383	struct commit *c = ctx->commits.items[i];
1384	timestamp_t offset = compute_generation_offset(c);
1385	display_progress(ctx->progress, ++ctx->progress_cnt);
1386
1387	if (offset > GENERATION_NUMBER_V2_OFFSET_MAX) {
1388	hashwrite_be32(f, offset >> 32);
1389	hashwrite_be32(f, (uint32_t) offset);
1390	}
1391	}
1392
1393	return 0;
1394	}
1395
1396	static int write_graph_chunk_extra_edges(struct hashfile *f,
1397	void *data)
1398	{
1399	struct write_commit_graph_context *ctx = data;
1400	struct commit **list = ctx->commits.items;
1401	struct commit **last = ctx->commits.items + ctx->commits.nr;
1402	struct commit_list *parent;
1403
1404	while (list < last) {
1405	int num_parents = 0;
1406
1407	display_progress(ctx->progress, ++ctx->progress_cnt);
1408
1409	for (parent = (*list)->parents; num_parents < 3 && parent;
1410	parent = parent->next)
1411	num_parents++;
1412
1413	if (num_parents <= 2) {
1414	list++;
1415	continue;
1416	}
1417
1418	/* Since num_parents > 2, this initializer is safe. */
1419	for (parent = (*list)->parents->next; parent; parent = parent->next) {
1420	int edge_value = oid_pos(&parent->item->object.oid,
1421	ctx->commits.items,
1422	ctx->commits.nr,
1423	commit_to_oid);
1424
1425	if (edge_value >= 0)
1426	edge_value += ctx->new_num_commits_in_base;
1427	else if (ctx->new_base_graph) {
1428	uint32_t pos;
1429	if (find_commit_pos_in_graph(parent->item,
1430	ctx->new_base_graph,
1431	&pos))
1432	edge_value = pos;
1433	}
1434
1435	if (edge_value < 0)
1436	BUG("missing parent %s for commit %s",
1437	oid_to_hex(&parent->item->object.oid),
1438	oid_to_hex(&(*list)->object.oid));
1439	else if (!parent->next)
1440	edge_value \|= GRAPH_LAST_EDGE;
1441
1442	hashwrite_be32(f, edge_value);
1443	}
1444
1445	list++;
1446	}
1447
1448	return 0;
1449	}
1450
1451	static int write_graph_chunk_bloom_indexes(struct hashfile *f,
1452	void *data)
1453	{
1454	struct write_commit_graph_context *ctx = data;
1455	struct commit **list = ctx->commits.items;
1456	struct commit **last = ctx->commits.items + ctx->commits.nr;
1457	uint32_t cur_pos = 0;
1458
1459	while (list < last) {
1460	struct bloom_filter filter = get_bloom_filter(ctx->r, list);
1461	size_t len = filter ? filter->len : 0;
1462	cur_pos += len;
1463	display_progress(ctx->progress, ++ctx->progress_cnt);
1464	hashwrite_be32(f, cur_pos);
1465	list++;
1466	}
1467
1468	return 0;
1469	}
1470
1471	static void trace2_bloom_filter_settings(struct write_commit_graph_context *ctx)
1472	{
1473	struct json_writer jw = JSON_WRITER_INIT;
1474
1475	jw_object_begin(&jw, 0);
1476	jw_object_intmax(&jw, "hash_version", ctx->bloom_settings->hash_version);
1477	jw_object_intmax(&jw, "num_hashes", ctx->bloom_settings->num_hashes);
1478	jw_object_intmax(&jw, "bits_per_entry", ctx->bloom_settings->bits_per_entry);
1479	jw_object_intmax(&jw, "max_changed_paths", ctx->bloom_settings->max_changed_paths);
1480	jw_end(&jw);
1481
1482	trace2_data_json("bloom", ctx->r, "settings", &jw);
1483
1484	jw_release(&jw);
1485	}
1486
1487	static int write_graph_chunk_bloom_data(struct hashfile *f,
1488	void *data)
1489	{
1490	struct write_commit_graph_context *ctx = data;
1491	struct commit **list = ctx->commits.items;
1492	struct commit **last = ctx->commits.items + ctx->commits.nr;
1493
1494	trace2_bloom_filter_settings(ctx);
1495
1496	hashwrite_be32(f, ctx->bloom_settings->hash_version);
1497	hashwrite_be32(f, ctx->bloom_settings->num_hashes);
1498	hashwrite_be32(f, ctx->bloom_settings->bits_per_entry);
1499
1500	while (list < last) {
1501	struct bloom_filter filter = get_bloom_filter(ctx->r, list);
1502	size_t len = filter ? filter->len : 0;
1503
1504	display_progress(ctx->progress, ++ctx->progress_cnt);
1505	if (len)
1506	hashwrite(f, filter->data, len * sizeof(unsigned char));
1507	list++;
1508	}
1509
1510	return 0;
1511	}
1512
1513	static int add_packed_commits_oi(const struct object_id *oid,
1514	struct object_info *oi,
1515	void *data)
1516	{
1517	struct write_commit_graph_context ctx = (struct write_commit_graph_context)data;
1518
1519	if (ctx->progress)
1520	display_progress(ctx->progress, ++ctx->progress_done);
1521
1522	if (*oi->typep != OBJ_COMMIT)
1523	return 0;
1524
1525	oid_array_append(&ctx->oids, oid);
1526	set_commit_pos(ctx->r, oid);
1527
1528	return 0;
1529	}
1530
1531	static int add_packed_commits(const struct object_id *oid,
1532	struct packed_git *pack,
1533	uint32_t pos,
1534	void *data)
1535	{
1536	enum object_type type;
1537	off_t offset = nth_packed_object_offset(pack, pos);
1538	struct object_info oi = OBJECT_INFO_INIT;
1539
1540	oi.typep = &type;
1541	if (packed_object_info(NULL, pack, offset, &oi) < 0)
1542	die(_("unable to get type of object %s"), oid_to_hex(oid));
1543
1544	return add_packed_commits_oi(oid, &oi, data);
1545	}
1546
1547	static void add_missing_parents(struct write_commit_graph_context ctx, struct commit commit)
1548	{
1549	struct commit_list *parent;
1550	for (parent = commit->parents; parent; parent = parent->next) {
1551	if (!(parent->item->object.flags & REACHABLE)) {
1552	oid_array_append(&ctx->oids, &parent->item->object.oid);
1553	parent->item->object.flags \|= REACHABLE;
1554	}
1555	}
1556	}
1557
1558	static void close_reachable(struct write_commit_graph_context *ctx)
1559	{
1560	int i;
1561	struct commit *commit;
1562	enum commit_graph_split_flags flags = ctx->opts ?
1563	ctx->opts->split_flags : COMMIT_GRAPH_SPLIT_UNSPECIFIED;
1564
1565	if (ctx->report_progress)
1566	ctx->progress = start_delayed_progress(
1567	ctx->r,
1568	_("Loading known commits in commit graph"),
1569	ctx->oids.nr);
1570	for (i = 0; i < ctx->oids.nr; i++) {
1571	display_progress(ctx->progress, i + 1);
1572	commit = lookup_commit(ctx->r, &ctx->oids.oid[i]);
1573	if (commit)
1574	commit->object.flags \|= REACHABLE;
1575	}
1576	stop_progress(&ctx->progress);
1577
1578	/*
1579	* As this loop runs, ctx->oids.nr may grow, but not more
1580	* than the number of missing commits in the reachable
1581	* closure.
1582	*/
1583	if (ctx->report_progress)
1584	ctx->progress = start_delayed_progress(
1585	ctx->r,
1586	_("Expanding reachable commits in commit graph"),
1587	0);
1588	for (i = 0; i < ctx->oids.nr; i++) {
1589	display_progress(ctx->progress, i + 1);
1590	commit = lookup_commit(ctx->r, &ctx->oids.oid[i]);
1591
1592	if (!commit)
1593	continue;
1594	if (ctx->split) {
1595	if ((!repo_parse_commit(ctx->r, commit) &&
1596	commit_graph_position(commit) == COMMIT_NOT_FROM_GRAPH) \|\|
1597	flags == COMMIT_GRAPH_SPLIT_REPLACE)
1598	add_missing_parents(ctx, commit);
1599	} else if (!repo_parse_commit_no_graph(ctx->r, commit))
1600	add_missing_parents(ctx, commit);
1601	}
1602	stop_progress(&ctx->progress);
1603
1604	if (ctx->report_progress)
1605	ctx->progress = start_delayed_progress(
1606	ctx->r,
1607	_("Clearing commit marks in commit graph"),
1608	ctx->oids.nr);
1609	for (i = 0; i < ctx->oids.nr; i++) {
1610	display_progress(ctx->progress, i + 1);
1611	commit = lookup_commit(ctx->r, &ctx->oids.oid[i]);
1612
1613	if (commit)
1614	commit->object.flags &= ~REACHABLE;
1615	}
1616	stop_progress(&ctx->progress);
1617	}
1618
1619	struct compute_generation_info {
1620	struct repository *r;
1621	struct commit_stack *commits;
1622	struct progress *progress;
1623	int progress_cnt;
1624
1625	timestamp_t (get_generation)(struct commit c, void *data);
1626	void (set_generation)(struct commit c, timestamp_t gen, void *data);
1627	void *data;
1628	};
1629
1630	static timestamp_t compute_generation_from_max(struct commit *c,
1631	timestamp_t max_gen,
1632	int generation_version)
1633	{
1634	switch (generation_version) {
1635	case 1: /* topological levels */
1636	if (max_gen > GENERATION_NUMBER_V1_MAX - 1)
1637	max_gen = GENERATION_NUMBER_V1_MAX - 1;
1638	return max_gen + 1;
1639
1640	case 2: /* corrected commit date */
1641	if (c->date && c->date > max_gen)
1642	max_gen = c->date - 1;
1643	return max_gen + 1;
1644
1645	default:
1646	BUG("attempting unimplemented version");
1647	}
1648	}
1649
1650	static void compute_reachable_generation_numbers(
1651	struct compute_generation_info *info,
1652	int generation_version)
1653	{
1654	int i;
1655	struct commit_list *list = NULL;
1656	intmax_t steps = 0;
1657
1658	for (i = 0; i < info->commits->nr; i++) {
1659	struct commit *c = info->commits->items[i];
1660	timestamp_t gen;
1661	repo_parse_commit(info->r, c);
1662	gen = info->get_generation(c, info->data);
1663	display_progress(info->progress, ++info->progress_cnt);
1664
1665	if (gen != GENERATION_NUMBER_ZERO && gen != GENERATION_NUMBER_INFINITY)
1666	continue;
1667
1668	commit_list_insert(c, &list);
1669	while (list) {
1670	struct commit *current = list->item;
1671	struct commit_list *parent;
1672	int all_parents_computed = 1;
1673	timestamp_t max_gen = 0;
1674
1675	steps++;
1676	for (parent = current->parents; parent; parent = parent->next) {
1677	repo_parse_commit(info->r, parent->item);
1678	gen = info->get_generation(parent->item, info->data);
1679
1680	if (gen == GENERATION_NUMBER_ZERO) {
1681	all_parents_computed = 0;
1682	commit_list_insert(parent->item, &list);
1683	break;
1684	}
1685
1686	if (gen > max_gen)
1687	max_gen = gen;
1688	}
1689
1690	if (all_parents_computed) {
1691	pop_commit(&list);
1692	gen = compute_generation_from_max(
1693	current, max_gen,
1694	generation_version);
1695	info->set_generation(current, gen, info->data);
1696	}
1697	}
1698	}
1699
1700	trace2_data_intmax("commit-graph", info->r,
1701	"generation-dfs-steps", steps);
1702	}
1703
1704	static timestamp_t get_topo_level(struct commit c, void data)
1705	{
1706	struct write_commit_graph_context *ctx = data;
1707	return *topo_level_slab_at(ctx->topo_levels, c);
1708	}
1709
1710	static void set_topo_level(struct commit c, timestamp_t t, void data)
1711	{
1712	struct write_commit_graph_context *ctx = data;
1713	*topo_level_slab_at(ctx->topo_levels, c) = (uint32_t)t;
1714	}
1715
1716	static void compute_topological_levels(struct write_commit_graph_context *ctx)
1717	{
1718	struct compute_generation_info info = {
1719	.r = ctx->r,
1720	.commits = &ctx->commits,
1721	.get_generation = get_topo_level,
1722	.set_generation = set_topo_level,
1723	.data = ctx,
1724	};
1725
1726	if (ctx->report_progress)
1727	info.progress = ctx->progress
1728	= start_delayed_progress(
1729	ctx->r,
1730	_("Computing commit graph topological levels"),
1731	ctx->commits.nr);
1732
1733	compute_reachable_generation_numbers(&info, 1);
1734
1735	stop_progress(&ctx->progress);
1736	}
1737
1738	static timestamp_t get_generation_from_graph_data(struct commit *c,
1739	void *data UNUSED)
1740	{
1741	return commit_graph_data_at(c)->generation;
1742	}
1743
1744	static void set_generation_v2(struct commit *c, timestamp_t t,
1745	void *data UNUSED)
1746	{
1747	struct commit_graph_data *g = commit_graph_data_at(c);
1748	g->generation = t;
1749	}
1750
1751	static void compute_generation_numbers(struct write_commit_graph_context *ctx)
1752	{
1753	int i;
1754	struct compute_generation_info info = {
1755	.r = ctx->r,
1756	.commits = &ctx->commits,
1757	.get_generation = get_generation_from_graph_data,
1758	.set_generation = set_generation_v2,
1759	};
1760
1761	if (ctx->report_progress)
1762	info.progress = ctx->progress
1763	= start_delayed_progress(
1764	ctx->r,
1765	_("Computing commit graph generation numbers"),
1766	ctx->commits.nr);
1767
1768	if (!ctx->trust_generation_numbers) {
1769	for (i = 0; i < ctx->commits.nr; i++) {
1770	struct commit *c = ctx->commits.items[i];
1771	repo_parse_commit(ctx->r, c);
1772	commit_graph_data_at(c)->generation = GENERATION_NUMBER_ZERO;
1773	}
1774	}
1775
1776	compute_reachable_generation_numbers(&info, 2);
1777
1778	for (i = 0; i < ctx->commits.nr; i++) {
1779	struct commit *c = ctx->commits.items[i];
1780	timestamp_t offset = compute_generation_offset(c);
1781	if (offset > GENERATION_NUMBER_V2_OFFSET_MAX)
1782	ctx->num_generation_data_overflows++;
1783	}
1784	stop_progress(&ctx->progress);
1785	}
1786
1787	static void set_generation_in_graph_data(struct commit *c, timestamp_t t,
1788	void *data UNUSED)
1789	{
1790	commit_graph_data_at(c)->generation = t;
1791	}
1792
1793	/*
1794	* After this method, all commits reachable from those in the given
1795	* list will have non-zero, non-infinite generation numbers.
1796	*/
1797	void ensure_generations_valid(struct repository *r,
1798	struct commit **commits, size_t nr)
1799	{
1800	int generation_version = get_configured_generation_version(r);
1801	struct commit_stack list = {
1802	.items = commits,
1803	.alloc = nr,
1804	.nr = nr,
1805	};
1806	struct compute_generation_info info = {
1807	.r = r,
1808	.commits = &list,
1809	.get_generation = get_generation_from_graph_data,
1810	.set_generation = set_generation_in_graph_data,
1811	};
1812
1813	compute_reachable_generation_numbers(&info, generation_version);
1814	}
1815
1816	static void trace2_bloom_filter_write_statistics(struct write_commit_graph_context *ctx)
1817	{
1818	trace2_data_intmax("commit-graph", ctx->r, "filter-computed",
1819	ctx->count_bloom_filter_computed);
1820	trace2_data_intmax("commit-graph", ctx->r, "filter-not-computed",
1821	ctx->count_bloom_filter_not_computed);
1822	trace2_data_intmax("commit-graph", ctx->r, "filter-trunc-empty",
1823	ctx->count_bloom_filter_trunc_empty);
1824	trace2_data_intmax("commit-graph", ctx->r, "filter-trunc-large",
1825	ctx->count_bloom_filter_trunc_large);
1826	trace2_data_intmax("commit-graph", ctx->r, "filter-upgraded",
1827	ctx->count_bloom_filter_upgraded);
1828	}
1829
1830	static void compute_bloom_filters(struct write_commit_graph_context *ctx)
1831	{
1832	int i;
1833	struct progress *progress = NULL;
1834	struct commit **sorted_commits;
1835	int max_new_filters;
1836
1837	init_bloom_filters();
1838
1839	if (ctx->report_progress)
1840	progress = start_delayed_progress(
1841	ctx->r,
1842	_("Computing commit changed paths Bloom filters"),
1843	ctx->commits.nr);
1844
1845	DUP_ARRAY(sorted_commits, ctx->commits.items, ctx->commits.nr);
1846
1847	if (ctx->order_by_pack)
1848	QSORT(sorted_commits, ctx->commits.nr, commit_pos_cmp);
1849	else
1850	QSORT(sorted_commits, ctx->commits.nr, commit_gen_cmp);
1851
1852	max_new_filters = ctx->opts && ctx->opts->max_new_filters >= 0 ?
1853	ctx->opts->max_new_filters : ctx->commits.nr;
1854
1855	for (i = 0; i < ctx->commits.nr; i++) {
1856	enum bloom_filter_computed computed = 0;
1857	struct commit *c = sorted_commits[i];
1858	struct bloom_filter *filter = get_or_compute_bloom_filter(
1859	ctx->r,
1860	c,
1861	ctx->count_bloom_filter_computed < max_new_filters,
1862	ctx->bloom_settings,
1863	&computed);
1864	if (computed & BLOOM_COMPUTED) {
1865	ctx->count_bloom_filter_computed++;
1866	if (computed & BLOOM_TRUNC_EMPTY)
1867	ctx->count_bloom_filter_trunc_empty++;
1868	if (computed & BLOOM_TRUNC_LARGE)
1869	ctx->count_bloom_filter_trunc_large++;
1870	} else if (computed & BLOOM_UPGRADED) {
1871	ctx->count_bloom_filter_upgraded++;
1872	} else if (computed & BLOOM_NOT_COMPUTED)
1873	ctx->count_bloom_filter_not_computed++;
1874	ctx->total_bloom_filter_data_size += filter
1875	? sizeof(unsigned char) * filter->len : 0;
1876	display_progress(progress, i + 1);
1877	}
1878
1879	if (trace2_is_enabled())
1880	trace2_bloom_filter_write_statistics(ctx);
1881
1882	free(sorted_commits);
1883	stop_progress(&progress);
1884	}
1885
1886	struct refs_cb_data {
1887	struct repository *repo;
1888	struct oidset *commits;
1889	struct progress *progress;
1890	};
1891
1892	static int add_ref_to_set(const struct reference ref, void cb_data)
1893	{
1894	const struct object_id *maybe_peeled = ref->oid;
1895	struct object_id peeled;
1896	struct refs_cb_data data = (struct refs_cb_data )cb_data;
1897
1898	if (!reference_get_peeled_oid(data->repo, ref, &peeled))
1899	maybe_peeled = &peeled;
1900	if (odb_read_object_info(data->repo->objects, maybe_peeled, NULL) == OBJ_COMMIT)
1901	oidset_insert(data->commits, maybe_peeled);
1902
1903	display_progress(data->progress, oidset_size(data->commits));
1904
1905	return 0;
1906	}
1907
1908	int write_commit_graph_reachable(struct odb_source *source,
1909	enum commit_graph_write_flags flags,
1910	const struct commit_graph_opts *opts)
1911	{
1912	struct oidset commits = OIDSET_INIT;
1913	struct refs_cb_data data;
1914	int result;
1915
1916	memset(&data, 0, sizeof(data));
1917	data.repo = source->odb->repo;
1918	data.commits = &commits;
1919
1920	if (flags & COMMIT_GRAPH_WRITE_PROGRESS)
1921	data.progress = start_delayed_progress(
1922	source->odb->repo,
1923	_("Collecting referenced commits"), 0);
1924
1925	refs_for_each_ref(get_main_ref_store(source->odb->repo), add_ref_to_set,
1926	&data);
1927
1928	stop_progress(&data.progress);
1929
1930	result = write_commit_graph(source, NULL, &commits,
1931	flags, opts);
1932
1933	oidset_clear(&commits);
1934	return result;
1935	}
1936
1937	static int fill_oids_from_packs(struct write_commit_graph_context *ctx,
1938	const struct string_list *pack_indexes)
1939	{
1940	uint32_t i;
1941	struct strbuf progress_title = STRBUF_INIT;
1942	struct strbuf packname = STRBUF_INIT;
1943	int dirlen;
1944	int ret = 0;
1945
1946	strbuf_addf(&packname, "%s/pack/", ctx->odb_source->path);
1947	dirlen = packname.len;
1948	if (ctx->report_progress) {
1949	strbuf_addf(&progress_title,
1950	Q_("Finding commits for commit graph in %"PRIuMAX" pack",
1951	"Finding commits for commit graph in %"PRIuMAX" packs",
1952	pack_indexes->nr),
1953	(uintmax_t)pack_indexes->nr);
1954	ctx->progress = start_delayed_progress(ctx->r,
1955	progress_title.buf, 0);
1956	ctx->progress_done = 0;
1957	}
1958	for (i = 0; i < pack_indexes->nr; i++) {
1959	struct packed_git *p;
1960	strbuf_setlen(&packname, dirlen);
1961	strbuf_addstr(&packname, pack_indexes->items[i].string);
1962	p = add_packed_git(ctx->r, packname.buf, packname.len, 1);
1963	if (!p) {
1964	ret = error(_("error adding pack %s"), packname.buf);
1965	goto cleanup;
1966	}
1967	if (open_pack_index(p)) {
1968	ret = error(_("error opening index for %s"), packname.buf);
1969	close_pack(p);
1970	free(p);
1971	goto cleanup;
1972	}
1973	for_each_object_in_pack(p, add_packed_commits, ctx,
1974	ODB_FOR_EACH_OBJECT_PACK_ORDER);
1975	close_pack(p);
1976	free(p);
1977	}
1978
1979	cleanup:
1980	stop_progress(&ctx->progress);
1981	strbuf_release(&progress_title);
1982	strbuf_release(&packname);
1983
1984	return ret;
1985	}
1986
1987	static int fill_oids_from_commits(struct write_commit_graph_context *ctx,
1988	struct oidset *commits)
1989	{
1990	struct oidset_iter iter;
1991	struct object_id *oid;
1992
1993	if (!oidset_size(commits))
1994	return 0;
1995
1996	oidset_iter_init(commits, &iter);
1997	while ((oid = oidset_iter_next(&iter))) {
1998	oid_array_append(&ctx->oids, oid);
1999	}
2000
2001	return 0;
2002	}
2003
2004	static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx)
2005	{
2006	struct odb_source *source;
2007	enum object_type type;
2008	struct odb_for_each_object_options opts = {
2009	.flags = ODB_FOR_EACH_OBJECT_PACK_ORDER,
2010	};
2011	struct object_info oi = {
2012	.typep = &type,
2013	};
2014
2015	if (ctx->report_progress)
2016	ctx->progress = start_delayed_progress(
2017	ctx->r,
2018	_("Finding commits for commit graph among packed objects"),
2019	ctx->approx_nr_objects);
2020
2021	odb_prepare_alternates(ctx->r->objects);
2022	for (source = ctx->r->objects->sources; source; source = source->next) {
2023	struct odb_source_files *files = odb_source_files_downcast(source);
2024	odb_source_for_each_object(&files->packed->base, &oi, add_packed_commits_oi,
2025	ctx, &opts);
2026	}
2027
2028	if (ctx->progress_done < ctx->approx_nr_objects)
2029	display_progress(ctx->progress, ctx->approx_nr_objects);
2030	stop_progress(&ctx->progress);
2031	}
2032
2033	static void copy_oids_to_commits(struct write_commit_graph_context *ctx)
2034	{
2035	uint32_t i;
2036	enum commit_graph_split_flags flags = ctx->opts ?
2037	ctx->opts->split_flags : COMMIT_GRAPH_SPLIT_UNSPECIFIED;
2038
2039	ctx->num_extra_edges = 0;
2040	if (ctx->report_progress)
2041	ctx->progress = start_delayed_progress(
2042	ctx->r,
2043	_("Finding extra edges in commit graph"),
2044	ctx->oids.nr);
2045	oid_array_sort(&ctx->oids);
2046	for (i = 0; i < ctx->oids.nr; i = oid_array_next_unique(&ctx->oids, i)) {
2047	unsigned int num_parents;
2048	struct commit *commit;
2049
2050	display_progress(ctx->progress, i + 1);
2051
2052	commit = lookup_commit(ctx->r, &ctx->oids.oid[i]);
2053
2054	if (ctx->split && flags != COMMIT_GRAPH_SPLIT_REPLACE &&
2055	commit_graph_position(commit) != COMMIT_NOT_FROM_GRAPH)
2056	continue;
2057
2058	if (ctx->split && flags == COMMIT_GRAPH_SPLIT_REPLACE)
2059	repo_parse_commit(ctx->r, commit);
2060	else
2061	repo_parse_commit_no_graph(ctx->r, commit);
2062
2063	num_parents = commit_list_count(commit->parents);
2064	if (num_parents > 2)
2065	ctx->num_extra_edges += num_parents - 1;
2066
2067	commit_stack_push(&ctx->commits, commit);
2068	}
2069	stop_progress(&ctx->progress);
2070	}
2071
2072	static int write_graph_chunk_base_1(struct hashfile *f,
2073	struct commit_graph *g)
2074	{
2075	int num = 0;
2076
2077	if (!g)
2078	return 0;
2079
2080	num = write_graph_chunk_base_1(f, g->base_graph);
2081	hashwrite(f, g->oid.hash, g->hash_algo->rawsz);
2082	return num + 1;
2083	}
2084
2085	static int write_graph_chunk_base(struct hashfile *f,
2086	void *data)
2087	{
2088	struct write_commit_graph_context *ctx = data;
2089	int num = write_graph_chunk_base_1(f, ctx->new_base_graph);
2090
2091	if (num != ctx->num_commit_graphs_after - 1) {
2092	error(_("failed to write correct number of base graph ids"));
2093	return -1;
2094	}
2095
2096	return 0;
2097	}
2098
2099	static int write_commit_graph_file(struct write_commit_graph_context *ctx)
2100	{
2101	uint32_t i;
2102	struct hashfile *f;
2103	struct tempfile graph_layer; / when ctx->split is non-zero */
2104	struct lock_file lk = LOCK_INIT;
2105	const unsigned hashsz = ctx->r->hash_algo->rawsz;
2106	struct strbuf progress_title = STRBUF_INIT;
2107	struct chunkfile *cf;
2108	unsigned char file_hash[GIT_MAX_RAWSZ];
2109
2110	if (ctx->split) {
2111	struct strbuf tmp_file = STRBUF_INIT;
2112
2113	strbuf_addf(&tmp_file,
2114	"%s/info/commit-graphs/tmp_graph_XXXXXX",
2115	ctx->odb_source->path);
2116	ctx->graph_name = strbuf_detach(&tmp_file, NULL);
2117	} else {
2118	ctx->graph_name = get_commit_graph_filename(ctx->odb_source);
2119	}
2120
2121	if (safe_create_leading_directories(ctx->r, ctx->graph_name)) {
2122	error(_("unable to create leading directories of %s"),
2123	ctx->graph_name);
2124	return -1;
2125	}
2126
2127	if (ctx->split) {
2128	char *lock_name = get_commit_graph_chain_filename(ctx->odb_source);
2129
2130	repo_hold_lock_file_for_update_mode(ctx->r, &lk, lock_name,
2131	LOCK_DIE_ON_ERROR, 0444);
2132	free(lock_name);
2133
2134	graph_layer = mks_tempfile_m(ctx->graph_name, 0444);
2135	if (!graph_layer) {
2136	error(_("unable to create temporary graph layer"));
2137	return -1;
2138	}
2139
2140	if (adjust_shared_perm(ctx->r, get_tempfile_path(graph_layer))) {
2141	error(_("unable to adjust shared permissions for '%s'"),
2142	get_tempfile_path(graph_layer));
2143	return -1;
2144	}
2145
2146	f = hashfd(ctx->r->hash_algo,
2147	get_tempfile_fd(graph_layer), get_tempfile_path(graph_layer));
2148	} else {
2149	repo_hold_lock_file_for_update_mode(ctx->r, &lk,
2150	ctx->graph_name,
2151	LOCK_DIE_ON_ERROR, 0444);
2152	f = hashfd(ctx->r->hash_algo,
2153	get_lock_file_fd(&lk), get_lock_file_path(&lk));
2154	}
2155
2156	cf = init_chunkfile(f);
2157
2158	add_chunk(cf, GRAPH_CHUNKID_OIDFANOUT, GRAPH_FANOUT_SIZE,
2159	write_graph_chunk_fanout);
2160	add_chunk(cf, GRAPH_CHUNKID_OIDLOOKUP, st_mult(hashsz, ctx->commits.nr),
2161	write_graph_chunk_oids);
2162	add_chunk(cf, GRAPH_CHUNKID_DATA, st_mult(hashsz + 16, ctx->commits.nr),
2163	write_graph_chunk_data);
2164
2165	if (ctx->write_generation_data)
2166	add_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA,
2167	st_mult(sizeof(uint32_t), ctx->commits.nr),
2168	write_graph_chunk_generation_data);
2169	if (ctx->num_generation_data_overflows)
2170	add_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA_OVERFLOW,
2171	st_mult(sizeof(timestamp_t), ctx->num_generation_data_overflows),
2172	write_graph_chunk_generation_data_overflow);
2173	if (ctx->num_extra_edges)
2174	add_chunk(cf, GRAPH_CHUNKID_EXTRAEDGES,
2175	st_mult(4, ctx->num_extra_edges),
2176	write_graph_chunk_extra_edges);
2177	if (ctx->changed_paths) {
2178	add_chunk(cf, GRAPH_CHUNKID_BLOOMINDEXES,
2179	st_mult(sizeof(uint32_t), ctx->commits.nr),
2180	write_graph_chunk_bloom_indexes);
2181	add_chunk(cf, GRAPH_CHUNKID_BLOOMDATA,
2182	st_add(sizeof(uint32_t) * 3,
2183	ctx->total_bloom_filter_data_size),
2184	write_graph_chunk_bloom_data);
2185	}
2186	if (ctx->num_commit_graphs_after > 1)
2187	add_chunk(cf, GRAPH_CHUNKID_BASE,
2188	st_mult(hashsz, ctx->num_commit_graphs_after - 1),
2189	write_graph_chunk_base);
2190
2191	hashwrite_be32(f, GRAPH_SIGNATURE);
2192
2193	hashwrite_u8(f, GRAPH_VERSION);
2194	hashwrite_u8(f, oid_version(ctx->r->hash_algo));
2195	hashwrite_u8(f, get_num_chunks(cf));
2196	hashwrite_u8(f, ctx->num_commit_graphs_after - 1);
2197
2198	if (ctx->report_progress) {
2199	strbuf_addf(&progress_title,
2200	Q_("Writing out commit graph in %d pass",
2201	"Writing out commit graph in %d passes",
2202	get_num_chunks(cf)),
2203	get_num_chunks(cf));
2204	ctx->progress = start_delayed_progress(
2205	ctx->r,
2206	progress_title.buf,
2207	st_mult(get_num_chunks(cf), ctx->commits.nr));
2208	}
2209
2210	write_chunkfile(cf, ctx);
2211
2212	stop_progress(&ctx->progress);
2213	strbuf_release(&progress_title);
2214
2215	if (ctx->split && ctx->base_graph_name && ctx->num_commit_graphs_after > 1) {
2216	char *new_base_hash = xstrdup(oid_to_hex(&ctx->new_base_graph->oid));
2217	char *new_base_name = get_split_graph_filename(ctx->new_base_graph->odb_source, new_base_hash);
2218
2219	free(ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 2]);
2220	free(ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 2]);
2221	ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 2] = new_base_name;
2222	ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 2] = new_base_hash;
2223	}
2224
2225	close_commit_graph(ctx->r->objects);
2226	finalize_hashfile(f, file_hash, FSYNC_COMPONENT_COMMIT_GRAPH,
2227	CSUM_HASH_IN_STREAM \| CSUM_FSYNC);
2228	free_chunkfile(cf);
2229
2230	if (ctx->split) {
2231	FILE *chainf = fdopen_lock_file(&lk, "w");
2232	char *final_graph_name;
2233	int result;
2234
2235	if (!chainf) {
2236	error(_("unable to open commit-graph chain file"));
2237	return -1;
2238	}
2239
2240	if (ctx->base_graph_name) {
2241	const char *dest;
2242	int idx = ctx->num_commit_graphs_after - 1;
2243	if (ctx->num_commit_graphs_after > 1)
2244	idx--;
2245
2246	dest = ctx->commit_graph_filenames_after[idx];
2247
2248	if (strcmp(ctx->base_graph_name, dest)) {
2249	result = rename(ctx->base_graph_name, dest);
2250
2251	if (result) {
2252	error(_("failed to rename base commit-graph file"));
2253	return -1;
2254	}
2255	}
2256	} else {
2257	char *graph_name = get_commit_graph_filename(ctx->odb_source);
2258	unlink(graph_name);
2259	free(graph_name);
2260	}
2261
2262	free(ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1]);
2263	ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1] =
2264	xstrdup(hash_to_hex_algop(file_hash, ctx->r->hash_algo));
2265	final_graph_name = get_split_graph_filename(ctx->odb_source,
2266	ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1]);
2267	free(ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 1]);
2268	ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 1] = final_graph_name;
2269
2270	result = rename_tempfile(&graph_layer, final_graph_name);
2271
2272	for (i = 0; i < ctx->num_commit_graphs_after; i++)
2273	fprintf(get_lock_file_fp(&lk), "%s\n", ctx->commit_graph_hash_after[i]);
2274
2275	if (result) {
2276	error(_("failed to rename temporary commit-graph file"));
2277	return -1;
2278	}
2279	}
2280
2281	commit_lock_file(&lk);
2282
2283	return 0;
2284	}
2285
2286	static void split_graph_merge_strategy(struct write_commit_graph_context *ctx,
2287	struct commit_graph *graph_to_merge)
2288	{
2289	struct commit_graph *g;
2290	uint32_t num_commits;
2291	enum commit_graph_split_flags flags = COMMIT_GRAPH_SPLIT_UNSPECIFIED;
2292	uint32_t i;
2293
2294	int max_commits = 0;
2295	int size_mult = 2;
2296
2297	if (ctx->opts) {
2298	max_commits = ctx->opts->max_commits;
2299
2300	if (ctx->opts->size_multiple)
2301	size_mult = ctx->opts->size_multiple;
2302
2303	flags = ctx->opts->split_flags;
2304	}
2305
2306	g = graph_to_merge;
2307	num_commits = ctx->commits.nr;
2308	if (flags == COMMIT_GRAPH_SPLIT_REPLACE)
2309	ctx->num_commit_graphs_after = 1;
2310	else
2311	ctx->num_commit_graphs_after = ctx->num_commit_graphs_before + 1;
2312
2313	if (flags != COMMIT_GRAPH_SPLIT_MERGE_PROHIBITED &&
2314	flags != COMMIT_GRAPH_SPLIT_REPLACE) {
2315	while (g && (g->num_commits <= st_mult(size_mult, num_commits) \|\|
2316	(max_commits && num_commits > max_commits))) {
2317	if (g->odb_source != ctx->odb_source)
2318	break;
2319
2320	if (unsigned_add_overflows(num_commits, g->num_commits))
2321	die(_("cannot merge graphs with %"PRIuMAX", "
2322	"%"PRIuMAX" commits"),
2323	(uintmax_t)num_commits,
2324	(uintmax_t)g->num_commits);
2325	num_commits += g->num_commits;
2326	g = g->base_graph;
2327
2328	ctx->num_commit_graphs_after--;
2329	}
2330	}
2331
2332	if (flags != COMMIT_GRAPH_SPLIT_REPLACE)
2333	ctx->new_base_graph = g;
2334	else if (ctx->num_commit_graphs_after != 1)
2335	BUG("split_graph_merge_strategy: num_commit_graphs_after "
2336	"should be 1 with --split=replace");
2337
2338	if (ctx->num_commit_graphs_after == 2) {
2339	char *old_graph_name = get_commit_graph_filename(g->odb_source);
2340
2341	if (!strcmp(g->filename, old_graph_name) &&
2342	g->odb_source != ctx->odb_source) {
2343	ctx->num_commit_graphs_after = 1;
2344	ctx->new_base_graph = NULL;
2345	}
2346
2347	free(old_graph_name);
2348	}
2349
2350	CALLOC_ARRAY(ctx->commit_graph_filenames_after, ctx->num_commit_graphs_after);
2351	CALLOC_ARRAY(ctx->commit_graph_hash_after, ctx->num_commit_graphs_after);
2352
2353	for (i = 0; i < ctx->num_commit_graphs_after &&
2354	i < ctx->num_commit_graphs_before; i++)
2355	ctx->commit_graph_filenames_after[i] = xstrdup(ctx->commit_graph_filenames_before[i]);
2356
2357	i = ctx->num_commit_graphs_before - 1;
2358	g = graph_to_merge;
2359
2360	while (g) {
2361	if (i < ctx->num_commit_graphs_after)
2362	ctx->commit_graph_hash_after[i] = xstrdup(oid_to_hex(&g->oid));
2363
2364	/*
2365	* If the topmost remaining layer has generation data chunk, the
2366	* resultant layer also has generation data chunk.
2367	*/
2368	if (i == ctx->num_commit_graphs_after - 2)
2369	ctx->write_generation_data = !!g->chunk_generation_data;
2370
2371	i--;
2372	g = g->base_graph;
2373	}
2374	}
2375
2376	static void merge_commit_graph(struct write_commit_graph_context *ctx,
2377	struct commit_graph *g)
2378	{
2379	uint32_t i;
2380	uint32_t offset = g->num_commits_in_base;
2381
2382	if (unsigned_add_overflows(ctx->commits.nr, g->num_commits))
2383	die(_("cannot merge graph %s, too many commits: %"PRIuMAX),
2384	oid_to_hex(&g->oid),
2385	(uintmax_t)st_add(ctx->commits.nr, g->num_commits));
2386
2387	commit_stack_grow(&ctx->commits, g->num_commits);
2388
2389	for (i = 0; i < g->num_commits; i++) {
2390	struct object_id oid;
2391	struct commit *result;
2392
2393	display_progress(ctx->progress, i + 1);
2394
2395	load_oid_from_graph(g, i + offset, &oid);
2396
2397	/* only add commits if they still exist in the repo */
2398	result = lookup_commit_reference_gently(ctx->r, &oid, 1);
2399
2400	if (result)
2401	commit_stack_push(&ctx->commits, result);
2402	}
2403	}
2404
2405	static int commit_compare(const void _a, const void _b)
2406	{
2407	const struct commit a = (const struct commit **)_a;
2408	const struct commit b = (const struct commit **)_b;
2409	return oidcmp(&a->object.oid, &b->object.oid);
2410	}
2411
2412	static void sort_and_scan_merged_commits(struct write_commit_graph_context *ctx)
2413	{
2414	uint32_t i, dedup_i = 0;
2415
2416	if (ctx->report_progress)
2417	ctx->progress = start_delayed_progress(
2418	ctx->r,
2419	_("Scanning merged commits"),
2420	ctx->commits.nr);
2421
2422	QSORT(ctx->commits.items, ctx->commits.nr, commit_compare);
2423
2424	ctx->num_extra_edges = 0;
2425	for (i = 0; i < ctx->commits.nr; i++) {
2426	display_progress(ctx->progress, i + 1);
2427
2428	if (i && oideq(&ctx->commits.items[i - 1]->object.oid,
2429	&ctx->commits.items[i]->object.oid)) {
2430	/*
2431	* Silently ignore duplicates. These were likely
2432	* created due to a commit appearing in multiple
2433	* layers of the chain, which is unexpected but
2434	* not invalid. We should make sure there is a
2435	* unique copy in the new layer.
2436	*/
2437	} else {
2438	unsigned int num_parents;
2439
2440	ctx->commits.items[dedup_i] = ctx->commits.items[i];
2441	dedup_i++;
2442
2443	num_parents = commit_list_count(ctx->commits.items[i]->parents);
2444	if (num_parents > 2)
2445	ctx->num_extra_edges += num_parents - 1;
2446	}
2447	}
2448
2449	ctx->commits.nr = dedup_i;
2450
2451	stop_progress(&ctx->progress);
2452	}
2453
2454	static void merge_commit_graphs(struct write_commit_graph_context *ctx,
2455	struct commit_graph *g)
2456	{
2457	uint32_t current_graph_number = ctx->num_commit_graphs_before;
2458
2459	while (g && current_graph_number >= ctx->num_commit_graphs_after) {
2460	current_graph_number--;
2461
2462	if (ctx->report_progress)
2463	ctx->progress = start_delayed_progress(ctx->r,
2464	_("Merging commit-graph"), 0);
2465
2466	merge_commit_graph(ctx, g);
2467	stop_progress(&ctx->progress);
2468
2469	g = g->base_graph;
2470	}
2471
2472	if (g) {
2473	ctx->new_base_graph = g;
2474	ctx->new_num_commits_in_base = g->num_commits + g->num_commits_in_base;
2475	}
2476
2477	if (ctx->new_base_graph)
2478	ctx->base_graph_name = xstrdup(ctx->new_base_graph->filename);
2479
2480	sort_and_scan_merged_commits(ctx);
2481	}
2482
2483	static void mark_commit_graphs(struct write_commit_graph_context *ctx)
2484	{
2485	uint32_t i;
2486	time_t now = time(NULL);
2487
2488	for (i = ctx->num_commit_graphs_after - 1; i < ctx->num_commit_graphs_before; i++) {
2489	struct stat st;
2490	struct utimbuf updated_time;
2491
2492	if (stat(ctx->commit_graph_filenames_before[i], &st) < 0)
2493	continue;
2494
2495	updated_time.actime = st.st_atime;
2496	updated_time.modtime = now;
2497	utime(ctx->commit_graph_filenames_before[i], &updated_time);
2498	}
2499	}
2500
2501	static void expire_commit_graphs(struct write_commit_graph_context *ctx)
2502	{
2503	struct strbuf path = STRBUF_INIT;
2504	DIR *dir;
2505	struct dirent *de;
2506	size_t dirnamelen;
2507	timestamp_t expire_time = time(NULL);
2508
2509	if (ctx->opts && ctx->opts->expire_time)
2510	expire_time = ctx->opts->expire_time;
2511	if (!ctx->split) {
2512	char *chain_file_name = get_commit_graph_chain_filename(ctx->odb_source);
2513	unlink(chain_file_name);
2514	free(chain_file_name);
2515	ctx->num_commit_graphs_after = 0;
2516	}
2517
2518	strbuf_addstr(&path, ctx->odb_source->path);
2519	strbuf_addstr(&path, "/info/commit-graphs");
2520	dir = opendir(path.buf);
2521
2522	if (!dir)
2523	goto out;
2524
2525	strbuf_addch(&path, '/');
2526	dirnamelen = path.len;
2527	while ((de = readdir(dir)) != NULL) {
2528	struct stat st;
2529	uint32_t i, found = 0;
2530
2531	strbuf_setlen(&path, dirnamelen);
2532	strbuf_addstr(&path, de->d_name);
2533
2534	if (stat(path.buf, &st) < 0)
2535	continue;
2536
2537	if (st.st_mtime > expire_time)
2538	continue;
2539	if (path.len < 6 \|\| strcmp(path.buf + path.len - 6, ".graph"))
2540	continue;
2541
2542	for (i = 0; i < ctx->num_commit_graphs_after; i++) {
2543	if (!strcmp(ctx->commit_graph_filenames_after[i],
2544	path.buf)) {
2545	found = 1;
2546	break;
2547	}
2548	}
2549
2550	if (!found)
2551	unlink(path.buf);
2552	}
2553
2554	out:
2555	if(dir)
2556	closedir(dir);
2557	strbuf_release(&path);
2558	}
2559
2560	int write_commit_graph(struct odb_source *source,
2561	const struct string_list *const pack_indexes,
2562	struct oidset *commits,
2563	enum commit_graph_write_flags flags,
2564	const struct commit_graph_opts *opts)
2565	{
2566	struct repository *r = source->odb->repo;
2567	struct write_commit_graph_context ctx = {
2568	.r = r,
2569	.odb_source = source,
2570	.append = flags & COMMIT_GRAPH_WRITE_APPEND ? 1 : 0,
2571	.report_progress = flags & COMMIT_GRAPH_WRITE_PROGRESS ? 1 : 0,
2572	.split = flags & COMMIT_GRAPH_WRITE_SPLIT ? 1 : 0,
2573	.opts = opts,
2574	.total_bloom_filter_data_size = 0,
2575	.write_generation_data = (get_configured_generation_version(r) == 2),
2576	.num_generation_data_overflows = 0,
2577	};
2578	uint32_t i;
2579	int res = 0;
2580	int replace = 0;
2581	struct bloom_filter_settings bloom_settings = DEFAULT_BLOOM_FILTER_SETTINGS;
2582	struct topo_level_slab topo_levels;
2583	struct commit_graph *g;
2584
2585	prepare_repo_settings(r);
2586	if (!r->settings.core_commit_graph) {
2587	warning(_("attempting to write a commit-graph, but 'core.commitGraph' is disabled"));
2588	return 0;
2589	}
2590	if (!commit_graph_compatible(r))
2591	return 0;
2592	if (r->settings.commit_graph_changed_paths_version < -1
2593	\|\| r->settings.commit_graph_changed_paths_version > 2) {
2594	warning(_("attempting to write a commit-graph, but "
2595	"'commitGraph.changedPathsVersion' (%d) is not supported"),
2596	r->settings.commit_graph_changed_paths_version);
2597	return 0;
2598	}
2599
2600	bloom_settings.hash_version = r->settings.commit_graph_changed_paths_version;
2601	bloom_settings.bits_per_entry = git_env_ulong("GIT_TEST_BLOOM_SETTINGS_BITS_PER_ENTRY",
2602	bloom_settings.bits_per_entry);
2603	bloom_settings.num_hashes = git_env_ulong("GIT_TEST_BLOOM_SETTINGS_NUM_HASHES",
2604	bloom_settings.num_hashes);
2605	bloom_settings.max_changed_paths = git_env_ulong("GIT_TEST_BLOOM_SETTINGS_MAX_CHANGED_PATHS",
2606	bloom_settings.max_changed_paths);
2607	ctx.bloom_settings = &bloom_settings;
2608
2609	init_topo_level_slab(&topo_levels);
2610	ctx.topo_levels = &topo_levels;
2611
2612	g = prepare_commit_graph(ctx.r);
2613	for (struct commit_graph *chain = g; chain; chain = chain->base_graph)
2614	chain->topo_levels = &topo_levels;
2615
2616	if (flags & COMMIT_GRAPH_WRITE_BLOOM_FILTERS)
2617	ctx.changed_paths = 1;
2618	if (!(flags & COMMIT_GRAPH_NO_WRITE_BLOOM_FILTERS)) {
2619	/* We have changed-paths already. Keep them in the next graph */
2620	if (g && g->bloom_filter_settings) {
2621	ctx.changed_paths = 1;
2622
2623	/* don't propagate the hash_version unless unspecified */
2624	if (bloom_settings.hash_version == -1)
2625	bloom_settings.hash_version = g->bloom_filter_settings->hash_version;
2626	bloom_settings.bits_per_entry = g->bloom_filter_settings->bits_per_entry;
2627	bloom_settings.num_hashes = g->bloom_filter_settings->num_hashes;
2628	bloom_settings.max_changed_paths = g->bloom_filter_settings->max_changed_paths;
2629	}
2630	}
2631
2632	bloom_settings.hash_version = bloom_settings.hash_version == 2 ? 2 : 1;
2633
2634	if (ctx.split) {
2635	for (struct commit_graph *chain = g; chain; chain = chain->base_graph)
2636	ctx.num_commit_graphs_before++;
2637
2638	if (ctx.num_commit_graphs_before) {
2639	ALLOC_ARRAY(ctx.commit_graph_filenames_before, ctx.num_commit_graphs_before);
2640	i = ctx.num_commit_graphs_before;
2641
2642	for (struct commit_graph *chain = g; chain; chain = chain->base_graph)
2643	ctx.commit_graph_filenames_before[--i] = xstrdup(chain->filename);
2644	}
2645
2646	if (ctx.opts)
2647	replace = ctx.opts->split_flags & COMMIT_GRAPH_SPLIT_REPLACE;
2648	}
2649
2650	if (odb_count_objects(r->objects, ODB_COUNT_OBJECTS_APPROXIMATE, &ctx.approx_nr_objects) < 0)
2651	ctx.approx_nr_objects = 0;
2652
2653	if (ctx.append && g) {
2654	for (i = 0; i < g->num_commits; i++) {
2655	struct object_id oid;
2656	oidread(&oid, g->chunk_oid_lookup + st_mult(g->hash_algo->rawsz, i),
2657	r->hash_algo);
2658	oid_array_append(&ctx.oids, &oid);
2659	}
2660	}
2661
2662	if (pack_indexes) {
2663	ctx.order_by_pack = 1;
2664	if ((res = fill_oids_from_packs(&ctx, pack_indexes)))
2665	goto cleanup;
2666	}
2667
2668	if (commits) {
2669	if ((res = fill_oids_from_commits(&ctx, commits)))
2670	goto cleanup;
2671	}
2672
2673	if (!pack_indexes && !commits) {
2674	ctx.order_by_pack = 1;
2675	fill_oids_from_all_packs(&ctx);
2676	}
2677
2678	close_reachable(&ctx);
2679
2680	copy_oids_to_commits(&ctx);
2681
2682	if (ctx.commits.nr >= GRAPH_EDGE_LAST_MASK) {
2683	error(_("too many commits to write graph"));
2684	res = -1;
2685	goto cleanup;
2686	}
2687
2688	if (!ctx.commits.nr && !replace)
2689	goto cleanup;
2690
2691	if (ctx.split) {
2692	split_graph_merge_strategy(&ctx, g);
2693
2694	if (!replace)
2695	merge_commit_graphs(&ctx, g);
2696	} else {
2697	ctx.num_commit_graphs_after = 1;
2698	}
2699
2700	ctx.trust_generation_numbers = validate_mixed_generation_chain(g);
2701
2702	compute_topological_levels(&ctx);
2703	if (ctx.write_generation_data)
2704	compute_generation_numbers(&ctx);
2705
2706	if (ctx.changed_paths)
2707	compute_bloom_filters(&ctx);
2708
2709	res = write_commit_graph_file(&ctx);
2710
2711	if (ctx.changed_paths)
2712	deinit_bloom_filters();
2713
2714	if (ctx.split)
2715	mark_commit_graphs(&ctx);
2716
2717	expire_commit_graphs(&ctx);
2718
2719	cleanup:
2720	free(ctx.graph_name);
2721	free(ctx.base_graph_name);
2722	commit_stack_clear(&ctx.commits);
2723	oid_array_clear(&ctx.oids);
2724	clear_topo_level_slab(&topo_levels);
2725
2726	if (ctx.r->objects->commit_graph) {
2727	struct commit_graph *g = ctx.r->objects->commit_graph;
2728
2729	while (g) {
2730	g->topo_levels = NULL;
2731	g = g->base_graph;
2732	}
2733	}
2734
2735	for (i = 0; i < ctx.num_commit_graphs_before; i++)
2736	free(ctx.commit_graph_filenames_before[i]);
2737	free(ctx.commit_graph_filenames_before);
2738
2739	for (i = 0; i < ctx.num_commit_graphs_after; i++) {
2740	free(ctx.commit_graph_filenames_after[i]);
2741	free(ctx.commit_graph_hash_after[i]);
2742	}
2743	free(ctx.commit_graph_filenames_after);
2744	free(ctx.commit_graph_hash_after);
2745
2746	return res;
2747	}
2748
2749	#define VERIFY_COMMIT_GRAPH_ERROR_HASH 2
2750	static int verify_commit_graph_error;
2751
2752	__attribute__((format (printf, 1, 2)))
2753	static void graph_report(const char *fmt, ...)
2754	{
2755	va_list ap;
2756
2757	verify_commit_graph_error = 1;
2758	va_start(ap, fmt);
2759	vfprintf(stderr, fmt, ap);
2760	fprintf(stderr, "\n");
2761	va_end(ap);
2762	}
2763
2764	static int commit_graph_checksum_valid(struct commit_graph *g)
2765	{
2766	return hashfile_checksum_valid(g->hash_algo,
2767	g->data, g->data_len);
2768	}
2769
2770	static int verify_one_commit_graph(struct commit_graph *g,
2771	struct progress *progress,
2772	uint64_t *seen)
2773	{
2774	struct repository *r = g->odb_source->odb->repo;
2775	uint32_t i, cur_fanout_pos = 0;
2776	struct object_id prev_oid, cur_oid;
2777	struct commit *seen_gen_zero = NULL;
2778	struct commit *seen_gen_non_zero = NULL;
2779
2780	if (!commit_graph_checksum_valid(g)) {
2781	graph_report(_("the commit-graph file has incorrect checksum and is likely corrupt"));
2782	verify_commit_graph_error = VERIFY_COMMIT_GRAPH_ERROR_HASH;
2783	}
2784
2785	for (i = 0; i < g->num_commits; i++) {
2786	struct commit *graph_commit;
2787
2788	oidread(&cur_oid, g->chunk_oid_lookup + st_mult(g->hash_algo->rawsz, i),
2789	g->hash_algo);
2790
2791	if (i && oidcmp(&prev_oid, &cur_oid) >= 0)
2792	graph_report(_("commit-graph has incorrect OID order: %s then %s"),
2793	oid_to_hex(&prev_oid),
2794	oid_to_hex(&cur_oid));
2795
2796	oidcpy(&prev_oid, &cur_oid);
2797
2798	while (cur_oid.hash[0] > cur_fanout_pos) {
2799	uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
2800
2801	if (i != fanout_value)
2802	graph_report(_("commit-graph has incorrect fanout value: fanout[%d] = %u != %u"),
2803	cur_fanout_pos, fanout_value, i);
2804	cur_fanout_pos++;
2805	}
2806
2807	graph_commit = lookup_commit(r, &cur_oid);
2808	if (!parse_commit_in_graph_one(g, graph_commit))
2809	graph_report(_("failed to parse commit %s from commit-graph"),
2810	oid_to_hex(&cur_oid));
2811	}
2812
2813	while (cur_fanout_pos < 256) {
2814	uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
2815
2816	if (g->num_commits != fanout_value)
2817	graph_report(_("commit-graph has incorrect fanout value: fanout[%d] = %u != %u"),
2818	cur_fanout_pos, fanout_value, i);
2819
2820	cur_fanout_pos++;
2821	}
2822
2823	if (verify_commit_graph_error & ~VERIFY_COMMIT_GRAPH_ERROR_HASH)
2824	return verify_commit_graph_error;
2825
2826	for (i = 0; i < g->num_commits; i++) {
2827	struct commit graph_commit, odb_commit;
2828	struct commit_list graph_parents, odb_parents;
2829	timestamp_t max_generation = 0;
2830	timestamp_t generation;
2831
2832	display_progress(progress, ++(*seen));
2833	oidread(&cur_oid, g->chunk_oid_lookup + st_mult(g->hash_algo->rawsz, i),
2834	g->hash_algo);
2835
2836	graph_commit = lookup_commit(r, &cur_oid);
2837	odb_commit = (struct commit *)create_object(r, &cur_oid, alloc_commit_node(r));
2838	if (repo_parse_commit_internal(r, odb_commit, 0, 0)) {
2839	graph_report(_("failed to parse commit %s from object database for commit-graph"),
2840	oid_to_hex(&cur_oid));
2841	continue;
2842	}
2843
2844	if (!oideq(&get_commit_tree_in_graph_one(g, graph_commit)->object.oid,
2845	get_commit_tree_oid(odb_commit)))
2846	graph_report(_("root tree OID for commit %s in commit-graph is %s != %s"),
2847	oid_to_hex(&cur_oid),
2848	oid_to_hex(get_commit_tree_oid(graph_commit)),
2849	oid_to_hex(get_commit_tree_oid(odb_commit)));
2850
2851	graph_parents = graph_commit->parents;
2852	odb_parents = odb_commit->parents;
2853
2854	while (graph_parents) {
2855	if (!odb_parents) {
2856	graph_report(_("commit-graph parent list for commit %s is too long"),
2857	oid_to_hex(&cur_oid));
2858	break;
2859	}
2860
2861	/* parse parent in case it is in a base graph */
2862	parse_commit_in_graph_one(g, graph_parents->item);
2863
2864	if (!oideq(&graph_parents->item->object.oid, &odb_parents->item->object.oid))
2865	graph_report(_("commit-graph parent for %s is %s != %s"),
2866	oid_to_hex(&cur_oid),
2867	oid_to_hex(&graph_parents->item->object.oid),
2868	oid_to_hex(&odb_parents->item->object.oid));
2869
2870	generation = commit_graph_generation_from_graph(graph_parents->item);
2871	if (generation > max_generation)
2872	max_generation = generation;
2873
2874	graph_parents = graph_parents->next;
2875	odb_parents = odb_parents->next;
2876	}
2877
2878	if (odb_parents)
2879	graph_report(_("commit-graph parent list for commit %s terminates early"),
2880	oid_to_hex(&cur_oid));
2881
2882	if (commit_graph_generation_from_graph(graph_commit))
2883	seen_gen_non_zero = graph_commit;
2884	else
2885	seen_gen_zero = graph_commit;
2886
2887	if (seen_gen_zero)
2888	continue;
2889
2890	/*
2891	* If we are using topological level and one of our parents has
2892	* generation GENERATION_NUMBER_V1_MAX, then our generation is
2893	* also GENERATION_NUMBER_V1_MAX. Decrement to avoid extra logic
2894	* in the following condition.
2895	*/
2896	if (!g->read_generation_data && max_generation == GENERATION_NUMBER_V1_MAX)
2897	max_generation--;
2898
2899	generation = commit_graph_generation(graph_commit);
2900	if (generation < max_generation + 1)
2901	graph_report(_("commit-graph generation for commit %s is %"PRItime" < %"PRItime),
2902	oid_to_hex(&cur_oid),
2903	generation,
2904	max_generation + 1);
2905
2906	if (graph_commit->date != odb_commit->date)
2907	graph_report(_("commit date for commit %s in commit-graph is %"PRItime" != %"PRItime),
2908	oid_to_hex(&cur_oid),
2909	graph_commit->date,
2910	odb_commit->date);
2911	}
2912
2913	if (seen_gen_zero && seen_gen_non_zero)
2914	graph_report(_("commit-graph has both zero and non-zero "
2915	"generations (e.g., commits '%s' and '%s')"),
2916	oid_to_hex(&seen_gen_zero->object.oid),
2917	oid_to_hex(&seen_gen_non_zero->object.oid));
2918
2919	return verify_commit_graph_error;
2920	}
2921
2922	int verify_commit_graph(struct commit_graph *g, int flags)
2923	{
2924	struct progress *progress = NULL;
2925	int local_error = 0;
2926	uint64_t seen = 0;
2927
2928	if (!g) {
2929	graph_report("no commit-graph file loaded");
2930	return 1;
2931	}
2932
2933	if (flags & COMMIT_GRAPH_WRITE_PROGRESS) {
2934	uint64_t total = g->num_commits;
2935	if (!(flags & COMMIT_GRAPH_VERIFY_SHALLOW))
2936	total += g->num_commits_in_base;
2937
2938	progress = start_progress(g->odb_source->odb->repo,
2939	_("Verifying commits in commit graph"),
2940	total);
2941	}
2942
2943	for (; g; g = g->base_graph) {
2944	local_error \|= verify_one_commit_graph(g, progress, &seen);
2945	if (flags & COMMIT_GRAPH_VERIFY_SHALLOW)
2946	break;
2947	}
2948
2949	stop_progress(&progress);
2950
2951	return local_error;
2952	}
2953
2954	void free_commit_graph(struct commit_graph *g)
2955	{
2956	while (g) {
2957	struct commit_graph *next = g->base_graph;
2958
2959	if (g->data)
2960	munmap((void *)g->data, g->data_len);
2961	free(g->filename);
2962	free(g->bloom_filter_settings);
2963	free(g);
2964
2965	g = next;
2966	}
2967	}
2968
2969	void disable_commit_graph(struct repository *r)
2970	{
2971	r->commit_graph_disabled = 1;
2972	}