fast-import.c · matheusbsbs/matheus-bagliano

jch

bisect jch maint master next seen test todo

c 4,064 lines 107 KB

Raw

1	#define USE_THE_REPOSITORY_VARIABLE
2	#define DISABLE_SIGN_COMPARE_WARNINGS
3
4	#include "builtin.h"
5	#include "abspath.h"
6	#include "environment.h"
7	#include "gettext.h"
8	#include "hex.h"
9	#include "config.h"
10	#include "lockfile.h"
11	#include "object.h"
12	#include "blob.h"
13	#include "tree.h"
14	#include "commit.h"
15	#include "delta.h"
16	#include "pack.h"
17	#include "path.h"
18	#include "read-cache-ll.h"
19	#include "refs.h"
20	#include "csum-file.h"
21	#include "quote.h"
22	#include "dir.h"
23	#include "run-command.h"
24	#include "packfile.h"
25	#include "object-file.h"
26	#include "object-name.h"
27	#include "odb.h"
28	#include "mem-pool.h"
29	#include "commit-reach.h"
30	#include "khash.h"
31	#include "date.h"
32	#include "gpg-interface.h"
33
34	#define PACK_ID_BITS 16
35	#define MAX_PACK_ID ((1<<PACK_ID_BITS)-1)
36	#define DEPTH_BITS 13
37	#define MAX_DEPTH ((1<<DEPTH_BITS)-1)
38
39	/*
40	* We abuse the setuid bit on directories to mean "do not delta".
41	*/
42	#define NO_DELTA S_ISUID
43
44	/*
45	* The amount of additional space required in order to write an object into the
46	* current pack. This is the hash lengths at the end of the pack, plus the
47	* length of one object ID.
48	*/
49	#define PACK_SIZE_THRESHOLD (the_hash_algo->rawsz * 3)
50
51	struct object_entry {
52	struct pack_idx_entry idx;
53	struct hashmap_entry ent;
54	uint32_t type : TYPE_BITS,
55	pack_id : PACK_ID_BITS,
56	depth : DEPTH_BITS;
57	};
58
59	static int object_entry_hashcmp(const void *map_data UNUSED,
60	const struct hashmap_entry *eptr,
61	const struct hashmap_entry *entry_or_key,
62	const void *keydata)
63	{
64	const struct object_id *oid = keydata;
65	const struct object_entry e1, e2;
66
67	e1 = container_of(eptr, const struct object_entry, ent);
68	if (oid)
69	return oidcmp(&e1->idx.oid, oid);
70
71	e2 = container_of(entry_or_key, const struct object_entry, ent);
72	return oidcmp(&e1->idx.oid, &e2->idx.oid);
73	}
74
75	struct object_entry_pool {
76	struct object_entry_pool *next_pool;
77	struct object_entry *next_free;
78	struct object_entry *end;
79	struct object_entry entries[FLEX_ARRAY]; /* more */
80	};
81
82	struct mark_set {
83	union {
84	struct object_id *oids[1024];
85	struct object_entry *marked[1024];
86	struct mark_set *sets[1024];
87	} data;
88	unsigned int shift;
89	};
90
91	struct last_object {
92	struct strbuf data;
93	off_t offset;
94	unsigned int depth;
95	unsigned no_swap : 1;
96	};
97
98	struct atom_str {
99	struct atom_str *next_atom;
100	unsigned short str_len;
101	char str_dat[FLEX_ARRAY]; /* more */
102	};
103
104	struct tree_content;
105	struct tree_entry {
106	struct tree_content *tree;
107	struct atom_str *name;
108	struct tree_entry_ms {
109	uint16_t mode;
110	struct object_id oid;
111	} versions[2];
112	};
113
114	struct tree_content {
115	unsigned int entry_capacity; /* must match avail_tree_content */
116	unsigned int entry_count;
117	unsigned int delta_depth;
118	struct tree_entry entries[FLEX_ARRAY]; / more */
119	};
120
121	struct avail_tree_content {
122	unsigned int entry_capacity; /* must match tree_content */
123	struct avail_tree_content *next_avail;
124	};
125
126	struct branch {
127	struct branch *table_next_branch;
128	struct branch *active_next_branch;
129	const char *name;
130	struct tree_entry branch_tree;
131	uintmax_t last_commit;
132	uintmax_t num_notes;
133	unsigned active : 1;
134	unsigned delete : 1;
135	unsigned pack_id : PACK_ID_BITS;
136	struct object_id oid;
137	};
138
139	struct tag {
140	struct tag *next_tag;
141	const char *name;
142	unsigned int pack_id;
143	struct object_id oid;
144	};
145
146	struct hash_list {
147	struct hash_list *next;
148	struct object_id oid;
149	};
150
151	typedef enum {
152	WHENSPEC_RAW = 1,
153	WHENSPEC_RAW_PERMISSIVE,
154	WHENSPEC_RFC2822,
155	WHENSPEC_NOW
156	} whenspec_type;
157
158	struct recent_command {
159	struct recent_command *prev;
160	struct recent_command *next;
161	char *buf;
162	};
163
164	typedef void (mark_set_inserter_t)(struct mark_set s, struct object_id oid, uintmax_t mark);
165	typedef void (each_mark_fn_t)(uintmax_t mark, void obj, void *cbp);
166
167	/* Configured limits on output */
168	static unsigned long max_depth = 50;
169	static off_t max_packsize;
170	static int unpack_limit = 100;
171	static int force_update;
172
173	/* Stats and misc. counters */
174	static uintmax_t alloc_count;
175	static uintmax_t marks_set_count;
176	static uintmax_t object_count_by_type[1 << TYPE_BITS];
177	static uintmax_t duplicate_count_by_type[1 << TYPE_BITS];
178	static uintmax_t delta_count_by_type[1 << TYPE_BITS];
179	static uintmax_t delta_count_attempts_by_type[1 << TYPE_BITS];
180	static unsigned long object_count;
181	static unsigned long branch_count;
182	static unsigned long branch_load_count;
183	static int failure;
184	static FILE *pack_edges;
185	static unsigned int show_stats = 1;
186	static unsigned int quiet;
187	static int global_argc;
188	static const char **global_argv;
189	static const char *global_prefix;
190
191	static enum sign_mode signed_tag_mode = SIGN_VERBATIM;
192	static enum sign_mode signed_commit_mode = SIGN_VERBATIM;
193	static const char *signed_commit_keyid;
194	static const char *signed_tag_keyid;
195
196	/* Memory pools */
197	static struct mem_pool fi_mem_pool = {
198	.block_alloc = 210241024 - sizeof(struct mp_block),
199	};
200
201	/* Atom management */
202	static unsigned int atom_table_sz = 4451;
203	static unsigned int atom_cnt;
204	static struct atom_str **atom_table;
205
206	/* The .pack file being generated */
207	static struct pack_idx_option pack_idx_opts;
208	static unsigned int pack_id;
209	static struct hashfile *pack_file;
210	static struct packed_git *pack_data;
211	static struct packed_git **all_packs;
212	static off_t pack_size;
213
214	/* Table of objects we've written. */
215	static unsigned int object_entry_alloc = 5000;
216	static struct object_entry_pool *blocks;
217	static struct hashmap object_table;
218	static struct mark_set *marks;
219	static char *export_marks_file;
220	static char *import_marks_file;
221	static int import_marks_file_from_stream;
222	static int import_marks_file_ignore_missing;
223	static int import_marks_file_done;
224	static int relative_marks_paths;
225
226	/* Our last blob */
227	static struct last_object last_blob = {
228	.data = STRBUF_INIT,
229	};
230
231	/* Tree management */
232	static unsigned int tree_entry_alloc = 1000;
233	static void *avail_tree_entry;
234	static unsigned int avail_tree_table_sz = 100;
235	static struct avail_tree_content **avail_tree_table;
236	static size_t tree_entry_allocd;
237	static struct strbuf old_tree = STRBUF_INIT;
238	static struct strbuf new_tree = STRBUF_INIT;
239
240	/* Branch data */
241	static unsigned long max_active_branches = 5;
242	static unsigned long cur_active_branches;
243	static unsigned long branch_table_sz = 1039;
244	static struct branch **branch_table;
245	static struct branch *active_branches;
246
247	/* Tag data */
248	static struct tag *first_tag;
249	static struct tag *last_tag;
250
251	/* Input stream parsing */
252	static whenspec_type whenspec = WHENSPEC_RAW;
253	static struct strbuf command_buf = STRBUF_INIT;
254	static int unread_command_buf;
255	static struct recent_command cmd_hist = {
256	.prev = &cmd_hist,
257	.next = &cmd_hist,
258	};
259	static struct recent_command *cmd_tail = &cmd_hist;
260	static struct recent_command *rc_free;
261	static unsigned int cmd_save = 100;
262	static uintmax_t next_mark;
263	static struct strbuf new_data = STRBUF_INIT;
264	static int seen_data_command;
265	static int require_explicit_termination;
266	static int allow_unsafe_features;
267
268	/* Signal handling */
269	static volatile sig_atomic_t checkpoint_requested;
270
271	/* Submodule marks */
272	static struct string_list sub_marks_from = STRING_LIST_INIT_DUP;
273	static struct string_list sub_marks_to = STRING_LIST_INIT_DUP;
274	static kh_oid_map_t *sub_oid_map;
275
276	/* Where to write output of cat-blob commands */
277	static int cat_blob_fd = STDOUT_FILENO;
278
279	static void parse_argv(void);
280	static void parse_get_mark(const char *p);
281	static void parse_cat_blob(const char *p);
282	static void parse_ls(const char p, struct branch b);
283
284	static void for_each_mark(struct mark_set m, uintmax_t base, each_mark_fn_t callback, void p)
285	{
286	uintmax_t k;
287	if (m->shift) {
288	for (k = 0; k < 1024; k++) {
289	if (m->data.sets[k])
290	for_each_mark(m->data.sets[k], base + (k << m->shift), callback, p);
291	}
292	} else {
293	for (k = 0; k < 1024; k++) {
294	if (m->data.marked[k])
295	callback(base + k, m->data.marked[k], p);
296	}
297	}
298	}
299
300	static void dump_marks_fn(uintmax_t mark, void object, void cbp) {
301	struct object_entry *e = object;
302	FILE *f = cbp;
303
304	fprintf(f, ":%" PRIuMAX " %s\n", mark, oid_to_hex(&e->idx.oid));
305	}
306
307	static void write_branch_report(FILE rpt, struct branch b)
308	{
309	fprintf(rpt, "%s:\n", b->name);
310
311	fprintf(rpt, " status :");
312	if (b->active)
313	fputs(" active", rpt);
314	if (b->branch_tree.tree)
315	fputs(" loaded", rpt);
316	if (is_null_oid(&b->branch_tree.versions[1].oid))
317	fputs(" dirty", rpt);
318	fputc('\n', rpt);
319
320	fprintf(rpt, " tip commit : %s\n", oid_to_hex(&b->oid));
321	fprintf(rpt, " old tree : %s\n",
322	oid_to_hex(&b->branch_tree.versions[0].oid));
323	fprintf(rpt, " cur tree : %s\n",
324	oid_to_hex(&b->branch_tree.versions[1].oid));
325	fprintf(rpt, " commit clock: %" PRIuMAX "\n", b->last_commit);
326
327	fputs(" last pack : ", rpt);
328	if (b->pack_id < MAX_PACK_ID)
329	fprintf(rpt, "%u", b->pack_id);
330	fputc('\n', rpt);
331
332	fputc('\n', rpt);
333	}
334
335	static void write_crash_report(const char *err)
336	{
337	char *loc = repo_git_path(the_repository, "fast_import_crash_%"PRIuMAX, (uintmax_t) getpid());
338	FILE *rpt = fopen(loc, "w");
339	struct branch *b;
340	unsigned long lu;
341	struct recent_command *rc;
342
343	if (!rpt) {
344	error_errno(_("can't write crash report %s"), loc);
345	free(loc);
346	return;
347	}
348
349	fprintf(stderr, _("fast-import: dumping crash report to %s\n"), loc);
350
351	fprintf(rpt, "fast-import crash report:\n");
352	fprintf(rpt, " fast-import process: %"PRIuMAX"\n", (uintmax_t) getpid());
353	fprintf(rpt, " parent process : %"PRIuMAX"\n", (uintmax_t) getppid());
354	fprintf(rpt, " at %s\n", show_date(time(NULL), 0, DATE_MODE(ISO8601)));
355	fputc('\n', rpt);
356
357	fputs("fatal: ", rpt);
358	fputs(err, rpt);
359	fputc('\n', rpt);
360
361	fputc('\n', rpt);
362	fputs("Most Recent Commands Before Crash\n", rpt);
363	fputs("---------------------------------\n", rpt);
364	for (rc = cmd_hist.next; rc != &cmd_hist; rc = rc->next) {
365	if (rc->next == &cmd_hist)
366	fputs("* ", rpt);
367	else
368	fputs(" ", rpt);
369	fputs(rc->buf, rpt);
370	fputc('\n', rpt);
371	}
372
373	fputc('\n', rpt);
374	fputs("Active Branch LRU\n", rpt);
375	fputs("-----------------\n", rpt);
376	fprintf(rpt, " active_branches = %lu cur, %lu max\n",
377	cur_active_branches,
378	max_active_branches);
379	fputc('\n', rpt);
380	fputs(" pos clock name\n", rpt);
381	fputs(" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n", rpt);
382	for (b = active_branches, lu = 0; b; b = b->active_next_branch)
383	fprintf(rpt, " %2lu) %6" PRIuMAX" %s\n",
384	++lu, b->last_commit, b->name);
385
386	fputc('\n', rpt);
387	fputs("Inactive Branches\n", rpt);
388	fputs("-----------------\n", rpt);
389	for (lu = 0; lu < branch_table_sz; lu++) {
390	for (b = branch_table[lu]; b; b = b->table_next_branch)
391	write_branch_report(rpt, b);
392	}
393
394	if (first_tag) {
395	struct tag *tg;
396	fputc('\n', rpt);
397	fputs("Annotated Tags\n", rpt);
398	fputs("--------------\n", rpt);
399	for (tg = first_tag; tg; tg = tg->next_tag) {
400	fputs(oid_to_hex(&tg->oid), rpt);
401	fputc(' ', rpt);
402	fputs(tg->name, rpt);
403	fputc('\n', rpt);
404	}
405	}
406
407	fputc('\n', rpt);
408	fputs("Marks\n", rpt);
409	fputs("-----\n", rpt);
410	if (export_marks_file)
411	fprintf(rpt, " exported to %s\n", export_marks_file);
412	else
413	for_each_mark(marks, 0, dump_marks_fn, rpt);
414
415	fputc('\n', rpt);
416	fputs("-------------------\n", rpt);
417	fputs("END OF CRASH REPORT\n", rpt);
418	fclose(rpt);
419	free(loc);
420	}
421
422	static void end_packfile(void);
423	static void unkeep_all_packs(void);
424	static void dump_marks(void);
425
426	static NORETURN void die_nicely(const char *err, va_list params)
427	{
428	va_list cp;
429	static int zombie;
430	report_fn die_message_fn = get_die_message_routine();
431
432	va_copy(cp, params);
433	die_message_fn(err, params);
434
435	if (!zombie) {
436	char message[2 * PATH_MAX];
437
438	zombie = 1;
439	vsnprintf(message, sizeof(message), err, cp);
440	write_crash_report(message);
441	end_packfile();
442	unkeep_all_packs();
443	dump_marks();
444	}
445	exit(128);
446	}
447
448	#ifndef SIGUSR1 /* Windows, for example */
449
450	static void set_checkpoint_signal(void)
451	{
452	}
453
454	#else
455
456	static void checkpoint_signal(int signo UNUSED)
457	{
458	checkpoint_requested = 1;
459	}
460
461	static void set_checkpoint_signal(void)
462	{
463	struct sigaction sa;
464
465	memset(&sa, 0, sizeof(sa));
466	sa.sa_handler = checkpoint_signal;
467	sigemptyset(&sa.sa_mask);
468	sa.sa_flags = SA_RESTART;
469	sigaction(SIGUSR1, &sa, NULL);
470	}
471
472	#endif
473
474	static void alloc_objects(unsigned int cnt)
475	{
476	struct object_entry_pool *b;
477
478	b = xmalloc(sizeof(struct object_entry_pool)
479	+ cnt * sizeof(struct object_entry));
480	b->next_pool = blocks;
481	b->next_free = b->entries;
482	b->end = b->entries + cnt;
483	blocks = b;
484	alloc_count += cnt;
485	}
486
487	static struct object_entry new_object(struct object_id oid)
488	{
489	struct object_entry *e;
490
491	if (blocks->next_free == blocks->end)
492	alloc_objects(object_entry_alloc);
493
494	e = blocks->next_free++;
495	oidcpy(&e->idx.oid, oid);
496	return e;
497	}
498
499	static struct object_entry find_object(struct object_id oid)
500	{
501	return hashmap_get_entry_from_hash(&object_table, oidhash(oid), oid,
502	struct object_entry, ent);
503	}
504
505	static struct object_entry insert_object(struct object_id oid)
506	{
507	struct object_entry *e;
508	unsigned int hash = oidhash(oid);
509
510	e = hashmap_get_entry_from_hash(&object_table, hash, oid,
511	struct object_entry, ent);
512	if (!e) {
513	e = new_object(oid);
514	e->idx.offset = 0;
515	hashmap_entry_init(&e->ent, hash);
516	hashmap_add(&object_table, &e->ent);
517	}
518
519	return e;
520	}
521
522	static void invalidate_pack_id(unsigned int id)
523	{
524	unsigned long lu;
525	struct tag *t;
526	struct hashmap_iter iter;
527	struct object_entry *e;
528
529	hashmap_for_each_entry(&object_table, &iter, e, ent) {
530	if (e->pack_id == id)
531	e->pack_id = MAX_PACK_ID;
532	}
533
534	for (lu = 0; lu < branch_table_sz; lu++) {
535	struct branch *b;
536
537	for (b = branch_table[lu]; b; b = b->table_next_branch)
538	if (b->pack_id == id)
539	b->pack_id = MAX_PACK_ID;
540	}
541
542	for (t = first_tag; t; t = t->next_tag)
543	if (t->pack_id == id)
544	t->pack_id = MAX_PACK_ID;
545	}
546
547	static unsigned int hc_str(const char *s, size_t len)
548	{
549	unsigned int r = 0;
550	while (len-- > 0)
551	r = r * 31 + *s++;
552	return r;
553	}
554
555	static void insert_mark(struct mark_set *top, uintmax_t idnum, struct object_entry oe)
556	{
557	struct mark_set s = top;
558
559	while ((idnum >> s->shift) >= 1024) {
560	s = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set));
561	s->shift = (*top)->shift + 10;
562	s->data.sets[0] = *top;
563	*top = s;
564	}
565	while (s->shift) {
566	uintmax_t i = idnum >> s->shift;
567	idnum -= i << s->shift;
568	if (!s->data.sets[i]) {
569	s->data.sets[i] = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set));
570	s->data.sets[i]->shift = s->shift - 10;
571	}
572	s = s->data.sets[i];
573	}
574	if (!s->data.marked[idnum])
575	marks_set_count++;
576	s->data.marked[idnum] = oe;
577	}
578
579	static void find_mark(struct mark_set s, uintmax_t idnum)
580	{
581	uintmax_t orig_idnum = idnum;
582	struct object_entry *oe = NULL;
583	if ((idnum >> s->shift) < 1024) {
584	while (s && s->shift) {
585	uintmax_t i = idnum >> s->shift;
586	idnum -= i << s->shift;
587	s = s->data.sets[i];
588	}
589	if (s)
590	oe = s->data.marked[idnum];
591	}
592	if (!oe)
593	die(_("mark :%" PRIuMAX " not declared"), orig_idnum);
594	return oe;
595	}
596
597	static struct atom_str to_atom(const char s, unsigned short len)
598	{
599	unsigned int hc = hc_str(s, len) % atom_table_sz;
600	struct atom_str *c;
601
602	for (c = atom_table[hc]; c; c = c->next_atom)
603	if (c->str_len == len && !strncmp(s, c->str_dat, len))
604	return c;
605
606	c = mem_pool_alloc(&fi_mem_pool, sizeof(struct atom_str) + len + 1);
607	c->str_len = len;
608	memcpy(c->str_dat, s, len);
609	c->str_dat[len] = 0;
610	c->next_atom = atom_table[hc];
611	atom_table[hc] = c;
612	atom_cnt++;
613	return c;
614	}
615
616	static struct branch lookup_branch(const char name)
617	{
618	unsigned int hc = hc_str(name, strlen(name)) % branch_table_sz;
619	struct branch *b;
620
621	for (b = branch_table[hc]; b; b = b->table_next_branch)
622	if (!strcmp(name, b->name))
623	return b;
624	return NULL;
625	}
626
627	static struct branch new_branch(const char name)
628	{
629	unsigned int hc = hc_str(name, strlen(name)) % branch_table_sz;
630	struct branch *b = lookup_branch(name);
631
632	if (b)
633	die(_("invalid attempt to create duplicate branch: %s"), name);
634	if (check_refname_format(name, REFNAME_ALLOW_ONELEVEL))
635	die(_("branch name doesn't conform to Git standards: %s"), name);
636
637	b = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct branch));
638	b->name = mem_pool_strdup(&fi_mem_pool, name);
639	b->table_next_branch = branch_table[hc];
640	b->branch_tree.versions[0].mode = S_IFDIR;
641	b->branch_tree.versions[1].mode = S_IFDIR;
642	b->num_notes = 0;
643	b->active = 0;
644	b->pack_id = MAX_PACK_ID;
645	branch_table[hc] = b;
646	branch_count++;
647	return b;
648	}
649
650	static unsigned int hc_entries(unsigned int cnt)
651	{
652	cnt = cnt & 7 ? (cnt / 8) + 1 : cnt / 8;
653	return cnt < avail_tree_table_sz ? cnt : avail_tree_table_sz - 1;
654	}
655
656	static struct tree_content *new_tree_content(unsigned int cnt)
657	{
658	struct avail_tree_content f, l = NULL;
659	struct tree_content *t;
660	unsigned int hc = hc_entries(cnt);
661
662	for (f = avail_tree_table[hc]; f; l = f, f = f->next_avail)
663	if (f->entry_capacity >= cnt)
664	break;
665
666	if (f) {
667	if (l)
668	l->next_avail = f->next_avail;
669	else
670	avail_tree_table[hc] = f->next_avail;
671	} else {
672	cnt = cnt & 7 ? ((cnt / 8) + 1) * 8 : cnt;
673	f = mem_pool_alloc(&fi_mem_pool, sizeof(t) + sizeof(t->entries[0]) cnt);
674	f->entry_capacity = cnt;
675	}
676
677	t = (struct tree_content*)f;
678	t->entry_count = 0;
679	t->delta_depth = 0;
680	return t;
681	}
682
683	static void release_tree_entry(struct tree_entry *e);
684	static void release_tree_content(struct tree_content *t)
685	{
686	struct avail_tree_content f = (struct avail_tree_content)t;
687	unsigned int hc = hc_entries(f->entry_capacity);
688	f->next_avail = avail_tree_table[hc];
689	avail_tree_table[hc] = f;
690	}
691
692	static void release_tree_content_recursive(struct tree_content *t)
693	{
694	unsigned int i;
695	for (i = 0; i < t->entry_count; i++)
696	release_tree_entry(t->entries[i]);
697	release_tree_content(t);
698	}
699
700	static struct tree_content *grow_tree_content(
701	struct tree_content *t,
702	int amt)
703	{
704	struct tree_content *r = new_tree_content(t->entry_count + amt);
705	r->entry_count = t->entry_count;
706	r->delta_depth = t->delta_depth;
707	COPY_ARRAY(r->entries, t->entries, t->entry_count);
708	release_tree_content(t);
709	return r;
710	}
711
712	static struct tree_entry *new_tree_entry(void)
713	{
714	struct tree_entry *e;
715
716	if (!avail_tree_entry) {
717	unsigned int n = tree_entry_alloc;
718	tree_entry_allocd += n * sizeof(struct tree_entry);
719	ALLOC_ARRAY(e, n);
720	avail_tree_entry = e;
721	while (n-- > 1) {
722	((void*)e) = e + 1;
723	e++;
724	}
725	((void*)e) = NULL;
726	}
727
728	e = avail_tree_entry;
729	avail_tree_entry = ((void*)e);
730	return e;
731	}
732
733	static void release_tree_entry(struct tree_entry *e)
734	{
735	if (e->tree)
736	release_tree_content_recursive(e->tree);
737	((void*)e) = avail_tree_entry;
738	avail_tree_entry = e;
739	}
740
741	static struct tree_content dup_tree_content(struct tree_content s)
742	{
743	struct tree_content *d;
744	struct tree_entry a, b;
745	unsigned int i;
746
747	if (!s)
748	return NULL;
749	d = new_tree_content(s->entry_count);
750	for (i = 0; i < s->entry_count; i++) {
751	a = s->entries[i];
752	b = new_tree_entry();
753	memcpy(b, a, sizeof(*a));
754	if (a->tree && is_null_oid(&b->versions[1].oid))
755	b->tree = dup_tree_content(a->tree);
756	else
757	b->tree = NULL;
758	d->entries[i] = b;
759	}
760	d->entry_count = s->entry_count;
761	d->delta_depth = s->delta_depth;
762
763	return d;
764	}
765
766	static void start_packfile(void)
767	{
768	struct strbuf tmp_file = STRBUF_INIT;
769	struct packed_git *p;
770	int pack_fd;
771
772	pack_fd = odb_mkstemp(the_repository->objects, &tmp_file,
773	"pack/tmp_pack_XXXXXX");
774	FLEX_ALLOC_STR(p, pack_name, tmp_file.buf);
775	strbuf_release(&tmp_file);
776
777	p->pack_fd = pack_fd;
778	p->do_not_close = 1;
779	p->repo = the_repository;
780	pack_file = hashfd(the_repository->hash_algo, pack_fd, p->pack_name);
781
782	pack_data = p;
783	pack_size = write_pack_header(pack_file, 0);
784	object_count = 0;
785
786	REALLOC_ARRAY(all_packs, pack_id + 1);
787	all_packs[pack_id] = p;
788	}
789
790	static const char *create_index(void)
791	{
792	const char *tmpfile;
793	struct pack_idx_entry idx, c, **last;
794	struct object_entry *e;
795	struct object_entry_pool *o;
796
797	/* Build the table of object IDs. */
798	ALLOC_ARRAY(idx, object_count);
799	c = idx;
800	for (o = blocks; o; o = o->next_pool)
801	for (e = o->next_free; e-- != o->entries;)
802	if (pack_id == e->pack_id)
803	*c++ = &e->idx;
804	last = idx + object_count;
805	if (c != last)
806	die(_("internal consistency error creating the index"));
807
808	tmpfile = write_idx_file(the_repository, NULL, idx, object_count,
809	&pack_idx_opts, pack_data->hash);
810	free(idx);
811	return tmpfile;
812	}
813
814	static char keep_pack(const char curr_index_name)
815	{
816	static const char *keep_msg = "fast-import";
817	struct strbuf name = STRBUF_INIT;
818	int keep_fd;
819
820	odb_pack_name(pack_data->repo, &name, pack_data->hash, "keep");
821	keep_fd = safe_create_file_with_leading_directories(pack_data->repo,
822	name.buf);
823	if (keep_fd < 0)
824	die_errno(_("cannot create keep file"));
825	write_or_die(keep_fd, keep_msg, strlen(keep_msg));
826	if (close(keep_fd))
827	die_errno(_("failed to write keep file"));
828
829	odb_pack_name(pack_data->repo, &name, pack_data->hash, "pack");
830	if (finalize_object_file(pack_data->repo, pack_data->pack_name, name.buf))
831	die(_("cannot store pack file"));
832
833	odb_pack_name(pack_data->repo, &name, pack_data->hash, "idx");
834	if (finalize_object_file(pack_data->repo, curr_index_name, name.buf))
835	die(_("cannot store index file"));
836	free((void *)curr_index_name);
837	return strbuf_detach(&name, NULL);
838	}
839
840	static void unkeep_all_packs(void)
841	{
842	struct strbuf name = STRBUF_INIT;
843	int k;
844
845	for (k = 0; k < pack_id; k++) {
846	struct packed_git *p = all_packs[k];
847	odb_pack_name(p->repo, &name, p->hash, "keep");
848	unlink_or_warn(name.buf);
849	}
850	strbuf_release(&name);
851	}
852
853	static int loosen_small_pack(const struct packed_git *p)
854	{
855	struct child_process unpack = CHILD_PROCESS_INIT;
856
857	if (lseek(p->pack_fd, 0, SEEK_SET) < 0)
858	die_errno(_("failed seeking to start of '%s'"), p->pack_name);
859
860	unpack.in = p->pack_fd;
861	unpack.git_cmd = 1;
862	unpack.stdout_to_stderr = 1;
863	strvec_push(&unpack.args, "unpack-objects");
864	if (!show_stats)
865	strvec_push(&unpack.args, "-q");
866
867	return run_command(&unpack);
868	}
869
870	static void end_packfile(void)
871	{
872	static int running;
873
874	if (running \|\| !pack_data)
875	return;
876
877	running = 1;
878	clear_delta_base_cache();
879	if (object_count) {
880	struct odb_source_files *files = odb_source_files_downcast(pack_data->repo->objects->sources);
881	struct packed_git *new_p;
882	struct object_id cur_pack_oid;
883	char *idx_name;
884	int i;
885	struct branch *b;
886	struct tag *t;
887
888	close_pack_windows(pack_data);
889	finalize_hashfile(pack_file, cur_pack_oid.hash, FSYNC_COMPONENT_PACK, 0);
890	fixup_pack_header_footer(the_hash_algo, pack_data->pack_fd,
891	pack_data->hash, pack_data->pack_name,
892	object_count, cur_pack_oid.hash,
893	pack_size);
894
895	if (object_count <= unpack_limit) {
896	if (!loosen_small_pack(pack_data)) {
897	invalidate_pack_id(pack_id);
898	goto discard_pack;
899	}
900	}
901
902	close(pack_data->pack_fd);
903	idx_name = keep_pack(create_index());
904
905	/* Register the packfile with core git's machinery. */
906	new_p = packfile_store_load_pack(files->packed, idx_name, 1);
907	if (!new_p)
908	die(_("core Git rejected index %s"), idx_name);
909	all_packs[pack_id] = new_p;
910	free(idx_name);
911
912	/* Print the boundary */
913	if (pack_edges) {
914	fprintf(pack_edges, "%s:", new_p->pack_name);
915	for (i = 0; i < branch_table_sz; i++) {
916	for (b = branch_table[i]; b; b = b->table_next_branch) {
917	if (b->pack_id == pack_id)
918	fprintf(pack_edges, " %s",
919	oid_to_hex(&b->oid));
920	}
921	}
922	for (t = first_tag; t; t = t->next_tag) {
923	if (t->pack_id == pack_id)
924	fprintf(pack_edges, " %s",
925	oid_to_hex(&t->oid));
926	}
927	fputc('\n', pack_edges);
928	fflush(pack_edges);
929	}
930
931	pack_id++;
932	}
933	else {
934	discard_pack:
935	close(pack_data->pack_fd);
936	unlink_or_warn(pack_data->pack_name);
937	}
938	FREE_AND_NULL(pack_data);
939	running = 0;
940
941	/* We can't carry a delta across packfiles. */
942	strbuf_release(&last_blob.data);
943	last_blob.offset = 0;
944	last_blob.depth = 0;
945	}
946
947	static void cycle_packfile(void)
948	{
949	end_packfile();
950	start_packfile();
951	}
952
953	static int store_object(
954	enum object_type type,
955	struct strbuf *dat,
956	struct last_object *last,
957	struct object_id *oidout,
958	uintmax_t mark)
959	{
960	struct odb_source *source;
961	void out, delta;
962	struct object_entry *e;
963	unsigned char hdr[96];
964	struct object_id oid;
965	unsigned long hdrlen, deltalen;
966	struct git_hash_ctx c;
967	git_zstream s;
968	struct repo_config_values *cfg = repo_config_values(the_repository);
969
970	hdrlen = format_object_header((char *)hdr, sizeof(hdr), type,
971	dat->len);
972	git_hash_init(&c, the_hash_algo);
973	git_hash_update(&c, hdr, hdrlen);
974	git_hash_update(&c, dat->buf, dat->len);
975	git_hash_final_oid(&oid, &c);
976	if (oidout)
977	oidcpy(oidout, &oid);
978
979	e = insert_object(&oid);
980	if (mark)
981	insert_mark(&marks, mark, e);
982	if (e->idx.offset) {
983	duplicate_count_by_type[type]++;
984	return 1;
985	}
986
987	for (source = the_repository->objects->sources; source; source = source->next) {
988	struct odb_source_files *files = odb_source_files_downcast(source);
989
990	if (!packfile_list_find_oid(packfile_store_get_packs(files->packed), &oid))
991	continue;
992	e->type = type;
993	e->pack_id = MAX_PACK_ID;
994	e->idx.offset = 1; /* just not zero! */
995	duplicate_count_by_type[type]++;
996	return 1;
997	}
998
999	if (last && last->data.len && last->data.buf && last->depth < max_depth
1000	&& dat->len > the_hash_algo->rawsz) {
1001
1002	delta_count_attempts_by_type[type]++;
1003	delta = diff_delta(last->data.buf, last->data.len,
1004	dat->buf, dat->len,
1005	&deltalen, dat->len - the_hash_algo->rawsz);
1006	} else
1007	delta = NULL;
1008
1009	git_deflate_init(&s, cfg->pack_compression_level);
1010	if (delta) {
1011	s.next_in = delta;
1012	s.avail_in = deltalen;
1013	} else {
1014	s.next_in = (void *)dat->buf;
1015	s.avail_in = dat->len;
1016	}
1017	s.avail_out = git_deflate_bound(&s, s.avail_in);
1018	s.next_out = out = xmalloc(s.avail_out);
1019	while (git_deflate(&s, Z_FINISH) == Z_OK)
1020	; /* nothing */
1021	git_deflate_end(&s);
1022
1023	/* Determine if we should auto-checkpoint. */
1024	if ((max_packsize
1025	&& (pack_size + PACK_SIZE_THRESHOLD + s.total_out) > max_packsize)
1026	\|\| (pack_size + PACK_SIZE_THRESHOLD + s.total_out) < pack_size) {
1027
1028	/* This new object needs to not have the current pack_id. */
1029	e->pack_id = pack_id + 1;
1030	cycle_packfile();
1031
1032	/* We cannot carry a delta into the new pack. */
1033	if (delta) {
1034	FREE_AND_NULL(delta);
1035
1036	git_deflate_init(&s, cfg->pack_compression_level);
1037	s.next_in = (void *)dat->buf;
1038	s.avail_in = dat->len;
1039	s.avail_out = git_deflate_bound(&s, s.avail_in);
1040	s.next_out = out = xrealloc(out, s.avail_out);
1041	while (git_deflate(&s, Z_FINISH) == Z_OK)
1042	; /* nothing */
1043	git_deflate_end(&s);
1044	}
1045	}
1046
1047	e->type = type;
1048	e->pack_id = pack_id;
1049	e->idx.offset = pack_size;
1050	object_count++;
1051	object_count_by_type[type]++;
1052
1053	crc32_begin(pack_file);
1054
1055	if (delta) {
1056	off_t ofs = e->idx.offset - last->offset;
1057	unsigned pos = sizeof(hdr) - 1;
1058
1059	delta_count_by_type[type]++;
1060	e->depth = last->depth + 1;
1061
1062	hdrlen = encode_in_pack_object_header(hdr, sizeof(hdr),
1063	OBJ_OFS_DELTA, deltalen);
1064	hashwrite(pack_file, hdr, hdrlen);
1065	pack_size += hdrlen;
1066
1067	hdr[pos] = ofs & 127;
1068	while (ofs >>= 7)
1069	hdr[--pos] = 128 \| (--ofs & 127);
1070	hashwrite(pack_file, hdr + pos, sizeof(hdr) - pos);
1071	pack_size += sizeof(hdr) - pos;
1072	} else {
1073	e->depth = 0;
1074	hdrlen = encode_in_pack_object_header(hdr, sizeof(hdr),
1075	type, dat->len);
1076	hashwrite(pack_file, hdr, hdrlen);
1077	pack_size += hdrlen;
1078	}
1079
1080	hashwrite(pack_file, out, s.total_out);
1081	pack_size += s.total_out;
1082
1083	e->idx.crc32 = crc32_end(pack_file);
1084
1085	free(out);
1086	free(delta);
1087	if (last) {
1088	if (last->no_swap) {
1089	last->data = *dat;
1090	} else {
1091	strbuf_swap(&last->data, dat);
1092	}
1093	last->offset = e->idx.offset;
1094	last->depth = e->depth;
1095	}
1096	return 0;
1097	}
1098
1099	static void truncate_pack(struct hashfile_checkpoint *checkpoint)
1100	{
1101	if (hashfile_truncate(pack_file, checkpoint))
1102	die_errno(_("cannot truncate pack to skip duplicate"));
1103	pack_size = checkpoint->offset;
1104	}
1105
1106	static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark)
1107	{
1108	size_t in_sz = 64 * 1024, out_sz = 64 * 1024;
1109	unsigned char *in_buf = xmalloc(in_sz);
1110	unsigned char *out_buf = xmalloc(out_sz);
1111	struct odb_source *source;
1112	struct object_entry *e;
1113	struct object_id oid;
1114	unsigned long hdrlen;
1115	off_t offset;
1116	struct git_hash_ctx c;
1117	git_zstream s;
1118	struct hashfile_checkpoint checkpoint;
1119	struct repo_config_values *cfg = repo_config_values(the_repository);
1120	int status = Z_OK;
1121
1122	/* Determine if we should auto-checkpoint. */
1123	if ((max_packsize
1124	&& (pack_size + PACK_SIZE_THRESHOLD + len) > max_packsize)
1125	\|\| (pack_size + PACK_SIZE_THRESHOLD + len) < pack_size)
1126	cycle_packfile();
1127
1128	hashfile_checkpoint_init(pack_file, &checkpoint);
1129	hashfile_checkpoint(pack_file, &checkpoint);
1130	offset = checkpoint.offset;
1131
1132	hdrlen = format_object_header((char *)out_buf, out_sz, OBJ_BLOB, len);
1133
1134	git_hash_init(&c, the_hash_algo);
1135	git_hash_update(&c, out_buf, hdrlen);
1136
1137	crc32_begin(pack_file);
1138
1139	git_deflate_init(&s, cfg->pack_compression_level);
1140
1141	hdrlen = encode_in_pack_object_header(out_buf, out_sz, OBJ_BLOB, len);
1142
1143	s.next_out = out_buf + hdrlen;
1144	s.avail_out = out_sz - hdrlen;
1145
1146	while (status != Z_STREAM_END) {
1147	if (0 < len && !s.avail_in) {
1148	size_t cnt = in_sz < len ? in_sz : (size_t)len;
1149	size_t n = fread(in_buf, 1, cnt, stdin);
1150	if (!n && feof(stdin))
1151	die(_("EOF in data (%" PRIuMAX " bytes remaining)"), len);
1152
1153	git_hash_update(&c, in_buf, n);
1154	s.next_in = in_buf;
1155	s.avail_in = n;
1156	len -= n;
1157	}
1158
1159	status = git_deflate(&s, len ? 0 : Z_FINISH);
1160
1161	if (!s.avail_out \|\| status == Z_STREAM_END) {
1162	size_t n = s.next_out - out_buf;
1163	hashwrite(pack_file, out_buf, n);
1164	pack_size += n;
1165	s.next_out = out_buf;
1166	s.avail_out = out_sz;
1167	}
1168
1169	switch (status) {
1170	case Z_OK:
1171	case Z_BUF_ERROR:
1172	case Z_STREAM_END:
1173	continue;
1174	default:
1175	die(_("unexpected deflate failure: %d"), status);
1176	}
1177	}
1178	git_deflate_end(&s);
1179	git_hash_final_oid(&oid, &c);
1180
1181	if (oidout)
1182	oidcpy(oidout, &oid);
1183
1184	e = insert_object(&oid);
1185
1186	if (mark)
1187	insert_mark(&marks, mark, e);
1188
1189	if (e->idx.offset) {
1190	duplicate_count_by_type[OBJ_BLOB]++;
1191	truncate_pack(&checkpoint);
1192	goto out;
1193	}
1194
1195	for (source = the_repository->objects->sources; source; source = source->next) {
1196	struct odb_source_files *files = odb_source_files_downcast(source);
1197
1198	if (!packfile_list_find_oid(packfile_store_get_packs(files->packed), &oid))
1199	continue;
1200	e->type = OBJ_BLOB;
1201	e->pack_id = MAX_PACK_ID;
1202	e->idx.offset = 1; /* just not zero! */
1203	duplicate_count_by_type[OBJ_BLOB]++;
1204	truncate_pack(&checkpoint);
1205	goto out;
1206	}
1207
1208	e->depth = 0;
1209	e->type = OBJ_BLOB;
1210	e->pack_id = pack_id;
1211	e->idx.offset = offset;
1212	e->idx.crc32 = crc32_end(pack_file);
1213	object_count++;
1214	object_count_by_type[OBJ_BLOB]++;
1215
1216	out:
1217	free(in_buf);
1218	free(out_buf);
1219	hashfile_checkpoint_release(&checkpoint);
1220	}
1221
1222	/* All calls must be guarded by find_object() or find_mark() to
1223	* ensure the 'struct object_entry' passed was written by this
1224	* process instance. We unpack the entry by the offset, avoiding
1225	* the need for the corresponding .idx file. This unpacking rule
1226	* works because we only use OBJ_REF_DELTA within the packfiles
1227	* created by fast-import.
1228	*
1229	* oe must not be NULL. Such an oe usually comes from giving
1230	* an unknown SHA-1 to find_object() or an undefined mark to
1231	* find_mark(). Callers must test for this condition and use
1232	* the standard read_sha1_file() when it happens.
1233	*
1234	* oe->pack_id must not be MAX_PACK_ID. Such an oe is usually from
1235	* find_mark(), where the mark was reloaded from an existing marks
1236	* file and is referencing an object that this fast-import process
1237	* instance did not write out to a packfile. Callers must test for
1238	* this condition and use read_sha1_file() instead.
1239	*/
1240	static void *gfi_unpack_entry(
1241	struct object_entry *oe,
1242	unsigned long *sizep)
1243	{
1244	enum object_type type;
1245	size_t size_st = 0;
1246	void *data;
1247	struct packed_git *p = all_packs[oe->pack_id];
1248	if (p == pack_data && p->pack_size < (pack_size + the_hash_algo->rawsz)) {
1249	/* The object is stored in the packfile we are writing to
1250	* and we have modified it since the last time we scanned
1251	* back to read a previously written object. If an old
1252	* window covered [p->pack_size, p->pack_size + rawsz) its
1253	* data is stale and is not valid. Closing all windows
1254	* and updating the packfile length ensures we can read
1255	* the newly written data.
1256	*/
1257	close_pack_windows(p);
1258	hashflush(pack_file);
1259
1260	/* We have to offer rawsz bytes additional on the end of
1261	* the packfile as the core unpacker code assumes the
1262	* footer is present at the file end and must promise
1263	* at least rawsz bytes within any window it maps. But
1264	* we don't actually create the footer here.
1265	*/
1266	p->pack_size = pack_size + the_hash_algo->rawsz;
1267	}
1268	data = unpack_entry(the_repository, p, oe->idx.offset, &type, &size_st);
1269	if (sizep)
1270	*sizep = cast_size_t_to_ulong(size_st);
1271	return data;
1272	}
1273
1274	static void load_tree(struct tree_entry *root)
1275	{
1276	struct object_id *oid = &root->versions[1].oid;
1277	struct object_entry *myoe;
1278	struct tree_content *t;
1279	unsigned long size;
1280	char *buf;
1281	const char *c;
1282
1283	root->tree = t = new_tree_content(8);
1284	if (is_null_oid(oid))
1285	return;
1286
1287	myoe = find_object(oid);
1288	if (myoe && myoe->pack_id != MAX_PACK_ID) {
1289	if (myoe->type != OBJ_TREE)
1290	die(_("not a tree: %s"), oid_to_hex(oid));
1291	t->delta_depth = myoe->depth;
1292	buf = gfi_unpack_entry(myoe, &size);
1293	if (!buf)
1294	die(_("can't load tree %s"), oid_to_hex(oid));
1295	} else {
1296	enum object_type type;
1297	size_t size_st = 0;
1298	buf = odb_read_object(the_repository->objects, oid, &type,
1299	&size_st);
1300	size = cast_size_t_to_ulong(size_st);
1301	if (!buf \|\| type != OBJ_TREE)
1302	die(_("can't load tree %s"), oid_to_hex(oid));
1303	}
1304
1305	c = buf;
1306	while (c != (buf + size)) {
1307	struct tree_entry *e = new_tree_entry();
1308
1309	if (t->entry_count == t->entry_capacity)
1310	root->tree = t = grow_tree_content(t, t->entry_count);
1311	t->entries[t->entry_count++] = e;
1312
1313	e->tree = NULL;
1314	c = parse_mode(c, &e->versions[1].mode);
1315	if (!c)
1316	die(_("corrupt mode in %s"), oid_to_hex(oid));
1317	e->versions[0].mode = e->versions[1].mode;
1318	e->name = to_atom(c, strlen(c));
1319	c += e->name->str_len + 1;
1320	oidread(&e->versions[0].oid, (unsigned char *)c,
1321	the_repository->hash_algo);
1322	oidread(&e->versions[1].oid, (unsigned char *)c,
1323	the_repository->hash_algo);
1324	c += the_hash_algo->rawsz;
1325	}
1326	free(buf);
1327	}
1328
1329	static int tecmp0 (const void _a, const void _b)
1330	{
1331	struct tree_entry a = ((struct tree_entry**)_a);
1332	struct tree_entry b = ((struct tree_entry**)_b);
1333	return base_name_compare(
1334	a->name->str_dat, a->name->str_len, a->versions[0].mode,
1335	b->name->str_dat, b->name->str_len, b->versions[0].mode);
1336	}
1337
1338	static int tecmp1 (const void _a, const void _b)
1339	{
1340	struct tree_entry a = ((struct tree_entry**)_a);
1341	struct tree_entry b = ((struct tree_entry**)_b);
1342	return base_name_compare(
1343	a->name->str_dat, a->name->str_len, a->versions[1].mode,
1344	b->name->str_dat, b->name->str_len, b->versions[1].mode);
1345	}
1346
1347	static void mktree(struct tree_content t, int v, struct strbuf b)
1348	{
1349	size_t maxlen = 0;
1350	unsigned int i;
1351
1352	if (!v)
1353	QSORT(t->entries, t->entry_count, tecmp0);
1354	else
1355	QSORT(t->entries, t->entry_count, tecmp1);
1356
1357	for (i = 0; i < t->entry_count; i++) {
1358	if (t->entries[i]->versions[v].mode)
1359	maxlen += t->entries[i]->name->str_len + 34;
1360	}
1361
1362	strbuf_reset(b);
1363	strbuf_grow(b, maxlen);
1364	for (i = 0; i < t->entry_count; i++) {
1365	struct tree_entry *e = t->entries[i];
1366	if (!e->versions[v].mode)
1367	continue;
1368	strbuf_addf(b, "%o %s%c",
1369	(unsigned int)(e->versions[v].mode & ~NO_DELTA),
1370	e->name->str_dat, '\0');
1371	strbuf_add(b, e->versions[v].oid.hash, the_hash_algo->rawsz);
1372	}
1373	}
1374
1375	static void store_tree(struct tree_entry *root)
1376	{
1377	struct tree_content *t;
1378	unsigned int i, j, del;
1379	struct last_object lo = { STRBUF_INIT, 0, 0, /* no_swap */ 1 };
1380	struct object_entry *le = NULL;
1381
1382	if (!is_null_oid(&root->versions[1].oid))
1383	return;
1384
1385	if (!root->tree)
1386	load_tree(root);
1387	t = root->tree;
1388
1389	for (i = 0; i < t->entry_count; i++) {
1390	if (t->entries[i]->tree)
1391	store_tree(t->entries[i]);
1392	}
1393
1394	if (!(root->versions[0].mode & NO_DELTA))
1395	le = find_object(&root->versions[0].oid);
1396	if (S_ISDIR(root->versions[0].mode) && le && le->pack_id == pack_id) {
1397	mktree(t, 0, &old_tree);
1398	lo.data = old_tree;
1399	lo.offset = le->idx.offset;
1400	lo.depth = t->delta_depth;
1401	}
1402
1403	mktree(t, 1, &new_tree);
1404	store_object(OBJ_TREE, &new_tree, &lo, &root->versions[1].oid, 0);
1405
1406	t->delta_depth = lo.depth;
1407	for (i = 0, j = 0, del = 0; i < t->entry_count; i++) {
1408	struct tree_entry *e = t->entries[i];
1409	if (e->versions[1].mode) {
1410	e->versions[0].mode = e->versions[1].mode;
1411	oidcpy(&e->versions[0].oid, &e->versions[1].oid);
1412	t->entries[j++] = e;
1413	} else {
1414	release_tree_entry(e);
1415	del++;
1416	}
1417	}
1418	t->entry_count -= del;
1419	}
1420
1421	static void tree_content_replace(
1422	struct tree_entry *root,
1423	const struct object_id *oid,
1424	const uint16_t mode,
1425	struct tree_content *newtree)
1426	{
1427	if (!S_ISDIR(mode))
1428	die(_("root cannot be a non-directory"));
1429	oidclr(&root->versions[0].oid, the_repository->hash_algo);
1430	oidcpy(&root->versions[1].oid, oid);
1431	if (root->tree)
1432	release_tree_content_recursive(root->tree);
1433	root->tree = newtree;
1434	}
1435
1436	static int tree_content_set(
1437	struct tree_entry *root,
1438	const char *p,
1439	const struct object_id *oid,
1440	const uint16_t mode,
1441	struct tree_content *subtree)
1442	{
1443	struct tree_content *t;
1444	const char *slash1;
1445	unsigned int i, n;
1446	struct tree_entry *e;
1447
1448	slash1 = strchrnul(p, '/');
1449	n = slash1 - p;
1450	if (!n)
1451	die(_("empty path component found in input"));
1452	if (!*slash1 && !S_ISDIR(mode) && subtree)
1453	die(_("non-directories cannot have subtrees"));
1454
1455	if (!root->tree)
1456	load_tree(root);
1457	t = root->tree;
1458	for (i = 0; i < t->entry_count; i++) {
1459	e = t->entries[i];
1460	if (e->name->str_len == n && !fspathncmp(p, e->name->str_dat, n)) {
1461	if (!*slash1) {
1462	if (!S_ISDIR(mode)
1463	&& e->versions[1].mode == mode
1464	&& oideq(&e->versions[1].oid, oid))
1465	return 0;
1466	e->versions[1].mode = mode;
1467	oidcpy(&e->versions[1].oid, oid);
1468	if (e->tree)
1469	release_tree_content_recursive(e->tree);
1470	e->tree = subtree;
1471
1472	/*
1473	* We need to leave e->versions[0].sha1 alone
1474	* to avoid modifying the preimage tree used
1475	* when writing out the parent directory.
1476	* But after replacing the subdir with a
1477	* completely different one, it's not a good
1478	* delta base any more, and besides, we've
1479	* thrown away the tree entries needed to
1480	* make a delta against it.
1481	*
1482	* So let's just explicitly disable deltas
1483	* for the subtree.
1484	*/
1485	if (S_ISDIR(e->versions[0].mode))
1486	e->versions[0].mode \|= NO_DELTA;
1487
1488	oidclr(&root->versions[1].oid, the_repository->hash_algo);
1489	return 1;
1490	}
1491	if (!S_ISDIR(e->versions[1].mode)) {
1492	e->tree = new_tree_content(8);
1493	e->versions[1].mode = S_IFDIR;
1494	}
1495	if (!e->tree)
1496	load_tree(e);
1497	if (tree_content_set(e, slash1 + 1, oid, mode, subtree)) {
1498	oidclr(&root->versions[1].oid, the_repository->hash_algo);
1499	return 1;
1500	}
1501	return 0;
1502	}
1503	}
1504
1505	if (t->entry_count == t->entry_capacity)
1506	root->tree = t = grow_tree_content(t, t->entry_count);
1507	e = new_tree_entry();
1508	e->name = to_atom(p, n);
1509	e->versions[0].mode = 0;
1510	oidclr(&e->versions[0].oid, the_repository->hash_algo);
1511	t->entries[t->entry_count++] = e;
1512	if (*slash1) {
1513	e->tree = new_tree_content(8);
1514	e->versions[1].mode = S_IFDIR;
1515	tree_content_set(e, slash1 + 1, oid, mode, subtree);
1516	} else {
1517	e->tree = subtree;
1518	e->versions[1].mode = mode;
1519	oidcpy(&e->versions[1].oid, oid);
1520	}
1521	oidclr(&root->versions[1].oid, the_repository->hash_algo);
1522	return 1;
1523	}
1524
1525	static int tree_content_remove(
1526	struct tree_entry *root,
1527	const char *p,
1528	struct tree_entry *backup_leaf,
1529	int allow_root)
1530	{
1531	struct tree_content *t;
1532	const char *slash1;
1533	unsigned int i, n;
1534	struct tree_entry *e;
1535
1536	slash1 = strchrnul(p, '/');
1537	n = slash1 - p;
1538
1539	if (!root->tree)
1540	load_tree(root);
1541
1542	if (!*p && allow_root) {
1543	e = root;
1544	goto del_entry;
1545	}
1546
1547	t = root->tree;
1548	for (i = 0; i < t->entry_count; i++) {
1549	e = t->entries[i];
1550	if (e->name->str_len == n && !fspathncmp(p, e->name->str_dat, n)) {
1551	if (*slash1 && !S_ISDIR(e->versions[1].mode))
1552	/*
1553	* If p names a file in some subdirectory, and a
1554	* file or symlink matching the name of the
1555	* parent directory of p exists, then p cannot
1556	* exist and need not be deleted.
1557	*/
1558	return 1;
1559	if (!*slash1 \|\| !S_ISDIR(e->versions[1].mode))
1560	goto del_entry;
1561	if (!e->tree)
1562	load_tree(e);
1563	if (tree_content_remove(e, slash1 + 1, backup_leaf, 0)) {
1564	for (n = 0; n < e->tree->entry_count; n++) {
1565	if (e->tree->entries[n]->versions[1].mode) {
1566	oidclr(&root->versions[1].oid,
1567	the_repository->hash_algo);
1568	return 1;
1569	}
1570	}
1571	backup_leaf = NULL;
1572	goto del_entry;
1573	}
1574	return 0;
1575	}
1576	}
1577	return 0;
1578
1579	del_entry:
1580	if (backup_leaf)
1581	memcpy(backup_leaf, e, sizeof(*backup_leaf));
1582	else if (e->tree)
1583	release_tree_content_recursive(e->tree);
1584	e->tree = NULL;
1585	e->versions[1].mode = 0;
1586	oidclr(&e->versions[1].oid, the_repository->hash_algo);
1587	oidclr(&root->versions[1].oid, the_repository->hash_algo);
1588	return 1;
1589	}
1590
1591	static int tree_content_get(
1592	struct tree_entry *root,
1593	const char *p,
1594	struct tree_entry *leaf,
1595	int allow_root)
1596	{
1597	struct tree_content *t;
1598	const char *slash1;
1599	unsigned int i, n;
1600	struct tree_entry *e;
1601
1602	slash1 = strchrnul(p, '/');
1603	n = slash1 - p;
1604	if (!n && !allow_root)
1605	die(_("empty path component found in input"));
1606
1607	if (!root->tree)
1608	load_tree(root);
1609
1610	if (!n) {
1611	e = root;
1612	goto found_entry;
1613	}
1614
1615	t = root->tree;
1616	for (i = 0; i < t->entry_count; i++) {
1617	e = t->entries[i];
1618	if (e->name->str_len == n && !fspathncmp(p, e->name->str_dat, n)) {
1619	if (!*slash1)
1620	goto found_entry;
1621	if (!S_ISDIR(e->versions[1].mode))
1622	return 0;
1623	if (!e->tree)
1624	load_tree(e);
1625	return tree_content_get(e, slash1 + 1, leaf, 0);
1626	}
1627	}
1628	return 0;
1629
1630	found_entry:
1631	memcpy(leaf, e, sizeof(*leaf));
1632	if (e->tree && is_null_oid(&e->versions[1].oid))
1633	leaf->tree = dup_tree_content(e->tree);
1634	else
1635	leaf->tree = NULL;
1636	return 1;
1637	}
1638
1639	static int update_branch(struct branch *b)
1640	{
1641	static const char *msg = "fast-import";
1642	struct ref_transaction *transaction;
1643	struct object_id old_oid;
1644	struct strbuf err = STRBUF_INIT;
1645	static const char *replace_prefix = "refs/replace/";
1646
1647	if (starts_with(b->name, replace_prefix) &&
1648	!strcmp(b->name + strlen(replace_prefix),
1649	oid_to_hex(&b->oid))) {
1650	if (!quiet)
1651	warning(_("dropping %s since it would point to "
1652	"itself (i.e. to %s)"),
1653	b->name, oid_to_hex(&b->oid));
1654	refs_delete_ref(get_main_ref_store(the_repository),
1655	NULL, b->name, NULL, 0);
1656	return 0;
1657	}
1658	if (is_null_oid(&b->oid)) {
1659	if (b->delete)
1660	refs_delete_ref(get_main_ref_store(the_repository),
1661	NULL, b->name, NULL, 0);
1662	return 0;
1663	}
1664	if (refs_read_ref(get_main_ref_store(the_repository), b->name, &old_oid))
1665	oidclr(&old_oid, the_repository->hash_algo);
1666	if (!force_update && !is_null_oid(&old_oid)) {
1667	struct commit old_cmit, new_cmit;
1668	int ret;
1669
1670	old_cmit = lookup_commit_reference_gently(the_repository,
1671	&old_oid, 0);
1672	new_cmit = lookup_commit_reference_gently(the_repository,
1673	&b->oid, 0);
1674	if (!old_cmit \|\| !new_cmit)
1675	return error(_("branch %s is missing commits."), b->name);
1676
1677	ret = repo_in_merge_bases(the_repository, old_cmit, new_cmit);
1678	if (ret < 0)
1679	exit(128);
1680	if (!ret) {
1681	warning(_("not updating %s"
1682	" (new tip %s does not contain %s)"),
1683	b->name, oid_to_hex(&b->oid),
1684	oid_to_hex(&old_oid));
1685	return -1;
1686	}
1687	}
1688	transaction = ref_store_transaction_begin(get_main_ref_store(the_repository),
1689	0, &err);
1690	if (!transaction \|\|
1691	ref_transaction_update(transaction, b->name, &b->oid, &old_oid,
1692	NULL, NULL, 0, msg, &err) \|\|
1693	ref_transaction_commit(transaction, &err)) {
1694	ref_transaction_free(transaction);
1695	error("%s", err.buf);
1696	strbuf_release(&err);
1697	return -1;
1698	}
1699	ref_transaction_free(transaction);
1700	strbuf_release(&err);
1701	return 0;
1702	}
1703
1704	static void dump_branches(void)
1705	{
1706	unsigned int i;
1707	struct branch *b;
1708
1709	for (i = 0; i < branch_table_sz; i++) {
1710	for (b = branch_table[i]; b; b = b->table_next_branch)
1711	failure \|= update_branch(b);
1712	}
1713	}
1714
1715	static void dump_tags(void)
1716	{
1717	static const char *msg = "fast-import";
1718	struct tag *t;
1719	struct strbuf ref_name = STRBUF_INIT;
1720	struct strbuf err = STRBUF_INIT;
1721	struct ref_transaction *transaction;
1722
1723	transaction = ref_store_transaction_begin(get_main_ref_store(the_repository),
1724	0, &err);
1725	if (!transaction) {
1726	failure \|= error("%s", err.buf);
1727	goto cleanup;
1728	}
1729	for (t = first_tag; t; t = t->next_tag) {
1730	strbuf_reset(&ref_name);
1731	strbuf_addf(&ref_name, "refs/tags/%s", t->name);
1732
1733	if (ref_transaction_update(transaction, ref_name.buf,
1734	&t->oid, NULL, NULL, NULL,
1735	0, msg, &err)) {
1736	failure \|= error("%s", err.buf);
1737	goto cleanup;
1738	}
1739	}
1740	if (ref_transaction_commit(transaction, &err))
1741	failure \|= error("%s", err.buf);
1742
1743	cleanup:
1744	ref_transaction_free(transaction);
1745	strbuf_release(&ref_name);
1746	strbuf_release(&err);
1747	}
1748
1749	static void dump_marks(void)
1750	{
1751	struct lock_file mark_lock = LOCK_INIT;
1752	FILE *f;
1753
1754	if (!export_marks_file \|\| (import_marks_file && !import_marks_file_done))
1755	return;
1756
1757	if (safe_create_leading_directories_const(the_repository, export_marks_file)) {
1758	failure \|= error_errno(_("unable to create leading directories of %s"),
1759	export_marks_file);
1760	return;
1761	}
1762
1763	if (hold_lock_file_for_update(&mark_lock, export_marks_file, 0) < 0) {
1764	failure \|= error_errno(_("unable to write marks file %s"),
1765	export_marks_file);
1766	return;
1767	}
1768
1769	f = fdopen_lock_file(&mark_lock, "w");
1770	if (!f) {
1771	int saved_errno = errno;
1772	rollback_lock_file(&mark_lock);
1773	failure \|= error(_("unable to write marks file %s: %s"),
1774	export_marks_file, strerror(saved_errno));
1775	return;
1776	}
1777
1778	for_each_mark(marks, 0, dump_marks_fn, f);
1779	if (commit_lock_file(&mark_lock)) {
1780	failure \|= error_errno(_("unable to write file %s"),
1781	export_marks_file);
1782	return;
1783	}
1784	}
1785
1786	static void insert_object_entry(struct mark_set *s, struct object_id oid, uintmax_t mark)
1787	{
1788	struct object_entry *e;
1789	e = find_object(oid);
1790	if (!e) {
1791	enum object_type type = odb_read_object_info(the_repository->objects,
1792	oid, NULL);
1793	if (type < 0)
1794	die(_("object not found: %s"), oid_to_hex(oid));
1795	e = insert_object(oid);
1796	e->type = type;
1797	e->pack_id = MAX_PACK_ID;
1798	e->idx.offset = 1; /* just not zero! */
1799	}
1800	insert_mark(s, mark, e);
1801	}
1802
1803	static void insert_oid_entry(struct mark_set *s, struct object_id oid, uintmax_t mark)
1804	{
1805	insert_mark(s, mark, xmemdupz(oid, sizeof(*oid)));
1806	}
1807
1808	static void read_mark_file(struct mark_set *s, FILE f, mark_set_inserter_t inserter)
1809	{
1810	char line[512];
1811	while (fgets(line, sizeof(line), f)) {
1812	uintmax_t mark;
1813	char *end;
1814	struct object_id oid;
1815
1816	/* Ensure SHA-1 objects are padded with zeros. */
1817	memset(oid.hash, 0, sizeof(oid.hash));
1818
1819	end = strchr(line, '\n');
1820	if (line[0] != ':' \|\| !end)
1821	die(_("corrupt mark line: %s"), line);
1822	*end = 0;
1823	mark = strtoumax(line + 1, &end, 10);
1824	if (!mark \|\| end == line + 1
1825	\|\| *end != ' '
1826	\|\| get_oid_hex_any(end + 1, &oid) == GIT_HASH_UNKNOWN)
1827	die(_("corrupt mark line: %s"), line);
1828	inserter(s, &oid, mark);
1829	}
1830	}
1831
1832	static void read_marks(void)
1833	{
1834	FILE *f = fopen(import_marks_file, "r");
1835	if (f)
1836	;
1837	else if (import_marks_file_ignore_missing && errno == ENOENT)
1838	goto done; /* Marks file does not exist */
1839	else
1840	die_errno(_("cannot read '%s'"), import_marks_file);
1841	read_mark_file(&marks, f, insert_object_entry);
1842	fclose(f);
1843	done:
1844	import_marks_file_done = 1;
1845	}
1846
1847
1848	static int read_next_command(void)
1849	{
1850	static int stdin_eof = 0;
1851
1852	if (stdin_eof) {
1853	unread_command_buf = 0;
1854	return EOF;
1855	}
1856
1857	for (;;) {
1858	if (unread_command_buf) {
1859	unread_command_buf = 0;
1860	} else {
1861	struct recent_command *rc;
1862
1863	stdin_eof = strbuf_getline_lf(&command_buf, stdin);
1864	if (stdin_eof)
1865	return EOF;
1866
1867	if (!seen_data_command
1868	&& !starts_with(command_buf.buf, "feature ")
1869	&& !starts_with(command_buf.buf, "option ")) {
1870	parse_argv();
1871	}
1872
1873	rc = rc_free;
1874	if (rc)
1875	rc_free = rc->next;
1876	else {
1877	rc = cmd_hist.next;
1878	cmd_hist.next = rc->next;
1879	cmd_hist.next->prev = &cmd_hist;
1880	free(rc->buf);
1881	}
1882
1883	rc->buf = xstrdup(command_buf.buf);
1884	rc->prev = cmd_tail;
1885	rc->next = cmd_hist.prev;
1886	rc->prev->next = rc;
1887	cmd_tail = rc;
1888	}
1889	if (command_buf.buf[0] == '#')
1890	continue;
1891	return 0;
1892	}
1893	}
1894
1895	static void skip_optional_lf(void)
1896	{
1897	int term_char = fgetc(stdin);
1898	if (term_char != '\n' && term_char != EOF)
1899	ungetc(term_char, stdin);
1900	}
1901
1902	static void parse_mark(void)
1903	{
1904	const char *v;
1905	if (skip_prefix(command_buf.buf, "mark :", &v)) {
1906	next_mark = strtoumax(v, NULL, 10);
1907	read_next_command();
1908	}
1909	else
1910	next_mark = 0;
1911	}
1912
1913	static void parse_original_identifier(void)
1914	{
1915	const char *v;
1916	if (skip_prefix(command_buf.buf, "original-oid ", &v))
1917	read_next_command();
1918	}
1919
1920	static int parse_data(struct strbuf sb, uintmax_t limit, uintmax_t len_res)
1921	{
1922	const char *data;
1923	strbuf_reset(sb);
1924
1925	if (!skip_prefix(command_buf.buf, "data ", &data))
1926	die(_("expected 'data n' command, found: %s"), command_buf.buf);
1927
1928	if (skip_prefix(data, "<<", &data)) {
1929	char *term = xstrdup(data);
1930	size_t term_len = command_buf.len - (data - command_buf.buf);
1931
1932	for (;;) {
1933	if (strbuf_getline_lf(&command_buf, stdin) == EOF)
1934	die(_("EOF in data (terminator '%s' not found)"), term);
1935	if (term_len == command_buf.len
1936	&& !strcmp(term, command_buf.buf))
1937	break;
1938	strbuf_addbuf(sb, &command_buf);
1939	strbuf_addch(sb, '\n');
1940	}
1941	free(term);
1942	}
1943	else {
1944	uintmax_t len = strtoumax(data, NULL, 10);
1945	size_t n = 0, length = (size_t)len;
1946
1947	if (limit && limit < len) {
1948	*len_res = len;
1949	return 0;
1950	}
1951	if (length < len)
1952	die(_("data is too large to use in this context"));
1953
1954	while (n < length) {
1955	size_t s = strbuf_fread(sb, length - n, stdin);
1956	if (!s && feof(stdin))
1957	die(_("EOF in data (%lu bytes remaining)"),
1958	(unsigned long)(length - n));
1959	n += s;
1960	}
1961	}
1962
1963	skip_optional_lf();
1964	return 1;
1965	}
1966
1967	static int validate_raw_date(const char src, struct strbuf result, int strict)
1968	{
1969	const char *orig_src = src;
1970	char *endp;
1971	unsigned long num;
1972
1973	errno = 0;
1974
1975	num = strtoul(src, &endp, 10);
1976	/*
1977	* NEEDSWORK: perhaps check for reasonable values? For example, we
1978	* could error on values representing times more than a
1979	* day in the future.
1980	*/
1981	if (errno \|\| endp == src \|\| *endp != ' ')
1982	return -1;
1983
1984	src = endp + 1;
1985	if (src != '-' && src != '+')
1986	return -1;
1987
1988	num = strtoul(src + 1, &endp, 10);
1989	/*
1990	* NEEDSWORK: check for brokenness other than num > 1400, such as
1991	* (num % 100) >= 60, or ((num % 100) % 15) != 0 ?
1992	*/
1993	if (errno \|\| endp == src + 1 \|\| endp \|\| / did not parse */
1994	(strict && (1400 < num)) /* parsed a broken timezone */
1995	)
1996	return -1;
1997
1998	strbuf_addstr(result, orig_src);
1999	return 0;
2000	}
2001
2002	static char parse_ident(const char buf)
2003	{
2004	const char *ltgt;
2005	size_t name_len;
2006	struct strbuf ident = STRBUF_INIT;
2007
2008	/* ensure there is a space delimiter even if there is no name */
2009	if (*buf == '<')
2010	--buf;
2011
2012	ltgt = buf + strcspn(buf, "<>");
2013	if (*ltgt != '<')
2014	die(_("missing < in ident string: %s"), buf);
2015	if (ltgt != buf && ltgt[-1] != ' ')
2016	die(_("missing space before < in ident string: %s"), buf);
2017	ltgt = ltgt + 1 + strcspn(ltgt + 1, "<>");
2018	if (*ltgt != '>')
2019	die(_("missing > in ident string: %s"), buf);
2020	ltgt++;
2021	if (*ltgt != ' ')
2022	die(_("missing space after > in ident string: %s"), buf);
2023	ltgt++;
2024	name_len = ltgt - buf;
2025	strbuf_add(&ident, buf, name_len);
2026
2027	switch (whenspec) {
2028	case WHENSPEC_RAW:
2029	if (validate_raw_date(ltgt, &ident, 1) < 0)
2030	die(_("invalid raw date \"%s\" in ident: %s"), ltgt, buf);
2031	break;
2032	case WHENSPEC_RAW_PERMISSIVE:
2033	if (validate_raw_date(ltgt, &ident, 0) < 0)
2034	die(_("invalid raw date \"%s\" in ident: %s"), ltgt, buf);
2035	break;
2036	case WHENSPEC_RFC2822:
2037	if (parse_date(ltgt, &ident) < 0)
2038	die(_("invalid rfc2822 date \"%s\" in ident: %s"), ltgt, buf);
2039	break;
2040	case WHENSPEC_NOW:
2041	if (strcmp("now", ltgt))
2042	die(_("date in ident must be 'now': %s"), buf);
2043	datestamp(&ident);
2044	break;
2045	}
2046
2047	return strbuf_detach(&ident, NULL);
2048	}
2049
2050	static void parse_and_store_blob(
2051	struct last_object *last,
2052	struct object_id *oidout,
2053	uintmax_t mark)
2054	{
2055	static struct strbuf buf = STRBUF_INIT;
2056	uintmax_t len;
2057
2058	if (parse_data(&buf, repo_settings_get_big_file_threshold(the_repository), &len))
2059	store_object(OBJ_BLOB, &buf, last, oidout, mark);
2060	else {
2061	if (last) {
2062	strbuf_release(&last->data);
2063	last->offset = 0;
2064	last->depth = 0;
2065	}
2066	stream_blob(len, oidout, mark);
2067	skip_optional_lf();
2068	}
2069	}
2070
2071	static void parse_new_blob(void)
2072	{
2073	read_next_command();
2074	parse_mark();
2075	parse_original_identifier();
2076	parse_and_store_blob(&last_blob, NULL, next_mark);
2077	}
2078
2079	static void unload_one_branch(void)
2080	{
2081	while (cur_active_branches
2082	&& cur_active_branches >= max_active_branches) {
2083	uintmax_t min_commit = ULONG_MAX;
2084	struct branch e, l = NULL, *p = NULL;
2085
2086	for (e = active_branches; e; e = e->active_next_branch) {
2087	if (e->last_commit < min_commit) {
2088	p = l;
2089	min_commit = e->last_commit;
2090	}
2091	l = e;
2092	}
2093
2094	if (p) {
2095	e = p->active_next_branch;
2096	p->active_next_branch = e->active_next_branch;
2097	} else {
2098	e = active_branches;
2099	active_branches = e->active_next_branch;
2100	}
2101	e->active = 0;
2102	e->active_next_branch = NULL;
2103	if (e->branch_tree.tree) {
2104	release_tree_content_recursive(e->branch_tree.tree);
2105	e->branch_tree.tree = NULL;
2106	}
2107	cur_active_branches--;
2108	}
2109	}
2110
2111	static void load_branch(struct branch *b)
2112	{
2113	load_tree(&b->branch_tree);
2114	if (!b->active) {
2115	b->active = 1;
2116	b->active_next_branch = active_branches;
2117	active_branches = b;
2118	cur_active_branches++;
2119	branch_load_count++;
2120	}
2121	}
2122
2123	static unsigned char convert_num_notes_to_fanout(uintmax_t num_notes)
2124	{
2125	unsigned char fanout = 0;
2126	while ((num_notes >>= 8))
2127	fanout++;
2128	return fanout;
2129	}
2130
2131	static void construct_path_with_fanout(const char *hex_sha1,
2132	unsigned char fanout, char *path)
2133	{
2134	unsigned int i = 0, j = 0;
2135	if (fanout >= the_hash_algo->rawsz)
2136	die(_("too large fanout (%u)"), fanout);
2137	while (fanout) {
2138	path[i++] = hex_sha1[j++];
2139	path[i++] = hex_sha1[j++];
2140	path[i++] = '/';
2141	fanout--;
2142	}
2143	memcpy(path + i, hex_sha1 + j, the_hash_algo->hexsz - j);
2144	path[i + the_hash_algo->hexsz - j] = '\0';
2145	}
2146
2147	static uintmax_t do_change_note_fanout(
2148	struct tree_entry orig_root, struct tree_entry root,
2149	char *hex_oid, unsigned int hex_oid_len,
2150	char *fullpath, unsigned int fullpath_len,
2151	unsigned char fanout)
2152	{
2153	struct tree_content *t;
2154	struct tree_entry *e, leaf;
2155	unsigned int i, tmp_hex_oid_len, tmp_fullpath_len;
2156	uintmax_t num_notes = 0;
2157	struct object_id oid;
2158	/* hex oid + '/' between each pair of hex digits + NUL */
2159	char realpath[GIT_MAX_HEXSZ + ((GIT_MAX_HEXSZ / 2) - 1) + 1];
2160	const unsigned hexsz = the_hash_algo->hexsz;
2161
2162	if (!root->tree)
2163	load_tree(root);
2164	t = root->tree;
2165
2166	for (i = 0; t && i < t->entry_count; i++) {
2167	e = t->entries[i];
2168	tmp_hex_oid_len = hex_oid_len + e->name->str_len;
2169	tmp_fullpath_len = fullpath_len;
2170
2171	/*
2172	* We're interested in EITHER existing note entries (entries
2173	* with exactly 40 hex chars in path, not including directory
2174	* separators), OR directory entries that may contain note
2175	* entries (with < 40 hex chars in path).
2176	* Also, each path component in a note entry must be a multiple
2177	* of 2 chars.
2178	*/
2179	if (!e->versions[1].mode \|\|
2180	tmp_hex_oid_len > hexsz \|\|
2181	e->name->str_len % 2)
2182	continue;
2183
2184	/* This _may_ be a note entry, or a subdir containing notes */
2185	memcpy(hex_oid + hex_oid_len, e->name->str_dat,
2186	e->name->str_len);
2187	if (tmp_fullpath_len)
2188	fullpath[tmp_fullpath_len++] = '/';
2189	memcpy(fullpath + tmp_fullpath_len, e->name->str_dat,
2190	e->name->str_len);
2191	tmp_fullpath_len += e->name->str_len;
2192	fullpath[tmp_fullpath_len] = '\0';
2193
2194	if (tmp_hex_oid_len == hexsz && !get_oid_hex(hex_oid, &oid)) {
2195	/* This is a note entry */
2196	if (fanout == 0xff) {
2197	/* Counting mode, no rename */
2198	num_notes++;
2199	continue;
2200	}
2201	construct_path_with_fanout(hex_oid, fanout, realpath);
2202	if (!strcmp(fullpath, realpath)) {
2203	/* Note entry is in correct location */
2204	num_notes++;
2205	continue;
2206	}
2207
2208	/* Rename fullpath to realpath */
2209	if (!tree_content_remove(orig_root, fullpath, &leaf, 0))
2210	die(_("failed to remove path %s"), fullpath);
2211	tree_content_set(orig_root, realpath,
2212	&leaf.versions[1].oid,
2213	leaf.versions[1].mode,
2214	leaf.tree);
2215	} else if (S_ISDIR(e->versions[1].mode)) {
2216	/* This is a subdir that may contain note entries */
2217	num_notes += do_change_note_fanout(orig_root, e,
2218	hex_oid, tmp_hex_oid_len,
2219	fullpath, tmp_fullpath_len, fanout);
2220	}
2221
2222	/* The above may have reallocated the current tree_content */
2223	t = root->tree;
2224	}
2225	return num_notes;
2226	}
2227
2228	static uintmax_t change_note_fanout(struct tree_entry *root,
2229	unsigned char fanout)
2230	{
2231	/*
2232	* The size of path is due to one slash between every two hex digits,
2233	* plus the terminating NUL. Note that there is no slash at the end, so
2234	* the number of slashes is one less than half the number of hex
2235	* characters.
2236	*/
2237	char hex_oid[GIT_MAX_HEXSZ], path[GIT_MAX_HEXSZ + (GIT_MAX_HEXSZ / 2) - 1 + 1];
2238	return do_change_note_fanout(root, root, hex_oid, 0, path, 0, fanout);
2239	}
2240
2241	static int parse_mapped_oid_hex(const char hex, struct object_id oid, const char **end)
2242	{
2243	int algo;
2244	khiter_t it;
2245
2246	/* Make SHA-1 object IDs have all-zero padding. */
2247	memset(oid->hash, 0, sizeof(oid->hash));
2248
2249	algo = parse_oid_hex_any(hex, oid, end);
2250	if (algo == GIT_HASH_UNKNOWN)
2251	return -1;
2252
2253	it = kh_get_oid_map(sub_oid_map, *oid);
2254	/* No such object? */
2255	if (it == kh_end(sub_oid_map)) {
2256	/* If we're using the same algorithm, pass it through. */
2257	if (hash_algos[algo].format_id == the_hash_algo->format_id)
2258	return 0;
2259	return -1;
2260	}
2261	oidcpy(oid, kh_value(sub_oid_map, it));
2262	return 0;
2263	}
2264
2265	/*
2266	* Given a pointer into a string, parse a mark reference:
2267	*
2268	* idnum ::= ':' bigint;
2269	*
2270	* Update *endptr to point to the first character after the value.
2271	*
2272	* Complain if the following character is not what is expected,
2273	* either a space or end of the string.
2274	*/
2275	static uintmax_t parse_mark_ref(const char p, char *endptr)
2276	{
2277	uintmax_t mark;
2278
2279	assert(*p == ':');
2280	p++;
2281	mark = strtoumax(p, endptr, 10);
2282	if (*endptr == p)
2283	die(_("no value after ':' in mark: %s"), command_buf.buf);
2284	return mark;
2285	}
2286
2287	/*
2288	* Parse the mark reference, and complain if this is not the end of
2289	* the string.
2290	*/
2291	static uintmax_t parse_mark_ref_eol(const char *p)
2292	{
2293	char *end;
2294	uintmax_t mark;
2295
2296	mark = parse_mark_ref(p, &end);
2297	if (*end != '\0')
2298	die(_("garbage after mark: %s"), command_buf.buf);
2299	return mark;
2300	}
2301
2302	/*
2303	* Parse the mark reference, demanding a trailing space. Update *p to
2304	* point to the first character after the space.
2305	*/
2306	static uintmax_t parse_mark_ref_space(const char **p)
2307	{
2308	uintmax_t mark;
2309	char *end;
2310
2311	mark = parse_mark_ref(*p, &end);
2312	if (*end++ != ' ')
2313	die(_("missing space after mark: %s"), command_buf.buf);
2314	*p = end;
2315	return mark;
2316	}
2317
2318	/*
2319	* Parse the path string into the strbuf. The path can either be quoted with
2320	* escape sequences or unquoted without escape sequences. Unquoted strings may
2321	* contain spaces only if `is_last_field` is nonzero; otherwise, it stops
2322	* parsing at the first space.
2323	*/
2324	static void parse_path(struct strbuf sb, const char p, const char **endp,
2325	int is_last_field, const char *field)
2326	{
2327	if (*p == '"') {
2328	if (unquote_c_style(sb, p, endp))
2329	die(_("invalid %s: %s"), field, command_buf.buf);
2330	if (strlen(sb->buf) != sb->len)
2331	die(_("NUL in %s: %s"), field, command_buf.buf);
2332	} else {
2333	/*
2334	* Unless we are parsing the last field of a line,
2335	* SP is the end of this field.
2336	*/
2337	*endp = is_last_field
2338	? p + strlen(p)
2339	: strchrnul(p, ' ');
2340	strbuf_add(sb, p, *endp - p);
2341	}
2342	}
2343
2344	/*
2345	* Parse the path string into the strbuf, and complain if this is not the end of
2346	* the string. Unquoted strings may contain spaces.
2347	*/
2348	static void parse_path_eol(struct strbuf sb, const char p, const char *field)
2349	{
2350	const char *end;
2351
2352	parse_path(sb, p, &end, 1, field);
2353	if (*end)
2354	die(_("garbage after %s: %s"), field, command_buf.buf);
2355	}
2356
2357	/*
2358	* Parse the path string into the strbuf, and ensure it is followed by a space.
2359	* Unquoted strings may not contain spaces. Update *endp to point to the first
2360	* character after the space.
2361	*/
2362	static void parse_path_space(struct strbuf sb, const char p,
2363	const char *endp, const char field)
2364	{
2365	parse_path(sb, p, endp, 0, field);
2366	if (**endp != ' ')
2367	die(_("missing space after %s: %s"), field, command_buf.buf);
2368	(*endp)++;
2369	}
2370
2371	static void file_change_m(const char p, struct branch b)
2372	{
2373	static struct strbuf path = STRBUF_INIT;
2374	struct object_entry *oe;
2375	struct object_id oid;
2376	uint16_t mode, inline_data = 0;
2377
2378	p = parse_mode(p, &mode);
2379	if (!p)
2380	die(_("corrupt mode: %s"), command_buf.buf);
2381	switch (mode) {
2382	case 0644:
2383	case 0755:
2384	mode \|= S_IFREG;
2385	case S_IFREG \| 0644:
2386	case S_IFREG \| 0755:
2387	case S_IFLNK:
2388	case S_IFDIR:
2389	case S_IFGITLINK:
2390	/* ok */
2391	break;
2392	default:
2393	die(_("corrupt mode: %s"), command_buf.buf);
2394	}
2395
2396	if (*p == ':') {
2397	oe = find_mark(marks, parse_mark_ref_space(&p));
2398	oidcpy(&oid, &oe->idx.oid);
2399	} else if (skip_prefix(p, "inline ", &p)) {
2400	inline_data = 1;
2401	oe = NULL; /* not used with inline_data, but makes gcc happy */
2402	} else {
2403	if (parse_mapped_oid_hex(p, &oid, &p))
2404	die(_("invalid dataref: %s"), command_buf.buf);
2405	oe = find_object(&oid);
2406	if (*p++ != ' ')
2407	die(_("missing space after SHA1: %s"), command_buf.buf);
2408	}
2409
2410	strbuf_reset(&path);
2411	parse_path_eol(&path, p, "path");
2412
2413	/* Git does not track empty, non-toplevel directories. */
2414	if (S_ISDIR(mode) &&
2415	is_empty_tree_oid(&oid, the_repository->hash_algo) &&
2416	*path.buf) {
2417	tree_content_remove(&b->branch_tree, path.buf, NULL, 0);
2418	return;
2419	}
2420
2421	if (S_ISGITLINK(mode)) {
2422	if (inline_data)
2423	die(_("Git links cannot be specified 'inline': %s"),
2424	command_buf.buf);
2425	else if (oe) {
2426	if (oe->type != OBJ_COMMIT)
2427	die(_("not a commit (actually a %s): %s"),
2428	type_name(oe->type), command_buf.buf);
2429	}
2430	/*
2431	* Accept the sha1 without checking; it expected to be in
2432	* another repository.
2433	*/
2434	} else if (inline_data) {
2435	if (S_ISDIR(mode))
2436	die(_("directories cannot be specified 'inline': %s"),
2437	command_buf.buf);
2438	while (read_next_command() != EOF) {
2439	const char *v;
2440	if (skip_prefix(command_buf.buf, "cat-blob ", &v))
2441	parse_cat_blob(v);
2442	else {
2443	parse_and_store_blob(&last_blob, &oid, 0);
2444	break;
2445	}
2446	}
2447	} else {
2448	enum object_type expected = S_ISDIR(mode) ?
2449	OBJ_TREE: OBJ_BLOB;
2450	enum object_type type = oe ? oe->type :
2451	odb_read_object_info(the_repository->objects,
2452	&oid, NULL);
2453	if (type < 0)
2454	die(_("%s not found: %s"),
2455	S_ISDIR(mode) ? _("tree") : _("blob"),
2456	command_buf.buf);
2457	if (type != expected)
2458	die(_("not a %s (actually a %s): %s"),
2459	type_name(expected), type_name(type),
2460	command_buf.buf);
2461	}
2462
2463	if (!*path.buf) {
2464	tree_content_replace(&b->branch_tree, &oid, mode, NULL);
2465	return;
2466	}
2467
2468	if (!verify_path(path.buf, mode))
2469	die(_("invalid path '%s'"), path.buf);
2470	tree_content_set(&b->branch_tree, path.buf, &oid, mode, NULL);
2471	}
2472
2473	static void file_change_d(const char p, struct branch b)
2474	{
2475	static struct strbuf path = STRBUF_INIT;
2476
2477	strbuf_reset(&path);
2478	parse_path_eol(&path, p, "path");
2479	tree_content_remove(&b->branch_tree, path.buf, NULL, 1);
2480	}
2481
2482	static void file_change_cr(const char p, struct branch b, int rename)
2483	{
2484	static struct strbuf source = STRBUF_INIT;
2485	static struct strbuf dest = STRBUF_INIT;
2486	struct tree_entry leaf;
2487
2488	strbuf_reset(&source);
2489	parse_path_space(&source, p, &p, "source");
2490	strbuf_reset(&dest);
2491	parse_path_eol(&dest, p, "dest");
2492
2493	memset(&leaf, 0, sizeof(leaf));
2494	if (rename)
2495	tree_content_remove(&b->branch_tree, source.buf, &leaf, 1);
2496	else
2497	tree_content_get(&b->branch_tree, source.buf, &leaf, 1);
2498	if (!leaf.versions[1].mode)
2499	die(_("path %s not in branch"), source.buf);
2500	if (!dest.buf) { / C "path/to/subdir" "" */
2501	tree_content_replace(&b->branch_tree,
2502	&leaf.versions[1].oid,
2503	leaf.versions[1].mode,
2504	leaf.tree);
2505	return;
2506	}
2507	if (!verify_path(dest.buf, leaf.versions[1].mode))
2508	die(_("invalid path '%s'"), dest.buf);
2509	tree_content_set(&b->branch_tree, dest.buf,
2510	&leaf.versions[1].oid,
2511	leaf.versions[1].mode,
2512	leaf.tree);
2513	}
2514
2515	static void note_change_n(const char p, struct branch b, unsigned char *old_fanout)
2516	{
2517	struct object_entry *oe;
2518	struct branch *s;
2519	struct object_id oid, commit_oid;
2520	char path[GIT_MAX_RAWSZ * 3];
2521	uint16_t inline_data = 0;
2522	unsigned char new_fanout;
2523
2524	/*
2525	* When loading a branch, we don't traverse its tree to count the real
2526	* number of notes (too expensive to do this for all non-note refs).
2527	* This means that recently loaded notes refs might incorrectly have
2528	* b->num_notes == 0, and consequently, old_fanout might be wrong.
2529	*
2530	* Fix this by traversing the tree and counting the number of notes
2531	* when b->num_notes == 0. If the notes tree is truly empty, the
2532	* calculation should not take long.
2533	*/
2534	if (b->num_notes == 0 && *old_fanout == 0) {
2535	/* Invoke change_note_fanout() in "counting mode". */
2536	b->num_notes = change_note_fanout(&b->branch_tree, 0xff);
2537	*old_fanout = convert_num_notes_to_fanout(b->num_notes);
2538	}
2539
2540	/* Now parse the notemodify command. */
2541	/* <dataref> or 'inline' */
2542	if (*p == ':') {
2543	oe = find_mark(marks, parse_mark_ref_space(&p));
2544	oidcpy(&oid, &oe->idx.oid);
2545	} else if (skip_prefix(p, "inline ", &p)) {
2546	inline_data = 1;
2547	oe = NULL; /* not used with inline_data, but makes gcc happy */
2548	} else {
2549	if (parse_mapped_oid_hex(p, &oid, &p))
2550	die(_("invalid dataref: %s"), command_buf.buf);
2551	oe = find_object(&oid);
2552	if (*p++ != ' ')
2553	die(_("missing space after SHA1: %s"), command_buf.buf);
2554	}
2555
2556	/* <commit-ish> */
2557	s = lookup_branch(p);
2558	if (s) {
2559	if (is_null_oid(&s->oid))
2560	die(_("can't add a note on empty branch."));
2561	oidcpy(&commit_oid, &s->oid);
2562	} else if (*p == ':') {
2563	uintmax_t commit_mark = parse_mark_ref_eol(p);
2564	struct object_entry *commit_oe = find_mark(marks, commit_mark);
2565	if (commit_oe->type != OBJ_COMMIT)
2566	die(_("mark :%" PRIuMAX " not a commit"), commit_mark);
2567	oidcpy(&commit_oid, &commit_oe->idx.oid);
2568	} else if (!repo_get_oid(the_repository, p, &commit_oid)) {
2569	size_t size;
2570	char *buf = odb_read_object_peeled(the_repository->objects,
2571	&commit_oid, OBJ_COMMIT, &size,
2572	&commit_oid);
2573	if (!buf \|\| size < the_hash_algo->hexsz + 6)
2574	die(_("not a valid commit: %s"), p);
2575	free(buf);
2576	} else
2577	die(_("invalid ref name or SHA1 expression: %s"), p);
2578
2579	if (inline_data) {
2580	read_next_command();
2581	parse_and_store_blob(&last_blob, &oid, 0);
2582	} else if (oe) {
2583	if (oe->type != OBJ_BLOB)
2584	die(_("not a blob (actually a %s): %s"),
2585	type_name(oe->type), command_buf.buf);
2586	} else if (!is_null_oid(&oid)) {
2587	enum object_type type = odb_read_object_info(the_repository->objects, &oid,
2588	NULL);
2589	if (type < 0)
2590	die(_("blob not found: %s"), command_buf.buf);
2591	if (type != OBJ_BLOB)
2592	die(_("not a blob (actually a %s): %s"),
2593	type_name(type), command_buf.buf);
2594	}
2595
2596	construct_path_with_fanout(oid_to_hex(&commit_oid), *old_fanout, path);
2597	if (tree_content_remove(&b->branch_tree, path, NULL, 0))
2598	b->num_notes--;
2599
2600	if (is_null_oid(&oid))
2601	return; /* nothing to insert */
2602
2603	b->num_notes++;
2604	new_fanout = convert_num_notes_to_fanout(b->num_notes);
2605	construct_path_with_fanout(oid_to_hex(&commit_oid), new_fanout, path);
2606	tree_content_set(&b->branch_tree, path, &oid, S_IFREG \| 0644, NULL);
2607	}
2608
2609	static void file_change_deleteall(struct branch *b)
2610	{
2611	release_tree_content_recursive(b->branch_tree.tree);
2612	oidclr(&b->branch_tree.versions[0].oid, the_repository->hash_algo);
2613	oidclr(&b->branch_tree.versions[1].oid, the_repository->hash_algo);
2614	load_tree(&b->branch_tree);
2615	b->num_notes = 0;
2616	}
2617
2618	static void parse_from_commit(struct branch b, char buf, unsigned long size)
2619	{
2620	if (!buf \|\| size < the_hash_algo->hexsz + 6)
2621	die(_("not a valid commit: %s"), oid_to_hex(&b->oid));
2622	if (memcmp("tree ", buf, 5)
2623	\|\| get_oid_hex(buf + 5, &b->branch_tree.versions[1].oid))
2624	die(_("the commit %s is corrupt"), oid_to_hex(&b->oid));
2625	oidcpy(&b->branch_tree.versions[0].oid,
2626	&b->branch_tree.versions[1].oid);
2627	}
2628
2629	static void parse_from_existing(struct branch *b)
2630	{
2631	if (is_null_oid(&b->oid)) {
2632	oidclr(&b->branch_tree.versions[0].oid, the_repository->hash_algo);
2633	oidclr(&b->branch_tree.versions[1].oid, the_repository->hash_algo);
2634	} else {
2635	unsigned long size;
2636	size_t size_st = 0;
2637	char *buf;
2638
2639	buf = odb_read_object_peeled(the_repository->objects, &b->oid,
2640	OBJ_COMMIT, &size_st, &b->oid);
2641	size = cast_size_t_to_ulong(size_st);
2642	parse_from_commit(b, buf, size);
2643	free(buf);
2644	}
2645	}
2646
2647	static int parse_objectish(struct branch b, const char objectish)
2648	{
2649	struct branch *s;
2650	struct object_id oid;
2651
2652	oidcpy(&oid, &b->branch_tree.versions[1].oid);
2653
2654	s = lookup_branch(objectish);
2655	if (b == s)
2656	die(_("can't create a branch from itself: %s"), b->name);
2657	else if (s) {
2658	struct object_id *t = &s->branch_tree.versions[1].oid;
2659	oidcpy(&b->oid, &s->oid);
2660	oidcpy(&b->branch_tree.versions[0].oid, t);
2661	oidcpy(&b->branch_tree.versions[1].oid, t);
2662	} else if (*objectish == ':') {
2663	uintmax_t idnum = parse_mark_ref_eol(objectish);
2664	struct object_entry *oe = find_mark(marks, idnum);
2665	if (oe->type != OBJ_COMMIT)
2666	die(_("mark :%" PRIuMAX " not a commit"), idnum);
2667	if (!oideq(&b->oid, &oe->idx.oid)) {
2668	oidcpy(&b->oid, &oe->idx.oid);
2669	if (oe->pack_id != MAX_PACK_ID) {
2670	unsigned long size;
2671	char *buf = gfi_unpack_entry(oe, &size);
2672	parse_from_commit(b, buf, size);
2673	free(buf);
2674	} else
2675	parse_from_existing(b);
2676	}
2677	} else if (!repo_get_oid(the_repository, objectish, &b->oid)) {
2678	parse_from_existing(b);
2679	if (is_null_oid(&b->oid))
2680	b->delete = 1;
2681	}
2682	else
2683	die(_("invalid ref name or SHA1 expression: %s"), objectish);
2684
2685	if (b->branch_tree.tree && !oideq(&oid, &b->branch_tree.versions[1].oid)) {
2686	release_tree_content_recursive(b->branch_tree.tree);
2687	b->branch_tree.tree = NULL;
2688	}
2689
2690	read_next_command();
2691	return 1;
2692	}
2693
2694	static int parse_from(struct branch *b)
2695	{
2696	const char *from;
2697
2698	if (!skip_prefix(command_buf.buf, "from ", &from))
2699	return 0;
2700
2701	return parse_objectish(b, from);
2702	}
2703
2704	static int parse_objectish_with_prefix(struct branch b, const char prefix)
2705	{
2706	const char *base;
2707
2708	if (!skip_prefix(command_buf.buf, prefix, &base))
2709	return 0;
2710
2711	return parse_objectish(b, base);
2712	}
2713
2714	static struct hash_list parse_merge(unsigned int count)
2715	{
2716	struct hash_list list = NULL, tail = &list, n;
2717	const char *from;
2718	struct branch *s;
2719
2720	*count = 0;
2721	while (skip_prefix(command_buf.buf, "merge ", &from)) {
2722	n = xmalloc(sizeof(*n));
2723	s = lookup_branch(from);
2724	if (s)
2725	oidcpy(&n->oid, &s->oid);
2726	else if (*from == ':') {
2727	uintmax_t idnum = parse_mark_ref_eol(from);
2728	struct object_entry *oe = find_mark(marks, idnum);
2729	if (oe->type != OBJ_COMMIT)
2730	die(_("mark :%" PRIuMAX " not a commit"), idnum);
2731	oidcpy(&n->oid, &oe->idx.oid);
2732	} else if (!repo_get_oid(the_repository, from, &n->oid)) {
2733	size_t size;
2734	char *buf = odb_read_object_peeled(the_repository->objects,
2735	&n->oid, OBJ_COMMIT,
2736	&size, &n->oid);
2737	if (!buf \|\| size < the_hash_algo->hexsz + 6)
2738	die(_("not a valid commit: %s"), from);
2739	free(buf);
2740	} else
2741	die(_("invalid ref name or SHA1 expression: %s"), from);
2742
2743	n->next = NULL;
2744	*tail = n;
2745	tail = &n->next;
2746
2747	(*count)++;
2748	read_next_command();
2749	}
2750	return list;
2751	}
2752
2753	struct signature_data {
2754	char hash_algo; / "sha1" or "sha256" */
2755	char sig_format; / "openpgp", "x509", "ssh", or "unknown" */
2756	struct strbuf data; /* The actual signature data */
2757	};
2758
2759	static void parse_one_signature(struct signature_data sig, const char v)
2760	{
2761	char args = xstrdup(v); / Will be freed when sig->hash_algo is freed */
2762	char *space = strchr(args, ' ');
2763
2764	if (!space)
2765	die(_("expected gpgsig format: 'gpgsig <hash-algo> <signature-format>', "
2766	"got 'gpgsig %s'"), args);
2767	*space = '\0';
2768
2769	sig->hash_algo = args;
2770	sig->sig_format = space + 1;
2771
2772	/* Validate hash algorithm */
2773	if (strcmp(sig->hash_algo, "sha1") &&
2774	strcmp(sig->hash_algo, "sha256"))
2775	die(_("unknown git hash algorithm in gpgsig: '%s'"), sig->hash_algo);
2776
2777	/* Validate signature format */
2778	if (!valid_signature_format(sig->sig_format))
2779	die(_("invalid signature format in gpgsig: '%s'"), sig->sig_format);
2780	if (!strcmp(sig->sig_format, "unknown"))
2781	warning(_("'unknown' signature format in gpgsig"));
2782
2783	/* Read signature data */
2784	read_next_command();
2785	parse_data(&sig->data, 0, NULL);
2786	}
2787
2788	static void discard_one_signature(void)
2789	{
2790	struct strbuf data = STRBUF_INIT;
2791
2792	read_next_command();
2793	parse_data(&data, 0, NULL);
2794	strbuf_release(&data);
2795	}
2796
2797	static void add_gpgsig_to_commit(struct strbuf *commit_data,
2798	const char *header,
2799	struct signature_data *sig)
2800	{
2801	struct string_list siglines = STRING_LIST_INIT_NODUP;
2802
2803	if (!sig \|\| !sig->hash_algo)
2804	return;
2805
2806	strbuf_addstr(commit_data, header);
2807	string_list_split_in_place(&siglines, sig->data.buf, "\n", -1);
2808	strbuf_add_separated_string_list(commit_data, "\n ", &siglines);
2809	strbuf_addch(commit_data, '\n');
2810	string_list_clear(&siglines, 1);
2811	strbuf_release(&sig->data);
2812	free(sig->hash_algo);
2813	}
2814
2815	static void store_signature(struct signature_data *stored_sig,
2816	struct signature_data *new_sig,
2817	const char *hash_type)
2818	{
2819	if (stored_sig->hash_algo) {
2820	warning(_("multiple %s signatures found, "
2821	"ignoring additional signature"),
2822	hash_type);
2823	strbuf_release(&new_sig->data);
2824	free(new_sig->hash_algo);
2825	} else {
2826	stored_sig = new_sig;
2827	}
2828	}
2829
2830	static void import_one_signature(struct signature_data *sig_sha1,
2831	struct signature_data *sig_sha256,
2832	const char *v)
2833	{
2834	struct signature_data sig = { NULL, NULL, STRBUF_INIT };
2835
2836	parse_one_signature(&sig, v);
2837
2838	if (!strcmp(sig.hash_algo, "sha1"))
2839	store_signature(sig_sha1, &sig, "SHA-1");
2840	else if (!strcmp(sig.hash_algo, "sha256"))
2841	store_signature(sig_sha256, &sig, "SHA-256");
2842	else
2843	die(_("parse_one_signature() returned unknown hash algo"));
2844	}
2845
2846	static void finalize_commit_buffer(struct strbuf *new_data,
2847	struct signature_data *sig_sha1,
2848	struct signature_data *sig_sha256,
2849	struct strbuf *msg)
2850	{
2851	add_gpgsig_to_commit(new_data, "gpgsig ", sig_sha1);
2852	add_gpgsig_to_commit(new_data, "gpgsig-sha256 ", sig_sha256);
2853
2854	strbuf_addch(new_data, '\n');
2855	strbuf_addbuf(new_data, msg);
2856	}
2857
2858	static void warn_invalid_signature(struct signature_check *check,
2859	const char *msg, enum sign_mode mode)
2860	{
2861	const char *signer = check->signer ? check->signer : _("unknown");
2862	const char *subject;
2863	int subject_len = find_commit_subject(msg, &subject);
2864
2865	switch (mode) {
2866	case SIGN_STRIP_IF_INVALID:
2867	if (subject_len > 100)
2868	warning(_("stripping invalid signature for commit '%.100s...'\n"
2869	" allegedly by %s"), subject, signer);
2870	else if (subject_len > 0)
2871	warning(_("stripping invalid signature for commit '%.*s'\n"
2872	" allegedly by %s"), subject_len, subject, signer);
2873	else
2874	warning(_("stripping invalid signature for commit\n"
2875	" allegedly by %s"), signer);
2876	break;
2877	case SIGN_SIGN_IF_INVALID:
2878	if (subject_len > 100)
2879	warning(_("replacing invalid signature for commit '%.100s...'\n"
2880	" allegedly by %s"), subject, signer);
2881	else if (subject_len > 0)
2882	warning(_("replacing invalid signature for commit '%.*s'\n"
2883	" allegedly by %s"), subject_len, subject, signer);
2884	else
2885	warning(_("replacing invalid signature for commit\n"
2886	" allegedly by %s"), signer);
2887	break;
2888	default:
2889	BUG("unsupported signing mode");
2890	}
2891	}
2892
2893	static void handle_signature_if_invalid(struct strbuf *new_data,
2894	struct signature_data *sig_sha1,
2895	struct signature_data *sig_sha256,
2896	struct strbuf *msg,
2897	enum sign_mode mode)
2898	{
2899	struct strbuf tmp_buf = STRBUF_INIT;
2900	struct signature_check signature_check = { 0 };
2901	int ret;
2902
2903	/* Check signature in a temporary commit buffer */
2904	strbuf_addbuf(&tmp_buf, new_data);
2905	finalize_commit_buffer(&tmp_buf, sig_sha1, sig_sha256, msg);
2906	ret = verify_commit_buffer(tmp_buf.buf, tmp_buf.len, &signature_check);
2907
2908	if (ret) {
2909	if (mode == SIGN_ABORT_IF_INVALID)
2910	die(_("aborting due to invalid signature"));
2911
2912	warn_invalid_signature(&signature_check, msg->buf, mode);
2913
2914	if (mode == SIGN_SIGN_IF_INVALID) {
2915	struct strbuf signature = STRBUF_INIT;
2916	struct strbuf payload = STRBUF_INIT;
2917
2918	/*
2919	* NEEDSWORK: To properly support interoperability mode
2920	* when signing commit signatures, the commit buffer
2921	* must be provided in both the repository and
2922	* compatibility object formats. As currently
2923	* implemented, only the repository object format is
2924	* considered meaning compatibility signatures cannot be
2925	* generated. Thus, attempting to sign commit signatures
2926	* in interoperability mode is currently unsupported.
2927	*/
2928	if (the_repository->compat_hash_algo)
2929	die(_("signing commits in interoperability mode is unsupported"));
2930
2931	strbuf_addstr(&payload, signature_check.payload);
2932	if (sign_buffer(&payload, &signature, signed_commit_keyid,
2933	SIGN_BUFFER_USE_DEFAULT_KEY))
2934	die(_("failed to sign commit object"));
2935	add_header_signature(new_data, &signature, the_hash_algo);
2936
2937	strbuf_release(&signature);
2938	strbuf_release(&payload);
2939	}
2940
2941	finalize_commit_buffer(new_data, NULL, NULL, msg);
2942	} else {
2943	strbuf_swap(new_data, &tmp_buf);
2944	}
2945
2946	signature_check_clear(&signature_check);
2947	strbuf_release(&tmp_buf);
2948	}
2949
2950	static void parse_new_commit(const char *arg)
2951	{
2952	static struct strbuf msg = STRBUF_INIT;
2953	struct signature_data sig_sha1 = { NULL, NULL, STRBUF_INIT };
2954	struct signature_data sig_sha256 = { NULL, NULL, STRBUF_INIT };
2955	struct branch *b;
2956	char *author = NULL;
2957	char *committer = NULL;
2958	char *encoding = NULL;
2959	struct hash_list *merge_list = NULL;
2960	unsigned int merge_count;
2961	unsigned char prev_fanout, new_fanout;
2962	const char *v;
2963
2964	b = lookup_branch(arg);
2965	if (!b)
2966	b = new_branch(arg);
2967
2968	read_next_command();
2969	parse_mark();
2970	parse_original_identifier();
2971	if (skip_prefix(command_buf.buf, "author ", &v)) {
2972	author = parse_ident(v);
2973	read_next_command();
2974	}
2975	if (skip_prefix(command_buf.buf, "committer ", &v)) {
2976	committer = parse_ident(v);
2977	read_next_command();
2978	}
2979	if (!committer)
2980	die(_("expected committer but didn't get one"));
2981
2982	while (skip_prefix(command_buf.buf, "gpgsig ", &v)) {
2983	switch (signed_commit_mode) {
2984
2985	/* First, modes that don't need the signature to be parsed */
2986	case SIGN_ABORT:
2987	die(_("encountered signed commit; use "
2988	"--signed-commits=<mode> to handle it"));
2989	case SIGN_WARN_STRIP:
2990	warning(_("stripping a commit signature"));
2991	/* fallthru */
2992	case SIGN_STRIP:
2993	discard_one_signature();
2994	break;
2995
2996	/* Second, modes that parse the signature */
2997	case SIGN_WARN_VERBATIM:
2998	warning(_("importing a commit signature verbatim"));
2999	/* fallthru */
3000	case SIGN_VERBATIM:
3001	case SIGN_STRIP_IF_INVALID:
3002	case SIGN_SIGN_IF_INVALID:
3003	case SIGN_ABORT_IF_INVALID:
3004	import_one_signature(&sig_sha1, &sig_sha256, v);
3005	break;
3006
3007	/* Third, BUG */
3008	default:
3009	BUG("invalid signed_commit_mode value %d", signed_commit_mode);
3010	}
3011	read_next_command();
3012	}
3013
3014	if (skip_prefix(command_buf.buf, "encoding ", &v)) {
3015	encoding = xstrdup(v);
3016	read_next_command();
3017	}
3018	parse_data(&msg, 0, NULL);
3019	read_next_command();
3020	parse_from(b);
3021	merge_list = parse_merge(&merge_count);
3022
3023	/* ensure the branch is active/loaded */
3024	if (!b->branch_tree.tree \|\| !max_active_branches) {
3025	unload_one_branch();
3026	load_branch(b);
3027	}
3028
3029	prev_fanout = convert_num_notes_to_fanout(b->num_notes);
3030
3031	/* file_change* */
3032	while (command_buf.len > 0) {
3033	if (skip_prefix(command_buf.buf, "M ", &v))
3034	file_change_m(v, b);
3035	else if (skip_prefix(command_buf.buf, "D ", &v))
3036	file_change_d(v, b);
3037	else if (skip_prefix(command_buf.buf, "R ", &v))
3038	file_change_cr(v, b, 1);
3039	else if (skip_prefix(command_buf.buf, "C ", &v))
3040	file_change_cr(v, b, 0);
3041	else if (skip_prefix(command_buf.buf, "N ", &v))
3042	note_change_n(v, b, &prev_fanout);
3043	else if (!strcmp("deleteall", command_buf.buf))
3044	file_change_deleteall(b);
3045	else if (skip_prefix(command_buf.buf, "ls ", &v))
3046	parse_ls(v, b);
3047	else if (skip_prefix(command_buf.buf, "cat-blob ", &v))
3048	parse_cat_blob(v);
3049	else {
3050	unread_command_buf = 1;
3051	break;
3052	}
3053	if (read_next_command() == EOF)
3054	break;
3055	}
3056
3057	new_fanout = convert_num_notes_to_fanout(b->num_notes);
3058	if (new_fanout != prev_fanout)
3059	b->num_notes = change_note_fanout(&b->branch_tree, new_fanout);
3060
3061	/* build the tree and the commit */
3062	store_tree(&b->branch_tree);
3063	oidcpy(&b->branch_tree.versions[0].oid,
3064	&b->branch_tree.versions[1].oid);
3065
3066	strbuf_reset(&new_data);
3067	strbuf_addf(&new_data, "tree %s\n",
3068	oid_to_hex(&b->branch_tree.versions[1].oid));
3069	if (!is_null_oid(&b->oid))
3070	strbuf_addf(&new_data, "parent %s\n",
3071	oid_to_hex(&b->oid));
3072	while (merge_list) {
3073	struct hash_list *next = merge_list->next;
3074	strbuf_addf(&new_data, "parent %s\n",
3075	oid_to_hex(&merge_list->oid));
3076	free(merge_list);
3077	merge_list = next;
3078	}
3079	strbuf_addf(&new_data,
3080	"author %s\n"
3081	"committer %s\n",
3082	author ? author : committer, committer);
3083	if (encoding)
3084	strbuf_addf(&new_data,
3085	"encoding %s\n",
3086	encoding);
3087
3088	if ((signed_commit_mode == SIGN_STRIP_IF_INVALID \|\|
3089	signed_commit_mode == SIGN_SIGN_IF_INVALID \|\|
3090	signed_commit_mode == SIGN_ABORT_IF_INVALID) &&
3091	(sig_sha1.hash_algo \|\| sig_sha256.hash_algo))
3092	handle_signature_if_invalid(&new_data, &sig_sha1, &sig_sha256,
3093	&msg, signed_commit_mode);
3094	else
3095	finalize_commit_buffer(&new_data, &sig_sha1, &sig_sha256, &msg);
3096
3097	free(author);
3098	free(committer);
3099	free(encoding);
3100
3101	if (!store_object(OBJ_COMMIT, &new_data, NULL, &b->oid, next_mark))
3102	b->pack_id = pack_id;
3103	b->last_commit = object_count_by_type[OBJ_COMMIT];
3104	}
3105
3106	static void handle_tag_signature_if_invalid(struct strbuf *buf,
3107	struct strbuf *msg,
3108	size_t sig_offset)
3109	{
3110	struct strbuf signature = STRBUF_INIT;
3111	struct strbuf payload = STRBUF_INIT;
3112	struct signature_check sigc = { 0 };
3113
3114	strbuf_addbuf(&payload, buf);
3115	strbuf_addch(&payload, '\n');
3116	strbuf_add(&payload, msg->buf, sig_offset);
3117	strbuf_add(&signature, msg->buf + sig_offset, msg->len - sig_offset);
3118
3119	sigc.payload_type = SIGNATURE_PAYLOAD_TAG;
3120	sigc.payload = strbuf_detach(&payload, &sigc.payload_len);
3121
3122	if (!check_signature(&sigc, signature.buf, signature.len))
3123	goto out;
3124
3125	if (signed_tag_mode == SIGN_ABORT_IF_INVALID)
3126	die(_("aborting due to invalid signature"));
3127
3128	strbuf_setlen(msg, sig_offset);
3129
3130	if (signed_tag_mode == SIGN_SIGN_IF_INVALID) {
3131	strbuf_attach(&payload, sigc.payload, sigc.payload_len,
3132	sigc.payload_len + 1);
3133	sigc.payload = NULL;
3134	strbuf_reset(&signature);
3135
3136	if (sign_buffer(&payload, &signature, signed_tag_keyid,
3137	SIGN_BUFFER_USE_DEFAULT_KEY))
3138	die(_("failed to sign tag object"));
3139
3140	strbuf_addbuf(msg, &signature);
3141	}
3142
3143	out:
3144	signature_check_clear(&sigc);
3145	strbuf_release(&signature);
3146	strbuf_release(&payload);
3147	}
3148
3149	static void handle_tag_signature(struct strbuf buf, struct strbuf msg, const char *name)
3150	{
3151	size_t sig_offset = parse_signed_buffer(msg->buf, msg->len);
3152
3153	/* If there is no signature, there is nothing to do. */
3154	if (sig_offset >= msg->len)
3155	return;
3156
3157	switch (signed_tag_mode) {
3158
3159	/* First, modes that don't change anything */
3160	case SIGN_WARN_VERBATIM:
3161	warning(_("importing a tag signature verbatim for tag '%s'"), name);
3162	/* fallthru */
3163	case SIGN_VERBATIM:
3164	/* Nothing to do, the signature will be put into the imported tag. */
3165	break;
3166
3167	/* Second, modes that remove the signature */
3168	case SIGN_WARN_STRIP:
3169	warning(_("stripping a tag signature for tag '%s'"), name);
3170	/* fallthru */
3171	case SIGN_STRIP:
3172	/* Truncate the buffer to remove the signature */
3173	strbuf_setlen(msg, sig_offset);
3174	break;
3175	case SIGN_ABORT_IF_INVALID:
3176	case SIGN_SIGN_IF_INVALID:
3177	case SIGN_STRIP_IF_INVALID:
3178	handle_tag_signature_if_invalid(buf, msg, sig_offset);
3179	break;
3180
3181	/* Third, aborting modes */
3182	case SIGN_ABORT:
3183	die(_("encountered signed tag; use "
3184	"--signed-tags=<mode> to handle it"));
3185	default:
3186	BUG("invalid signed_tag_mode value %d from tag '%s'",
3187	signed_tag_mode, name);
3188	}
3189	}
3190
3191	static void parse_new_tag(const char *arg)
3192	{
3193	static struct strbuf msg = STRBUF_INIT;
3194	const char *from;
3195	char *tagger;
3196	struct branch *s;
3197	struct tag *t;
3198	uintmax_t from_mark = 0;
3199	struct object_id oid;
3200	enum object_type type;
3201	const char *v;
3202
3203	t = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct tag));
3204	t->name = mem_pool_strdup(&fi_mem_pool, arg);
3205	if (last_tag)
3206	last_tag->next_tag = t;
3207	else
3208	first_tag = t;
3209	last_tag = t;
3210	read_next_command();
3211	parse_mark();
3212
3213	/* from ... */
3214	if (!skip_prefix(command_buf.buf, "from ", &from))
3215	die(_("expected 'from' command, got '%s'"), command_buf.buf);
3216	s = lookup_branch(from);
3217	if (s) {
3218	if (is_null_oid(&s->oid))
3219	die(_("can't tag an empty branch."));
3220	oidcpy(&oid, &s->oid);
3221	type = OBJ_COMMIT;
3222	} else if (*from == ':') {
3223	struct object_entry *oe;
3224	from_mark = parse_mark_ref_eol(from);
3225	oe = find_mark(marks, from_mark);
3226	type = oe->type;
3227	oidcpy(&oid, &oe->idx.oid);
3228	} else if (!repo_get_oid(the_repository, from, &oid)) {
3229	struct object_entry *oe = find_object(&oid);
3230	if (!oe) {
3231	type = odb_read_object_info(the_repository->objects,
3232	&oid, NULL);
3233	if (type < 0)
3234	die(_("not a valid object: %s"), from);
3235	} else
3236	type = oe->type;
3237	} else
3238	die(_("invalid ref name or SHA1 expression: %s"), from);
3239	read_next_command();
3240
3241	/* original-oid ... */
3242	parse_original_identifier();
3243
3244	/* tagger ... */
3245	if (skip_prefix(command_buf.buf, "tagger ", &v)) {
3246	tagger = parse_ident(v);
3247	read_next_command();
3248	} else
3249	tagger = NULL;
3250
3251	/* tag payload/message */
3252	parse_data(&msg, 0, NULL);
3253
3254	/* build the tag object */
3255	strbuf_reset(&new_data);
3256
3257	strbuf_addf(&new_data,
3258	"object %s\n"
3259	"type %s\n"
3260	"tag %s\n",
3261	oid_to_hex(&oid), type_name(type), t->name);
3262	if (tagger)
3263	strbuf_addf(&new_data,
3264	"tagger %s\n", tagger);
3265
3266	handle_tag_signature(&new_data, &msg, t->name);
3267
3268	strbuf_addch(&new_data, '\n');
3269	strbuf_addbuf(&new_data, &msg);
3270	free(tagger);
3271
3272	if (store_object(OBJ_TAG, &new_data, NULL, &t->oid, next_mark))
3273	t->pack_id = MAX_PACK_ID;
3274	else
3275	t->pack_id = pack_id;
3276	}
3277
3278	static void parse_reset_branch(const char *arg)
3279	{
3280	struct branch *b;
3281	const char *tag_name;
3282
3283	b = lookup_branch(arg);
3284	if (b) {
3285	oidclr(&b->oid, the_repository->hash_algo);
3286	oidclr(&b->branch_tree.versions[0].oid, the_repository->hash_algo);
3287	oidclr(&b->branch_tree.versions[1].oid, the_repository->hash_algo);
3288	if (b->branch_tree.tree) {
3289	release_tree_content_recursive(b->branch_tree.tree);
3290	b->branch_tree.tree = NULL;
3291	}
3292	}
3293	else
3294	b = new_branch(arg);
3295	read_next_command();
3296	parse_from(b);
3297	if (b->delete && skip_prefix(b->name, "refs/tags/", &tag_name)) {
3298	/*
3299	* Elsewhere, we call dump_branches() before dump_tags(),
3300	* and dump_branches() will handle ref deletions first, so
3301	* in order to make sure the deletion actually takes effect,
3302	* we need to remove the tag from our list of tags to update.
3303	*
3304	* NEEDSWORK: replace list of tags with hashmap for faster
3305	* deletion?
3306	*/
3307	struct tag t, prev = NULL;
3308	for (t = first_tag; t; t = t->next_tag) {
3309	if (!strcmp(t->name, tag_name))
3310	break;
3311	prev = t;
3312	}
3313	if (t) {
3314	if (prev)
3315	prev->next_tag = t->next_tag;
3316	else
3317	first_tag = t->next_tag;
3318	if (!t->next_tag)
3319	last_tag = prev;
3320	/* There is no mem_pool_free(t) function to call. */
3321	}
3322	}
3323	if (command_buf.len > 0)
3324	unread_command_buf = 1;
3325	}
3326
3327	static void cat_blob_write(const char *buf, unsigned long size)
3328	{
3329	if (write_in_full(cat_blob_fd, buf, size) < 0)
3330	die_errno(_("write to frontend failed"));
3331	}
3332
3333	static void cat_blob(struct object_entry oe, struct object_id oid)
3334	{
3335	struct strbuf line = STRBUF_INIT;
3336	unsigned long size;
3337	enum object_type type = 0;
3338	char *buf;
3339
3340	if (!oe \|\| oe->pack_id == MAX_PACK_ID) {
3341	size_t size_st = 0;
3342	buf = odb_read_object(the_repository->objects, oid, &type,
3343	&size_st);
3344	size = cast_size_t_to_ulong(size_st);
3345	} else {
3346	type = oe->type;
3347	buf = gfi_unpack_entry(oe, &size);
3348	}
3349
3350	/*
3351	* Output based on batch_one_object() from cat-file.c.
3352	*/
3353	if (type <= 0) {
3354	strbuf_reset(&line);
3355	strbuf_addf(&line, "%s missing\n", oid_to_hex(oid));
3356	cat_blob_write(line.buf, line.len);
3357	strbuf_release(&line);
3358	free(buf);
3359	return;
3360	}
3361	if (!buf)
3362	die(_("can't read object %s"), oid_to_hex(oid));
3363	if (type != OBJ_BLOB)
3364	die(_("object %s is a %s but a blob was expected."),
3365	oid_to_hex(oid), type_name(type));
3366	strbuf_reset(&line);
3367	strbuf_addf(&line, "%s %s %"PRIuMAX"\n", oid_to_hex(oid),
3368	type_name(type), (uintmax_t)size);
3369	cat_blob_write(line.buf, line.len);
3370	strbuf_release(&line);
3371	cat_blob_write(buf, size);
3372	cat_blob_write("\n", 1);
3373	if (oe && oe->pack_id == pack_id) {
3374	last_blob.offset = oe->idx.offset;
3375	strbuf_attach(&last_blob.data, buf, size, size + 1);
3376	last_blob.depth = oe->depth;
3377	} else
3378	free(buf);
3379	}
3380
3381	static void parse_get_mark(const char *p)
3382	{
3383	struct object_entry *oe;
3384	char output[GIT_MAX_HEXSZ + 2];
3385
3386	/* get-mark SP <object> LF */
3387	if (*p != ':')
3388	die(_("not a mark: %s"), p);
3389
3390	oe = find_mark(marks, parse_mark_ref_eol(p));
3391	if (!oe)
3392	die(_("unknown mark: %s"), command_buf.buf);
3393
3394	xsnprintf(output, sizeof(output), "%s\n", oid_to_hex(&oe->idx.oid));
3395	cat_blob_write(output, the_hash_algo->hexsz + 1);
3396	}
3397
3398	static void parse_cat_blob(const char *p)
3399	{
3400	struct object_entry *oe;
3401	struct object_id oid;
3402
3403	/* cat-blob SP <object> LF */
3404	if (*p == ':') {
3405	oe = find_mark(marks, parse_mark_ref_eol(p));
3406	if (!oe)
3407	die(_("unknown mark: %s"), command_buf.buf);
3408	oidcpy(&oid, &oe->idx.oid);
3409	} else {
3410	if (parse_mapped_oid_hex(p, &oid, &p))
3411	die(_("invalid dataref: %s"), command_buf.buf);
3412	if (*p)
3413	die(_("garbage after SHA1: %s"), command_buf.buf);
3414	oe = find_object(&oid);
3415	}
3416
3417	cat_blob(oe, &oid);
3418	}
3419
3420	static struct object_entry dereference(struct object_entry oe,
3421	struct object_id *oid)
3422	{
3423	unsigned long size;
3424	char *buf = NULL;
3425	const unsigned hexsz = the_hash_algo->hexsz;
3426
3427	if (!oe) {
3428	enum object_type type = odb_read_object_info(the_repository->objects,
3429	oid, NULL);
3430	if (type < 0)
3431	die(_("object not found: %s"), oid_to_hex(oid));
3432	/* cache it! */
3433	oe = insert_object(oid);
3434	oe->type = type;
3435	oe->pack_id = MAX_PACK_ID;
3436	oe->idx.offset = 1;
3437	}
3438	switch (oe->type) {
3439	case OBJ_TREE: /* easy case. */
3440	return oe;
3441	case OBJ_COMMIT:
3442	case OBJ_TAG:
3443	break;
3444	default:
3445	die(_("not a tree-ish: %s"), command_buf.buf);
3446	}
3447
3448	if (oe->pack_id != MAX_PACK_ID) { /* in a pack being written */
3449	buf = gfi_unpack_entry(oe, &size);
3450	} else {
3451	enum object_type unused;
3452	size_t size_st = 0;
3453	buf = odb_read_object(the_repository->objects, oid,
3454	&unused, &size_st);
3455	size = cast_size_t_to_ulong(size_st);
3456	}
3457	if (!buf)
3458	die(_("can't load object %s"), oid_to_hex(oid));
3459
3460	/* Peel one layer. */
3461	switch (oe->type) {
3462	case OBJ_TAG:
3463	if (size < hexsz + strlen("object ") \|\|
3464	get_oid_hex(buf + strlen("object "), oid))
3465	die(_("invalid SHA1 in tag: %s"), command_buf.buf);
3466	break;
3467	case OBJ_COMMIT:
3468	if (size < hexsz + strlen("tree ") \|\|
3469	get_oid_hex(buf + strlen("tree "), oid))
3470	die(_("invalid SHA1 in commit: %s"), command_buf.buf);
3471	}
3472
3473	free(buf);
3474	return find_object(oid);
3475	}
3476
3477	static void insert_mapped_mark(uintmax_t mark, void object, void cbp)
3478	{
3479	struct object_id *fromoid = object;
3480	struct object_id *tooid = find_mark(cbp, mark);
3481	int ret;
3482	khiter_t it;
3483
3484	it = kh_put_oid_map(sub_oid_map, *fromoid, &ret);
3485	/* We've already seen this object. */
3486	if (ret == 0)
3487	return;
3488	kh_value(sub_oid_map, it) = tooid;
3489	}
3490
3491	static void build_mark_map_one(struct mark_set from, struct mark_set to)
3492	{
3493	for_each_mark(from, 0, insert_mapped_mark, to);
3494	}
3495
3496	static void build_mark_map(struct string_list from, struct string_list to)
3497	{
3498	struct string_list_item fromp, top;
3499
3500	sub_oid_map = kh_init_oid_map();
3501
3502	for_each_string_list_item(fromp, from) {
3503	top = string_list_lookup(to, fromp->string);
3504	if (!fromp->util) {
3505	die(_("missing from marks for submodule '%s'"), fromp->string);
3506	} else if (!top \|\| !top->util) {
3507	die(_("missing to marks for submodule '%s'"), fromp->string);
3508	}
3509	build_mark_map_one(fromp->util, top->util);
3510	}
3511	}
3512
3513	static struct object_entry parse_treeish_dataref(const char *p)
3514	{
3515	struct object_id oid;
3516	struct object_entry *e;
3517
3518	if (*p == ':') { / <mark> */
3519	e = find_mark(marks, parse_mark_ref_space(p));
3520	if (!e)
3521	die(_("unknown mark: %s"), command_buf.buf);
3522	oidcpy(&oid, &e->idx.oid);
3523	} else { /* <sha1> */
3524	if (parse_mapped_oid_hex(*p, &oid, p))
3525	die(_("invalid dataref: %s"), command_buf.buf);
3526	e = find_object(&oid);
3527	if ((p)++ != ' ')
3528	die(_("missing space after tree-ish: %s"), command_buf.buf);
3529	}
3530
3531	while (!e \|\| e->type != OBJ_TREE)
3532	e = dereference(e, &oid);
3533	return e;
3534	}
3535
3536	static void print_ls(int mode, const unsigned char hash, const char path)
3537	{
3538	static struct strbuf line = STRBUF_INIT;
3539
3540	/* See show_tree(). */
3541	const char *type =
3542	S_ISGITLINK(mode) ? commit_type :
3543	S_ISDIR(mode) ? tree_type :
3544	blob_type;
3545
3546	if (!mode) {
3547	/* missing SP path LF */
3548	strbuf_reset(&line);
3549	strbuf_addstr(&line, "missing ");
3550	quote_c_style(path, &line, NULL, 0);
3551	strbuf_addch(&line, '\n');
3552	} else {
3553	/* mode SP type SP object_name TAB path LF */
3554	strbuf_reset(&line);
3555	strbuf_addf(&line, "%06o %s %s\t",
3556	mode & ~NO_DELTA, type, hash_to_hex(hash));
3557	quote_c_style(path, &line, NULL, 0);
3558	strbuf_addch(&line, '\n');
3559	}
3560	cat_blob_write(line.buf, line.len);
3561	}
3562
3563	static void parse_ls(const char p, struct branch b)
3564	{
3565	static struct strbuf path = STRBUF_INIT;
3566	struct tree_entry *root = NULL;
3567	struct tree_entry leaf = {NULL};
3568
3569	/* ls SP (<tree-ish> SP)? <path> */
3570	if (*p == '"') {
3571	if (!b)
3572	die(_("not in a commit: %s"), command_buf.buf);
3573	root = &b->branch_tree;
3574	} else {
3575	struct object_entry *e = parse_treeish_dataref(&p);
3576	root = new_tree_entry();
3577	oidcpy(&root->versions[1].oid, &e->idx.oid);
3578	if (!is_null_oid(&root->versions[1].oid))
3579	root->versions[1].mode = S_IFDIR;
3580	load_tree(root);
3581	}
3582	strbuf_reset(&path);
3583	parse_path_eol(&path, p, "path");
3584	tree_content_get(root, path.buf, &leaf, 1);
3585	/*
3586	* A directory in preparation would have a sha1 of zero
3587	* until it is saved. Save, for simplicity.
3588	*/
3589	if (S_ISDIR(leaf.versions[1].mode))
3590	store_tree(&leaf);
3591
3592	print_ls(leaf.versions[1].mode, leaf.versions[1].oid.hash, path.buf);
3593	if (leaf.tree)
3594	release_tree_content_recursive(leaf.tree);
3595	if (!b \|\| root != &b->branch_tree)
3596	release_tree_entry(root);
3597	}
3598
3599	static void checkpoint(void)
3600	{
3601	checkpoint_requested = 0;
3602	if (object_count) {
3603	cycle_packfile();
3604	}
3605	dump_branches();
3606	dump_tags();
3607	dump_marks();
3608	}
3609
3610	static void parse_checkpoint(void)
3611	{
3612	checkpoint_requested = 1;
3613	skip_optional_lf();
3614	}
3615
3616	static void parse_progress(void)
3617	{
3618	fwrite(command_buf.buf, 1, command_buf.len, stdout);
3619	fputc('\n', stdout);
3620	fflush(stdout);
3621	skip_optional_lf();
3622	}
3623
3624	static void parse_alias(void)
3625	{
3626	struct object_entry *e;
3627	struct branch b;
3628
3629	skip_optional_lf();
3630	read_next_command();
3631
3632	/* mark ... */
3633	parse_mark();
3634	if (!next_mark)
3635	die(_("expected 'mark' command, got %s"), command_buf.buf);
3636
3637	/* to ... */
3638	memset(&b, 0, sizeof(b));
3639	if (!parse_objectish_with_prefix(&b, "to "))
3640	die(_("expected 'to' command, got %s"), command_buf.buf);
3641	e = find_object(&b.oid);
3642	assert(e);
3643	insert_mark(&marks, next_mark, e);
3644	}
3645
3646	static char* make_fast_import_path(const char *path)
3647	{
3648	if (!relative_marks_paths \|\| is_absolute_path(path))
3649	return prefix_filename(global_prefix, path);
3650	return repo_git_path(the_repository, "info/fast-import/%s", path);
3651	}
3652
3653	static void option_import_marks(const char *marks,
3654	int from_stream, int ignore_missing)
3655	{
3656	if (import_marks_file) {
3657	if (from_stream)
3658	die(_("only one import-marks command allowed per stream"));
3659
3660	/* read previous mark file */
3661	if(!import_marks_file_from_stream)
3662	read_marks();
3663	}
3664
3665	free(import_marks_file);
3666	import_marks_file = make_fast_import_path(marks);
3667	import_marks_file_from_stream = from_stream;
3668	import_marks_file_ignore_missing = ignore_missing;
3669	}
3670
3671	static void option_date_format(const char *fmt)
3672	{
3673	if (!strcmp(fmt, "raw"))
3674	whenspec = WHENSPEC_RAW;
3675	else if (!strcmp(fmt, "raw-permissive"))
3676	whenspec = WHENSPEC_RAW_PERMISSIVE;
3677	else if (!strcmp(fmt, "rfc2822"))
3678	whenspec = WHENSPEC_RFC2822;
3679	else if (!strcmp(fmt, "now"))
3680	whenspec = WHENSPEC_NOW;
3681	else
3682	die(_("unknown --date-format argument %s"), fmt);
3683	}
3684
3685	static unsigned long ulong_arg(const char option, const char arg)
3686	{
3687	char *endptr;
3688	unsigned long rv = strtoul(arg, &endptr, 0);
3689	if (strchr(arg, '-') \|\| endptr == arg \|\| *endptr)
3690	die(_("%s: argument must be a non-negative integer"), option);
3691	return rv;
3692	}
3693
3694	static void option_depth(const char *depth)
3695	{
3696	max_depth = ulong_arg("--depth", depth);
3697	if (max_depth > MAX_DEPTH)
3698	die(_("--depth cannot exceed %u"), MAX_DEPTH);
3699	}
3700
3701	static void option_active_branches(const char *branches)
3702	{
3703	max_active_branches = ulong_arg("--active-branches", branches);
3704	}
3705
3706	static void option_export_marks(const char *marks)
3707	{
3708	free(export_marks_file);
3709	export_marks_file = make_fast_import_path(marks);
3710	}
3711
3712	static void option_cat_blob_fd(const char *fd)
3713	{
3714	unsigned long n = ulong_arg("--cat-blob-fd", fd);
3715	if (n > (unsigned long) INT_MAX)
3716	die(_("--cat-blob-fd cannot exceed %d"), INT_MAX);
3717	cat_blob_fd = (int) n;
3718	}
3719
3720	static void option_export_pack_edges(const char *edges)
3721	{
3722	char *fn = prefix_filename(global_prefix, edges);
3723	if (pack_edges)
3724	fclose(pack_edges);
3725	pack_edges = xfopen(fn, "a");
3726	free(fn);
3727	}
3728
3729	static void option_rewrite_submodules(const char arg, struct string_list list)
3730	{
3731	struct mark_set *ms;
3732	FILE *fp;
3733	char *s = xstrdup(arg);
3734	char *f = strchr(s, ':');
3735	if (!f)
3736	die(_("expected format name:filename for submodule rewrite option"));
3737	*f = '\0';
3738	f++;
3739	CALLOC_ARRAY(ms, 1);
3740
3741	f = prefix_filename(global_prefix, f);
3742	fp = fopen(f, "r");
3743	if (!fp)
3744	die_errno(_("cannot read '%s'"), f);
3745	read_mark_file(&ms, fp, insert_oid_entry);
3746	fclose(fp);
3747	free(f);
3748
3749	string_list_insert(list, s)->util = ms;
3750
3751	free(s);
3752	}
3753
3754	static int parse_one_option(const char *option)
3755	{
3756	if (skip_prefix(option, "max-pack-size=", &option)) {
3757	unsigned long v;
3758	if (!git_parse_ulong(option, &v))
3759	return 0;
3760	if (v < 8192) {
3761	warning(_("max-pack-size is now in bytes, assuming --max-pack-size=%lum"), v);
3762	v = 1024 1024;
3763	} else if (v < 1024 * 1024) {
3764	warning(_("minimum max-pack-size is 1 MiB"));
3765	v = 1024 * 1024;
3766	}
3767	max_packsize = v;
3768	} else if (skip_prefix(option, "big-file-threshold=", &option)) {
3769	unsigned long v;
3770	if (!git_parse_ulong(option, &v))
3771	return 0;
3772	repo_settings_set_big_file_threshold(the_repository, v);
3773	} else if (skip_prefix(option, "depth=", &option)) {
3774	option_depth(option);
3775	} else if (skip_prefix(option, "active-branches=", &option)) {
3776	option_active_branches(option);
3777	} else if (skip_prefix(option, "export-pack-edges=", &option)) {
3778	option_export_pack_edges(option);
3779	} else if (skip_prefix(option, "signed-commits=", &option)) {
3780	if (parse_sign_mode(option, &signed_commit_mode, &signed_commit_keyid))
3781	usagef(_("unknown --signed-commits mode '%s'"), option);
3782	} else if (skip_prefix(option, "signed-tags=", &option)) {
3783	if (parse_sign_mode(option, &signed_tag_mode, &signed_tag_keyid))
3784	usagef(_("unknown --signed-tags mode '%s'"), option);
3785	} else if (!strcmp(option, "quiet")) {
3786	show_stats = 0;
3787	quiet = 1;
3788	} else if (!strcmp(option, "stats")) {
3789	show_stats = 1;
3790	} else if (!strcmp(option, "allow-unsafe-features")) {
3791	; /* already handled during early option parsing */
3792	} else {
3793	return 0;
3794	}
3795
3796	return 1;
3797	}
3798
3799	static void check_unsafe_feature(const char *feature, int from_stream)
3800	{
3801	if (from_stream && !allow_unsafe_features)
3802	die(_("feature '%s' forbidden in input without --allow-unsafe-features"),
3803	feature);
3804	}
3805
3806	static int parse_one_feature(const char *feature, int from_stream)
3807	{
3808	const char *arg;
3809
3810	if (skip_prefix(feature, "date-format=", &arg)) {
3811	option_date_format(arg);
3812	} else if (skip_prefix(feature, "import-marks=", &arg)) {
3813	check_unsafe_feature("import-marks", from_stream);
3814	option_import_marks(arg, from_stream, 0);
3815	} else if (skip_prefix(feature, "import-marks-if-exists=", &arg)) {
3816	check_unsafe_feature("import-marks-if-exists", from_stream);
3817	option_import_marks(arg, from_stream, 1);
3818	} else if (skip_prefix(feature, "export-marks=", &arg)) {
3819	check_unsafe_feature(feature, from_stream);
3820	option_export_marks(arg);
3821	} else if (!strcmp(feature, "alias")) {
3822	; /* Don't die - this feature is supported */
3823	} else if (skip_prefix(feature, "rewrite-submodules-to=", &arg)) {
3824	option_rewrite_submodules(arg, &sub_marks_to);
3825	} else if (skip_prefix(feature, "rewrite-submodules-from=", &arg)) {
3826	option_rewrite_submodules(arg, &sub_marks_from);
3827	} else if (!strcmp(feature, "get-mark")) {
3828	; /* Don't die - this feature is supported */
3829	} else if (!strcmp(feature, "cat-blob")) {
3830	; /* Don't die - this feature is supported */
3831	} else if (!strcmp(feature, "relative-marks")) {
3832	relative_marks_paths = 1;
3833	} else if (!strcmp(feature, "no-relative-marks")) {
3834	relative_marks_paths = 0;
3835	} else if (!strcmp(feature, "done")) {
3836	require_explicit_termination = 1;
3837	} else if (!strcmp(feature, "force")) {
3838	force_update = 1;
3839	} else if (!strcmp(feature, "notes") \|\| !strcmp(feature, "ls")) {
3840	; /* do nothing; we have the feature */
3841	} else {
3842	return 0;
3843	}
3844
3845	return 1;
3846	}
3847
3848	static void parse_feature(const char *feature)
3849	{
3850	if (seen_data_command)
3851	die(_("got feature command '%s' after data command"), feature);
3852
3853	if (parse_one_feature(feature, 1))
3854	return;
3855
3856	die(_("this version of fast-import does not support feature %s."), feature);
3857	}
3858
3859	static void parse_option(const char *option)
3860	{
3861	if (seen_data_command)
3862	die(_("got option command '%s' after data command"), option);
3863
3864	if (parse_one_option(option))
3865	return;
3866
3867	die(_("this version of fast-import does not support option: %s"), option);
3868	}
3869
3870	static void git_pack_config(void)
3871	{
3872	int indexversion_value;
3873	int limit;
3874	unsigned long packsizelimit_value;
3875
3876	if (!repo_config_get_ulong(the_repository, "pack.depth", &max_depth)) {
3877	if (max_depth > MAX_DEPTH)
3878	max_depth = MAX_DEPTH;
3879	}
3880	if (!repo_config_get_int(the_repository, "pack.indexversion", &indexversion_value)) {
3881	pack_idx_opts.version = indexversion_value;
3882	if (pack_idx_opts.version > 2)
3883	git_die_config(the_repository, "pack.indexversion",
3884	"bad pack.indexVersion=%"PRIu32, pack_idx_opts.version);
3885	}
3886	if (!repo_config_get_ulong(the_repository, "pack.packsizelimit", &packsizelimit_value))
3887	max_packsize = packsizelimit_value;
3888
3889	if (!repo_config_get_int(the_repository, "fastimport.unpacklimit", &limit))
3890	unpack_limit = limit;
3891	else if (!repo_config_get_int(the_repository, "transfer.unpacklimit", &limit))
3892	unpack_limit = limit;
3893
3894	repo_config(the_repository, git_default_config, NULL);
3895	}
3896
3897	static const char fast_import_usage[] =
3898	"git fast-import [--date-format=<f>] [--max-pack-size=<n>] [--big-file-threshold=<n>] [--depth=<n>] [--active-branches=<n>] [--export-marks=<marks.file>]";
3899
3900	static void parse_argv(void)
3901	{
3902	unsigned int i;
3903
3904	for (i = 1; i < global_argc; i++) {
3905	const char *a = global_argv[i];
3906
3907	if (*a != '-' \|\| !strcmp(a, "--"))
3908	break;
3909
3910	if (!skip_prefix(a, "--", &a))
3911	die(_("unknown option %s"), a);
3912
3913	if (parse_one_option(a))
3914	continue;
3915
3916	if (parse_one_feature(a, 0))
3917	continue;
3918
3919	if (skip_prefix(a, "cat-blob-fd=", &a)) {
3920	option_cat_blob_fd(a);
3921	continue;
3922	}
3923
3924	die(_("unknown option --%s"), a);
3925	}
3926	if (i != global_argc)
3927	usage(fast_import_usage);
3928
3929	seen_data_command = 1;
3930	if (import_marks_file)
3931	read_marks();
3932	build_mark_map(&sub_marks_from, &sub_marks_to);
3933	}
3934
3935	int cmd_fast_import(int argc,
3936	const char **argv,
3937	const char *prefix,
3938	struct repository *repo)
3939	{
3940	unsigned int i;
3941
3942	show_usage_if_asked(argc, argv, fast_import_usage);
3943
3944	reset_pack_idx_option(&pack_idx_opts);
3945	git_pack_config();
3946
3947	alloc_objects(object_entry_alloc);
3948	strbuf_init(&command_buf, 0);
3949	CALLOC_ARRAY(atom_table, atom_table_sz);
3950	CALLOC_ARRAY(branch_table, branch_table_sz);
3951	CALLOC_ARRAY(avail_tree_table, avail_tree_table_sz);
3952	marks = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set));
3953
3954	hashmap_init(&object_table, object_entry_hashcmp, NULL, 0);
3955
3956	/*
3957	* We don't parse most options until after we've seen the set of
3958	* "feature" lines at the start of the stream (which allows the command
3959	* line to override stream data). But we must do an early parse of any
3960	* command-line options that impact how we interpret the feature lines.
3961	*/
3962	for (i = 1; i < argc; i++) {
3963	const char *arg = argv[i];
3964	if (*arg != '-' \|\| !strcmp(arg, "--"))
3965	break;
3966	if (!strcmp(arg, "--allow-unsafe-features"))
3967	allow_unsafe_features = 1;
3968	}
3969
3970	global_argc = argc;
3971	global_argv = argv;
3972	global_prefix = prefix;
3973
3974	rc_free = mem_pool_alloc(&fi_mem_pool, cmd_save * sizeof(*rc_free));
3975	for (i = 0; i < (cmd_save - 1); i++)
3976	rc_free[i].next = &rc_free[i + 1];
3977	rc_free[cmd_save - 1].next = NULL;
3978
3979	start_packfile();
3980	set_die_routine(die_nicely);
3981	set_checkpoint_signal();
3982	while (read_next_command() != EOF) {
3983	const char *v;
3984	if (!strcmp("blob", command_buf.buf))
3985	parse_new_blob();
3986	else if (skip_prefix(command_buf.buf, "commit ", &v))
3987	parse_new_commit(v);
3988	else if (skip_prefix(command_buf.buf, "tag ", &v))
3989	parse_new_tag(v);
3990	else if (skip_prefix(command_buf.buf, "reset ", &v))
3991	parse_reset_branch(v);
3992	else if (skip_prefix(command_buf.buf, "ls ", &v))
3993	parse_ls(v, NULL);
3994	else if (skip_prefix(command_buf.buf, "cat-blob ", &v))
3995	parse_cat_blob(v);
3996	else if (skip_prefix(command_buf.buf, "get-mark ", &v))
3997	parse_get_mark(v);
3998	else if (!strcmp("checkpoint", command_buf.buf))
3999	parse_checkpoint();
4000	else if (!strcmp("done", command_buf.buf))
4001	break;
4002	else if (!strcmp("alias", command_buf.buf))
4003	parse_alias();
4004	else if (starts_with(command_buf.buf, "progress "))
4005	parse_progress();
4006	else if (skip_prefix(command_buf.buf, "feature ", &v))
4007	parse_feature(v);
4008	else if (skip_prefix(command_buf.buf, "option git ", &v))
4009	parse_option(v);
4010	else if (starts_with(command_buf.buf, "option "))
4011	/* ignore non-git options*/;
4012	else
4013	die(_("unsupported command: %s"), command_buf.buf);
4014
4015	if (checkpoint_requested)
4016	checkpoint();
4017	}
4018
4019	/* argv hasn't been parsed yet, do so */
4020	if (!seen_data_command)
4021	parse_argv();
4022
4023	if (require_explicit_termination && feof(stdin))
4024	die(_("stream ends early"));
4025
4026	end_packfile();
4027
4028	dump_branches();
4029	dump_tags();
4030	unkeep_all_packs();
4031	dump_marks();
4032
4033	if (pack_edges)
4034	fclose(pack_edges);
4035
4036	if (show_stats) {
4037	uintmax_t total_count = 0, duplicate_count = 0;
4038	for (i = 0; i < ARRAY_SIZE(object_count_by_type); i++)
4039	total_count += object_count_by_type[i];
4040	for (i = 0; i < ARRAY_SIZE(duplicate_count_by_type); i++)
4041	duplicate_count += duplicate_count_by_type[i];
4042
4043	fprintf(stderr, "%s statistics:\n", argv[0]);
4044	fprintf(stderr, "---------------------------------------------------------------------\n");
4045	fprintf(stderr, "Alloc'd objects: %10" PRIuMAX "\n", alloc_count);
4046	fprintf(stderr, "Total objects: %10" PRIuMAX " (%10" PRIuMAX " duplicates )\n", total_count, duplicate_count);
4047	fprintf(stderr, " blobs : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB], delta_count_attempts_by_type[OBJ_BLOB]);
4048	fprintf(stderr, " trees : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE], delta_count_attempts_by_type[OBJ_TREE]);
4049	fprintf(stderr, " commits: %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT], delta_count_attempts_by_type[OBJ_COMMIT]);
4050	fprintf(stderr, " tags : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG], delta_count_attempts_by_type[OBJ_TAG]);
4051	fprintf(stderr, "Total branches: %10lu (%10lu loads )\n", branch_count, branch_load_count);
4052	fprintf(stderr, " marks: %10" PRIuMAX " (%10" PRIuMAX " unique )\n", (((uintmax_t)1) << marks->shift) * 1024, marks_set_count);
4053	fprintf(stderr, " atoms: %10u\n", atom_cnt);
4054	fprintf(stderr, "Memory total: %10" PRIuMAX " KiB\n", (tree_entry_allocd + fi_mem_pool.pool_alloc + alloc_count*sizeof(struct object_entry))/1024);
4055	fprintf(stderr, " pools: %10lu KiB\n", (unsigned long)((tree_entry_allocd + fi_mem_pool.pool_alloc) /1024));
4056	fprintf(stderr, " objects: %10" PRIuMAX " KiB\n", (alloc_count*sizeof(struct object_entry))/1024);
4057	fprintf(stderr, "---------------------------------------------------------------------\n");
4058	pack_report(repo);
4059	fprintf(stderr, "---------------------------------------------------------------------\n");
4060	fprintf(stderr, "\n");
4061	}
4062
4063	return failure ? 1 : 0;
4064	}