@leroysheep / CrazyWebTemps
Code Commits Pulls CI/CD 82,128
seen
bisect jch maint master next seen test todo
c 4,164 lines 112 KB
Raw
1 #define USE_THE_REPOSITORY_VARIABLE
2 #define DISABLE_SIGN_COMPARE_WARNINGS
3
4 #include "builtin.h"
5 #include "abspath.h"
6 #include "environment.h"
7 #include "gettext.h"
8 #include "hex.h"
9 #include "config.h"
10 #include "lockfile.h"
11 #include "object.h"
12 #include "blob.h"
13 #include "tree.h"
14 #include "commit.h"
15 #include "delta.h"
16 #include "pack.h"
17 #include "path.h"
18 #include "read-cache-ll.h"
19 #include "refs.h"
20 #include "csum-file.h"
21 #include "quote.h"
22 #include "dir.h"
23 #include "run-command.h"
24 #include "packfile.h"
25 #include "object-file.h"
26 #include "object-name.h"
27 #include "odb.h"
28 #include "mem-pool.h"
29 #include "commit-reach.h"
30 #include "khash.h"
31 #include "date.h"
32 #include "gpg-interface.h"
33 #include "parse-options.h"
34
35 #define PACK_ID_BITS 16
36 #define MAX_PACK_ID ((1<<PACK_ID_BITS)-1)
37 #define DEPTH_BITS 13
38 #define MAX_DEPTH ((1<<DEPTH_BITS)-1)
39
40 /*
41 * We abuse the setuid bit on directories to mean "do not delta".
42 */
43 #define NO_DELTA S_ISUID
44
45 /*
46 * The amount of additional space required in order to write an object into the
47 * current pack. This is the hash lengths at the end of the pack, plus the
48 * length of one object ID.
49 */
50 #define PACK_SIZE_THRESHOLD (the_hash_algo->rawsz * 3)
51
52 struct object_entry {
53 struct pack_idx_entry idx;
54 struct hashmap_entry ent;
55 uint32_t type : TYPE_BITS,
56 pack_id : PACK_ID_BITS,
57 depth : DEPTH_BITS;
58 };
59
60 static int object_entry_hashcmp(const void *map_data UNUSED,
61 const struct hashmap_entry *eptr,
62 const struct hashmap_entry *entry_or_key,
63 const void *keydata)
64 {
65 const struct object_id *oid = keydata;
66 const struct object_entry *e1, *e2;
67
68 e1 = container_of(eptr, const struct object_entry, ent);
69 if (oid)
70 return oidcmp(&e1->idx.oid, oid);
71
72 e2 = container_of(entry_or_key, const struct object_entry, ent);
73 return oidcmp(&e1->idx.oid, &e2->idx.oid);
74 }
75
76 struct object_entry_pool {
77 struct object_entry_pool *next_pool;
78 struct object_entry *next_free;
79 struct object_entry *end;
80 struct object_entry entries[FLEX_ARRAY]; /* more */
81 };
82
83 struct mark_set {
84 union {
85 struct object_id *oids[1024];
86 struct object_entry *marked[1024];
87 struct mark_set *sets[1024];
88 } data;
89 unsigned int shift;
90 };
91
92 struct last_object {
93 struct strbuf data;
94 off_t offset;
95 unsigned int depth;
96 unsigned no_swap : 1;
97 };
98
99 struct atom_str {
100 struct atom_str *next_atom;
101 unsigned short str_len;
102 char str_dat[FLEX_ARRAY]; /* more */
103 };
104
105 struct tree_content;
106 struct tree_entry {
107 struct tree_content *tree;
108 struct atom_str *name;
109 struct tree_entry_ms {
110 uint16_t mode;
111 struct object_id oid;
112 } versions[2];
113 };
114
115 struct tree_content {
116 unsigned int entry_capacity; /* must match avail_tree_content */
117 unsigned int entry_count;
118 unsigned int delta_depth;
119 struct tree_entry *entries[FLEX_ARRAY]; /* more */
120 };
121
122 struct avail_tree_content {
123 unsigned int entry_capacity; /* must match tree_content */
124 struct avail_tree_content *next_avail;
125 };
126
127 struct branch {
128 struct branch *table_next_branch;
129 struct branch *active_next_branch;
130 const char *name;
131 struct tree_entry branch_tree;
132 uintmax_t last_commit;
133 uintmax_t num_notes;
134 unsigned active : 1;
135 unsigned delete : 1;
136 unsigned pack_id : PACK_ID_BITS;
137 struct object_id oid;
138 };
139
140 struct tag {
141 struct tag *next_tag;
142 const char *name;
143 unsigned int pack_id;
144 struct object_id oid;
145 };
146
147 struct hash_list {
148 struct hash_list *next;
149 struct object_id oid;
150 };
151
152 typedef enum {
153 WHENSPEC_RAW = 1,
154 WHENSPEC_RAW_PERMISSIVE,
155 WHENSPEC_RFC2822,
156 WHENSPEC_NOW
157 } whenspec_type;
158
159 struct recent_command {
160 struct recent_command *prev;
161 struct recent_command *next;
162 char *buf;
163 };
164
165 typedef void (*mark_set_inserter_t)(struct mark_set **s, struct object_id *oid, uintmax_t mark);
166 typedef void (*each_mark_fn_t)(uintmax_t mark, void *obj, void *cbp);
167
168 /* Configured limits on output */
169 static unsigned long max_depth = 50;
170 static off_t max_packsize;
171 static int unpack_limit = 100;
172 static int force_update;
173
174 /* Stats and misc. counters */
175 static uintmax_t alloc_count;
176 static uintmax_t marks_set_count;
177 static uintmax_t object_count_by_type[1 << TYPE_BITS];
178 static uintmax_t duplicate_count_by_type[1 << TYPE_BITS];
179 static uintmax_t delta_count_by_type[1 << TYPE_BITS];
180 static uintmax_t delta_count_attempts_by_type[1 << TYPE_BITS];
181 static unsigned long object_count;
182 static unsigned long branch_count;
183 static unsigned long branch_load_count;
184 static int failure;
185 static FILE *pack_edges;
186 static unsigned int show_stats = 1;
187 static unsigned int quiet;
188 static enum sign_mode signed_tag_mode = SIGN_VERBATIM;
189 static enum sign_mode signed_commit_mode = SIGN_VERBATIM;
190 static const char *signed_commit_keyid;
191 static const char *signed_tag_keyid;
192
193 /* Memory pools */
194 static struct mem_pool fi_mem_pool = {
195 .block_alloc = 2*1024*1024 - sizeof(struct mp_block),
196 };
197
198 /* Atom management */
199 static unsigned int atom_table_sz = 4451;
200 static unsigned int atom_cnt;
201 static struct atom_str **atom_table;
202
203 /* The .pack file being generated */
204 static struct pack_idx_option pack_idx_opts;
205 static unsigned int pack_id;
206 static struct hashfile *pack_file;
207 static struct packed_git *pack_data;
208 static struct packed_git **all_packs;
209 static off_t pack_size;
210
211 /* Table of objects we've written. */
212 static unsigned int object_entry_alloc = 5000;
213 static struct object_entry_pool *blocks;
214 static struct hashmap object_table;
215 static struct mark_set *marks;
216 static char *export_marks_file;
217 static char *import_marks_file;
218 static int import_marks_file_from_stream;
219 static int import_marks_file_ignore_missing;
220 static int import_marks_file_done;
221 static int relative_marks_paths;
222
223 /* Our last blob */
224 static struct last_object last_blob = {
225 .data = STRBUF_INIT,
226 };
227
228 /* Tree management */
229 static unsigned int tree_entry_alloc = 1000;
230 static void *avail_tree_entry;
231 static unsigned int avail_tree_table_sz = 100;
232 static struct avail_tree_content **avail_tree_table;
233 static size_t tree_entry_allocd;
234 static struct strbuf old_tree = STRBUF_INIT;
235 static struct strbuf new_tree = STRBUF_INIT;
236
237 /* Branch data */
238 static unsigned long max_active_branches = 5;
239 static unsigned long cur_active_branches;
240 static unsigned long branch_table_sz = 1039;
241 static struct branch **branch_table;
242 static struct branch *active_branches;
243
244 /* Tag data */
245 static struct tag *first_tag;
246 static struct tag *last_tag;
247
248 /* Input stream parsing */
249 static whenspec_type whenspec = WHENSPEC_RAW;
250 static struct strbuf command_buf = STRBUF_INIT;
251 static int unread_command_buf;
252 static struct recent_command cmd_hist = {
253 .prev = &cmd_hist,
254 .next = &cmd_hist,
255 };
256 static struct recent_command *cmd_tail = &cmd_hist;
257 static struct recent_command *rc_free;
258 static unsigned int cmd_save = 100;
259 static uintmax_t next_mark;
260 static struct strbuf new_data = STRBUF_INIT;
261 static int require_explicit_termination;
262
263 /* Signal handling */
264 static volatile sig_atomic_t checkpoint_requested;
265
266 /* Submodule marks */
267 static struct string_list sub_marks_from = STRING_LIST_INIT_DUP;
268 static struct string_list sub_marks_to = STRING_LIST_INIT_DUP;
269 static kh_oid_map_t *sub_oid_map;
270
271 /* Where to write output of cat-blob commands */
272 static int cat_blob_fd = STDOUT_FILENO;
273
274 /* Command state */
275 struct fast_import_state {
276 int argc;
277 const char **argv;
278 const char *prefix;
279 int seen_data_command;
280 int allow_unsafe_features;
281 struct option *option;
282 };
283
284 static void fast_import_state_init(struct fast_import_state *state,
285 int argc, const char **argv,
286 const char *prefix, struct option *option)
287 {
288 memset(state, 0, sizeof(*state));
289 state->argc = argc;
290 state->argv = argv;
291 state->prefix = prefix;
292 state->option = option;
293 }
294
295 static void parse_argv(struct fast_import_state *state);
296 static void parse_get_mark(struct fast_import_state *state, const char *p);
297 static void parse_cat_blob(struct fast_import_state *state, const char *p);
298 static void parse_ls(struct fast_import_state *state, const char *p, struct branch *b);
299
300 static void for_each_mark(struct mark_set *m, uintmax_t base, each_mark_fn_t callback, void *p)
301 {
302 uintmax_t k;
303 if (m->shift) {
304 for (k = 0; k < 1024; k++) {
305 if (m->data.sets[k])
306 for_each_mark(m->data.sets[k], base + (k << m->shift), callback, p);
307 }
308 } else {
309 for (k = 0; k < 1024; k++) {
310 if (m->data.marked[k])
311 callback(base + k, m->data.marked[k], p);
312 }
313 }
314 }
315
316 static void dump_marks_fn(uintmax_t mark, void *object, void *cbp) {
317 struct object_entry *e = object;
318 FILE *f = cbp;
319
320 fprintf(f, ":%" PRIuMAX " %s\n", mark, oid_to_hex(&e->idx.oid));
321 }
322
323 static void write_branch_report(FILE *rpt, struct branch *b)
324 {
325 fprintf(rpt, "%s:\n", b->name);
326
327 fprintf(rpt, " status :");
328 if (b->active)
329 fputs(" active", rpt);
330 if (b->branch_tree.tree)
331 fputs(" loaded", rpt);
332 if (is_null_oid(&b->branch_tree.versions[1].oid))
333 fputs(" dirty", rpt);
334 fputc('\n', rpt);
335
336 fprintf(rpt, " tip commit : %s\n", oid_to_hex(&b->oid));
337 fprintf(rpt, " old tree : %s\n",
338 oid_to_hex(&b->branch_tree.versions[0].oid));
339 fprintf(rpt, " cur tree : %s\n",
340 oid_to_hex(&b->branch_tree.versions[1].oid));
341 fprintf(rpt, " commit clock: %" PRIuMAX "\n", b->last_commit);
342
343 fputs(" last pack : ", rpt);
344 if (b->pack_id < MAX_PACK_ID)
345 fprintf(rpt, "%u", b->pack_id);
346 fputc('\n', rpt);
347
348 fputc('\n', rpt);
349 }
350
351 static void write_crash_report(const char *err)
352 {
353 char *loc = repo_git_path(the_repository, "fast_import_crash_%"PRIuMAX, (uintmax_t) getpid());
354 FILE *rpt = fopen(loc, "w");
355 struct branch *b;
356 unsigned long lu;
357 struct recent_command *rc;
358
359 if (!rpt) {
360 error_errno(_("can't write crash report %s"), loc);
361 free(loc);
362 return;
363 }
364
365 fprintf(stderr, _("fast-import: dumping crash report to %s\n"), loc);
366
367 fprintf(rpt, "fast-import crash report:\n");
368 fprintf(rpt, " fast-import process: %"PRIuMAX"\n", (uintmax_t) getpid());
369 fprintf(rpt, " parent process : %"PRIuMAX"\n", (uintmax_t) getppid());
370 fprintf(rpt, " at %s\n", show_date(time(NULL), 0, DATE_MODE(ISO8601)));
371 fputc('\n', rpt);
372
373 fputs("fatal: ", rpt);
374 fputs(err, rpt);
375 fputc('\n', rpt);
376
377 fputc('\n', rpt);
378 fputs("Most Recent Commands Before Crash\n", rpt);
379 fputs("---------------------------------\n", rpt);
380 for (rc = cmd_hist.next; rc != &cmd_hist; rc = rc->next) {
381 if (rc->next == &cmd_hist)
382 fputs("* ", rpt);
383 else
384 fputs(" ", rpt);
385 fputs(rc->buf, rpt);
386 fputc('\n', rpt);
387 }
388
389 fputc('\n', rpt);
390 fputs("Active Branch LRU\n", rpt);
391 fputs("-----------------\n", rpt);
392 fprintf(rpt, " active_branches = %lu cur, %lu max\n",
393 cur_active_branches,
394 max_active_branches);
395 fputc('\n', rpt);
396 fputs(" pos clock name\n", rpt);
397 fputs(" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n", rpt);
398 for (b = active_branches, lu = 0; b; b = b->active_next_branch)
399 fprintf(rpt, " %2lu) %6" PRIuMAX" %s\n",
400 ++lu, b->last_commit, b->name);
401
402 fputc('\n', rpt);
403 fputs("Inactive Branches\n", rpt);
404 fputs("-----------------\n", rpt);
405 for (lu = 0; lu < branch_table_sz; lu++) {
406 for (b = branch_table[lu]; b; b = b->table_next_branch)
407 write_branch_report(rpt, b);
408 }
409
410 if (first_tag) {
411 struct tag *tg;
412 fputc('\n', rpt);
413 fputs("Annotated Tags\n", rpt);
414 fputs("--------------\n", rpt);
415 for (tg = first_tag; tg; tg = tg->next_tag) {
416 fputs(oid_to_hex(&tg->oid), rpt);
417 fputc(' ', rpt);
418 fputs(tg->name, rpt);
419 fputc('\n', rpt);
420 }
421 }
422
423 fputc('\n', rpt);
424 fputs("Marks\n", rpt);
425 fputs("-----\n", rpt);
426 if (export_marks_file)
427 fprintf(rpt, " exported to %s\n", export_marks_file);
428 else
429 for_each_mark(marks, 0, dump_marks_fn, rpt);
430
431 fputc('\n', rpt);
432 fputs("-------------------\n", rpt);
433 fputs("END OF CRASH REPORT\n", rpt);
434 fclose(rpt);
435 free(loc);
436 }
437
438 static void end_packfile(void);
439 static void unkeep_all_packs(void);
440 static void dump_marks(void);
441
442 static NORETURN void die_nicely(const char *err, va_list params)
443 {
444 va_list cp;
445 static int zombie;
446 report_fn die_message_fn = get_die_message_routine();
447
448 va_copy(cp, params);
449 die_message_fn(err, params);
450
451 if (!zombie) {
452 char message[2 * PATH_MAX];
453
454 zombie = 1;
455 vsnprintf(message, sizeof(message), err, cp);
456 write_crash_report(message);
457 end_packfile();
458 unkeep_all_packs();
459 dump_marks();
460 }
461 exit(128);
462 }
463
464 #ifndef SIGUSR1 /* Windows, for example */
465
466 static void set_checkpoint_signal(void)
467 {
468 }
469
470 #else
471
472 static void checkpoint_signal(int signo UNUSED)
473 {
474 checkpoint_requested = 1;
475 }
476
477 static void set_checkpoint_signal(void)
478 {
479 struct sigaction sa;
480
481 memset(&sa, 0, sizeof(sa));
482 sa.sa_handler = checkpoint_signal;
483 sigemptyset(&sa.sa_mask);
484 sa.sa_flags = SA_RESTART;
485 sigaction(SIGUSR1, &sa, NULL);
486 }
487
488 #endif
489
490 static void alloc_objects(unsigned int cnt)
491 {
492 struct object_entry_pool *b;
493
494 b = xmalloc(sizeof(struct object_entry_pool)
495 + cnt * sizeof(struct object_entry));
496 b->next_pool = blocks;
497 b->next_free = b->entries;
498 b->end = b->entries + cnt;
499 blocks = b;
500 alloc_count += cnt;
501 }
502
503 static struct object_entry *new_object(struct object_id *oid)
504 {
505 struct object_entry *e;
506
507 if (blocks->next_free == blocks->end)
508 alloc_objects(object_entry_alloc);
509
510 e = blocks->next_free++;
511 oidcpy(&e->idx.oid, oid);
512 return e;
513 }
514
515 static struct object_entry *find_object(struct object_id *oid)
516 {
517 return hashmap_get_entry_from_hash(&object_table, oidhash(oid), oid,
518 struct object_entry, ent);
519 }
520
521 static struct object_entry *insert_object(struct object_id *oid)
522 {
523 struct object_entry *e;
524 unsigned int hash = oidhash(oid);
525
526 e = hashmap_get_entry_from_hash(&object_table, hash, oid,
527 struct object_entry, ent);
528 if (!e) {
529 e = new_object(oid);
530 e->idx.offset = 0;
531 hashmap_entry_init(&e->ent, hash);
532 hashmap_add(&object_table, &e->ent);
533 }
534
535 return e;
536 }
537
538 static void invalidate_pack_id(unsigned int id)
539 {
540 unsigned long lu;
541 struct tag *t;
542 struct hashmap_iter iter;
543 struct object_entry *e;
544
545 hashmap_for_each_entry(&object_table, &iter, e, ent) {
546 if (e->pack_id == id)
547 e->pack_id = MAX_PACK_ID;
548 }
549
550 for (lu = 0; lu < branch_table_sz; lu++) {
551 struct branch *b;
552
553 for (b = branch_table[lu]; b; b = b->table_next_branch)
554 if (b->pack_id == id)
555 b->pack_id = MAX_PACK_ID;
556 }
557
558 for (t = first_tag; t; t = t->next_tag)
559 if (t->pack_id == id)
560 t->pack_id = MAX_PACK_ID;
561 }
562
563 static unsigned int hc_str(const char *s, size_t len)
564 {
565 unsigned int r = 0;
566 while (len-- > 0)
567 r = r * 31 + *s++;
568 return r;
569 }
570
571 static void insert_mark(struct mark_set **top, uintmax_t idnum, struct object_entry *oe)
572 {
573 struct mark_set *s = *top;
574
575 while ((idnum >> s->shift) >= 1024) {
576 s = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set));
577 s->shift = (*top)->shift + 10;
578 s->data.sets[0] = *top;
579 *top = s;
580 }
581 while (s->shift) {
582 uintmax_t i = idnum >> s->shift;
583 idnum -= i << s->shift;
584 if (!s->data.sets[i]) {
585 s->data.sets[i] = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set));
586 s->data.sets[i]->shift = s->shift - 10;
587 }
588 s = s->data.sets[i];
589 }
590 if (!s->data.marked[idnum])
591 marks_set_count++;
592 s->data.marked[idnum] = oe;
593 }
594
595 static void *find_mark(struct mark_set *s, uintmax_t idnum)
596 {
597 uintmax_t orig_idnum = idnum;
598 struct object_entry *oe = NULL;
599 if ((idnum >> s->shift) < 1024) {
600 while (s && s->shift) {
601 uintmax_t i = idnum >> s->shift;
602 idnum -= i << s->shift;
603 s = s->data.sets[i];
604 }
605 if (s)
606 oe = s->data.marked[idnum];
607 }
608 if (!oe)
609 die(_("mark :%" PRIuMAX " not declared"), orig_idnum);
610 return oe;
611 }
612
613 static struct atom_str *to_atom(const char *s, unsigned short len)
614 {
615 unsigned int hc = hc_str(s, len) % atom_table_sz;
616 struct atom_str *c;
617
618 for (c = atom_table[hc]; c; c = c->next_atom)
619 if (c->str_len == len && !strncmp(s, c->str_dat, len))
620 return c;
621
622 c = mem_pool_alloc(&fi_mem_pool, sizeof(struct atom_str) + len + 1);
623 c->str_len = len;
624 memcpy(c->str_dat, s, len);
625 c->str_dat[len] = 0;
626 c->next_atom = atom_table[hc];
627 atom_table[hc] = c;
628 atom_cnt++;
629 return c;
630 }
631
632 static struct branch *lookup_branch(const char *name)
633 {
634 unsigned int hc = hc_str(name, strlen(name)) % branch_table_sz;
635 struct branch *b;
636
637 for (b = branch_table[hc]; b; b = b->table_next_branch)
638 if (!strcmp(name, b->name))
639 return b;
640 return NULL;
641 }
642
643 static struct branch *new_branch(const char *name)
644 {
645 unsigned int hc = hc_str(name, strlen(name)) % branch_table_sz;
646 struct branch *b = lookup_branch(name);
647
648 if (b)
649 die(_("invalid attempt to create duplicate branch: %s"), name);
650 if (check_refname_format(name, REFNAME_ALLOW_ONELEVEL))
651 die(_("branch name doesn't conform to Git standards: %s"), name);
652
653 b = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct branch));
654 b->name = mem_pool_strdup(&fi_mem_pool, name);
655 b->table_next_branch = branch_table[hc];
656 b->branch_tree.versions[0].mode = S_IFDIR;
657 b->branch_tree.versions[1].mode = S_IFDIR;
658 b->num_notes = 0;
659 b->active = 0;
660 b->pack_id = MAX_PACK_ID;
661 branch_table[hc] = b;
662 branch_count++;
663 return b;
664 }
665
666 static unsigned int hc_entries(unsigned int cnt)
667 {
668 cnt = cnt & 7 ? (cnt / 8) + 1 : cnt / 8;
669 return cnt < avail_tree_table_sz ? cnt : avail_tree_table_sz - 1;
670 }
671
672 static struct tree_content *new_tree_content(unsigned int cnt)
673 {
674 struct avail_tree_content *f, *l = NULL;
675 struct tree_content *t;
676 unsigned int hc = hc_entries(cnt);
677
678 for (f = avail_tree_table[hc]; f; l = f, f = f->next_avail)
679 if (f->entry_capacity >= cnt)
680 break;
681
682 if (f) {
683 if (l)
684 l->next_avail = f->next_avail;
685 else
686 avail_tree_table[hc] = f->next_avail;
687 } else {
688 cnt = cnt & 7 ? ((cnt / 8) + 1) * 8 : cnt;
689 f = mem_pool_alloc(&fi_mem_pool, sizeof(*t) + sizeof(t->entries[0]) * cnt);
690 f->entry_capacity = cnt;
691 }
692
693 t = (struct tree_content*)f;
694 t->entry_count = 0;
695 t->delta_depth = 0;
696 return t;
697 }
698
699 static void release_tree_entry(struct tree_entry *e);
700 static void release_tree_content(struct tree_content *t)
701 {
702 struct avail_tree_content *f = (struct avail_tree_content*)t;
703 unsigned int hc = hc_entries(f->entry_capacity);
704 f->next_avail = avail_tree_table[hc];
705 avail_tree_table[hc] = f;
706 }
707
708 static void release_tree_content_recursive(struct tree_content *t)
709 {
710 unsigned int i;
711 for (i = 0; i < t->entry_count; i++)
712 release_tree_entry(t->entries[i]);
713 release_tree_content(t);
714 }
715
716 static struct tree_content *grow_tree_content(
717 struct tree_content *t,
718 int amt)
719 {
720 struct tree_content *r = new_tree_content(t->entry_count + amt);
721 r->entry_count = t->entry_count;
722 r->delta_depth = t->delta_depth;
723 COPY_ARRAY(r->entries, t->entries, t->entry_count);
724 release_tree_content(t);
725 return r;
726 }
727
728 static struct tree_entry *new_tree_entry(void)
729 {
730 struct tree_entry *e;
731
732 if (!avail_tree_entry) {
733 unsigned int n = tree_entry_alloc;
734 tree_entry_allocd += n * sizeof(struct tree_entry);
735 ALLOC_ARRAY(e, n);
736 avail_tree_entry = e;
737 while (n-- > 1) {
738 *((void**)e) = e + 1;
739 e++;
740 }
741 *((void**)e) = NULL;
742 }
743
744 e = avail_tree_entry;
745 avail_tree_entry = *((void**)e);
746 return e;
747 }
748
749 static void release_tree_entry(struct tree_entry *e)
750 {
751 if (e->tree)
752 release_tree_content_recursive(e->tree);
753 *((void**)e) = avail_tree_entry;
754 avail_tree_entry = e;
755 }
756
757 static struct tree_content *dup_tree_content(struct tree_content *s)
758 {
759 struct tree_content *d;
760 struct tree_entry *a, *b;
761 unsigned int i;
762
763 if (!s)
764 return NULL;
765 d = new_tree_content(s->entry_count);
766 for (i = 0; i < s->entry_count; i++) {
767 a = s->entries[i];
768 b = new_tree_entry();
769 memcpy(b, a, sizeof(*a));
770 if (a->tree && is_null_oid(&b->versions[1].oid))
771 b->tree = dup_tree_content(a->tree);
772 else
773 b->tree = NULL;
774 d->entries[i] = b;
775 }
776 d->entry_count = s->entry_count;
777 d->delta_depth = s->delta_depth;
778
779 return d;
780 }
781
782 static void start_packfile(void)
783 {
784 struct strbuf tmp_file = STRBUF_INIT;
785 struct packed_git *p;
786 int pack_fd;
787
788 pack_fd = odb_mkstemp(the_repository->objects, &tmp_file,
789 "pack/tmp_pack_XXXXXX");
790 FLEX_ALLOC_STR(p, pack_name, tmp_file.buf);
791 strbuf_release(&tmp_file);
792
793 p->pack_fd = pack_fd;
794 p->do_not_close = 1;
795 p->repo = the_repository;
796 pack_file = hashfd(the_repository->hash_algo, pack_fd, p->pack_name);
797
798 pack_data = p;
799 pack_size = write_pack_header(pack_file, 0);
800 object_count = 0;
801
802 REALLOC_ARRAY(all_packs, pack_id + 1);
803 all_packs[pack_id] = p;
804 }
805
806 static const char *create_index(void)
807 {
808 const char *tmpfile;
809 struct pack_idx_entry **idx, **c, **last;
810 struct object_entry *e;
811 struct object_entry_pool *o;
812
813 /* Build the table of object IDs. */
814 ALLOC_ARRAY(idx, object_count);
815 c = idx;
816 for (o = blocks; o; o = o->next_pool)
817 for (e = o->next_free; e-- != o->entries;)
818 if (pack_id == e->pack_id)
819 *c++ = &e->idx;
820 last = idx + object_count;
821 if (c != last)
822 die(_("internal consistency error creating the index"));
823
824 tmpfile = write_idx_file(the_repository, NULL, idx, object_count,
825 &pack_idx_opts, pack_data->hash);
826 free(idx);
827 return tmpfile;
828 }
829
830 static char *keep_pack(const char *curr_index_name)
831 {
832 static const char *keep_msg = "fast-import";
833 struct strbuf name = STRBUF_INIT;
834 int keep_fd;
835
836 odb_pack_name(pack_data->repo, &name, pack_data->hash, "keep");
837 keep_fd = safe_create_file_with_leading_directories(pack_data->repo,
838 name.buf);
839 if (keep_fd < 0)
840 die_errno(_("cannot create keep file"));
841 write_or_die(keep_fd, keep_msg, strlen(keep_msg));
842 if (close(keep_fd))
843 die_errno(_("failed to write keep file"));
844
845 odb_pack_name(pack_data->repo, &name, pack_data->hash, "pack");
846 if (finalize_object_file(pack_data->repo, pack_data->pack_name, name.buf))
847 die(_("cannot store pack file"));
848
849 odb_pack_name(pack_data->repo, &name, pack_data->hash, "idx");
850 if (finalize_object_file(pack_data->repo, curr_index_name, name.buf))
851 die(_("cannot store index file"));
852 free((void *)curr_index_name);
853 return strbuf_detach(&name, NULL);
854 }
855
856 static void unkeep_all_packs(void)
857 {
858 struct strbuf name = STRBUF_INIT;
859 int k;
860
861 for (k = 0; k < pack_id; k++) {
862 struct packed_git *p = all_packs[k];
863 odb_pack_name(p->repo, &name, p->hash, "keep");
864 unlink_or_warn(name.buf);
865 }
866 strbuf_release(&name);
867 }
868
869 static int loosen_small_pack(const struct packed_git *p)
870 {
871 struct child_process unpack = CHILD_PROCESS_INIT;
872
873 if (lseek(p->pack_fd, 0, SEEK_SET) < 0)
874 die_errno(_("failed seeking to start of '%s'"), p->pack_name);
875
876 unpack.in = p->pack_fd;
877 unpack.git_cmd = 1;
878 unpack.stdout_to_stderr = 1;
879 strvec_push(&unpack.args, "unpack-objects");
880 if (!show_stats)
881 strvec_push(&unpack.args, "-q");
882
883 return run_command(&unpack);
884 }
885
886 static void end_packfile(void)
887 {
888 static int running;
889
890 if (running || !pack_data)
891 return;
892
893 running = 1;
894 clear_delta_base_cache();
895 if (object_count) {
896 struct odb_source_files *files = odb_source_files_downcast(pack_data->repo->objects->sources);
897 struct packed_git *new_p;
898 struct object_id cur_pack_oid;
899 char *idx_name;
900 int i;
901 struct branch *b;
902 struct tag *t;
903
904 close_pack_windows(pack_data);
905 finalize_hashfile(pack_file, cur_pack_oid.hash, FSYNC_COMPONENT_PACK, 0);
906 fixup_pack_header_footer(the_hash_algo, pack_data->pack_fd,
907 pack_data->hash, pack_data->pack_name,
908 object_count, cur_pack_oid.hash,
909 pack_size);
910
911 if (object_count <= unpack_limit) {
912 if (!loosen_small_pack(pack_data)) {
913 invalidate_pack_id(pack_id);
914 goto discard_pack;
915 }
916 }
917
918 close(pack_data->pack_fd);
919 idx_name = keep_pack(create_index());
920
921 /* Register the packfile with core git's machinery. */
922 new_p = packfile_store_load_pack(files->packed, idx_name, 1);
923 if (!new_p)
924 die(_("core Git rejected index %s"), idx_name);
925 all_packs[pack_id] = new_p;
926 free(idx_name);
927
928 /* Print the boundary */
929 if (pack_edges) {
930 fprintf(pack_edges, "%s:", new_p->pack_name);
931 for (i = 0; i < branch_table_sz; i++) {
932 for (b = branch_table[i]; b; b = b->table_next_branch) {
933 if (b->pack_id == pack_id)
934 fprintf(pack_edges, " %s",
935 oid_to_hex(&b->oid));
936 }
937 }
938 for (t = first_tag; t; t = t->next_tag) {
939 if (t->pack_id == pack_id)
940 fprintf(pack_edges, " %s",
941 oid_to_hex(&t->oid));
942 }
943 fputc('\n', pack_edges);
944 fflush(pack_edges);
945 }
946
947 pack_id++;
948 }
949 else {
950 discard_pack:
951 close(pack_data->pack_fd);
952 unlink_or_warn(pack_data->pack_name);
953 }
954 FREE_AND_NULL(pack_data);
955 running = 0;
956
957 /* We can't carry a delta across packfiles. */
958 strbuf_release(&last_blob.data);
959 last_blob.offset = 0;
960 last_blob.depth = 0;
961 }
962
963 static void cycle_packfile(void)
964 {
965 end_packfile();
966 start_packfile();
967 }
968
969 static int store_object(
970 enum object_type type,
971 struct strbuf *dat,
972 struct last_object *last,
973 struct object_id *oidout,
974 uintmax_t mark)
975 {
976 struct odb_source *source;
977 void *out, *delta;
978 struct object_entry *e;
979 unsigned char hdr[96];
980 struct object_id oid;
981 unsigned long hdrlen, deltalen = 0;
982 struct git_hash_ctx c;
983 git_zstream s;
984 struct repo_config_values *cfg = repo_config_values(the_repository);
985
986 hdrlen = format_object_header((char *)hdr, sizeof(hdr), type,
987 dat->len);
988 git_hash_init(&c, the_hash_algo);
989 git_hash_update(&c, hdr, hdrlen);
990 git_hash_update(&c, dat->buf, dat->len);
991 git_hash_final_oid(&oid, &c);
992 if (oidout)
993 oidcpy(oidout, &oid);
994
995 e = insert_object(&oid);
996 if (mark)
997 insert_mark(&marks, mark, e);
998 if (e->idx.offset) {
999 duplicate_count_by_type[type]++;
1000 return 1;
1001 }
1002
1003 for (source = the_repository->objects->sources; source; source = source->next) {
1004 struct odb_source_files *files = odb_source_files_downcast(source);
1005
1006 if (!packfile_list_find_oid(packfile_store_get_packs(files->packed), &oid))
1007 continue;
1008 e->type = type;
1009 e->pack_id = MAX_PACK_ID;
1010 e->idx.offset = 1; /* just not zero! */
1011 duplicate_count_by_type[type]++;
1012 return 1;
1013 }
1014
1015 if (last && last->data.len && last->data.buf && last->depth < max_depth
1016 && dat->len > the_hash_algo->rawsz) {
1017 size_t deltalen_st;
1018
1019 delta_count_attempts_by_type[type]++;
1020 delta = diff_delta(last->data.buf, last->data.len,
1021 dat->buf, dat->len,
1022 &deltalen_st, dat->len - the_hash_algo->rawsz);
1023 deltalen = cast_size_t_to_ulong(deltalen_st);
1024 } else
1025 delta = NULL;
1026
1027 git_deflate_init(&s, cfg->pack_compression_level);
1028 if (delta) {
1029 s.next_in = delta;
1030 s.avail_in = deltalen;
1031 } else {
1032 s.next_in = (void *)dat->buf;
1033 s.avail_in = dat->len;
1034 }
1035 s.avail_out = git_deflate_bound(&s, s.avail_in);
1036 s.next_out = out = xmalloc(s.avail_out);
1037 while (git_deflate(&s, Z_FINISH) == Z_OK)
1038 ; /* nothing */
1039 git_deflate_end(&s);
1040
1041 /* Determine if we should auto-checkpoint. */
1042 if ((max_packsize
1043 && (pack_size + PACK_SIZE_THRESHOLD + s.total_out) > max_packsize)
1044 || (pack_size + PACK_SIZE_THRESHOLD + s.total_out) < pack_size) {
1045
1046 /* This new object needs to *not* have the current pack_id. */
1047 e->pack_id = pack_id + 1;
1048 cycle_packfile();
1049
1050 /* We cannot carry a delta into the new pack. */
1051 if (delta) {
1052 FREE_AND_NULL(delta);
1053
1054 git_deflate_init(&s, cfg->pack_compression_level);
1055 s.next_in = (void *)dat->buf;
1056 s.avail_in = dat->len;
1057 s.avail_out = git_deflate_bound(&s, s.avail_in);
1058 s.next_out = out = xrealloc(out, s.avail_out);
1059 while (git_deflate(&s, Z_FINISH) == Z_OK)
1060 ; /* nothing */
1061 git_deflate_end(&s);
1062 }
1063 }
1064
1065 e->type = type;
1066 e->pack_id = pack_id;
1067 e->idx.offset = pack_size;
1068 object_count++;
1069 object_count_by_type[type]++;
1070
1071 crc32_begin(pack_file);
1072
1073 if (delta) {
1074 off_t ofs = e->idx.offset - last->offset;
1075 unsigned pos = sizeof(hdr) - 1;
1076
1077 delta_count_by_type[type]++;
1078 e->depth = last->depth + 1;
1079
1080 hdrlen = encode_in_pack_object_header(hdr, sizeof(hdr),
1081 OBJ_OFS_DELTA, deltalen);
1082 hashwrite(pack_file, hdr, hdrlen);
1083 pack_size += hdrlen;
1084
1085 hdr[pos] = ofs & 127;
1086 while (ofs >>= 7)
1087 hdr[--pos] = 128 | (--ofs & 127);
1088 hashwrite(pack_file, hdr + pos, sizeof(hdr) - pos);
1089 pack_size += sizeof(hdr) - pos;
1090 } else {
1091 e->depth = 0;
1092 hdrlen = encode_in_pack_object_header(hdr, sizeof(hdr),
1093 type, dat->len);
1094 hashwrite(pack_file, hdr, hdrlen);
1095 pack_size += hdrlen;
1096 }
1097
1098 hashwrite(pack_file, out, s.total_out);
1099 pack_size += s.total_out;
1100
1101 e->idx.crc32 = crc32_end(pack_file);
1102
1103 free(out);
1104 free(delta);
1105 if (last) {
1106 if (last->no_swap) {
1107 last->data = *dat;
1108 } else {
1109 strbuf_swap(&last->data, dat);
1110 }
1111 last->offset = e->idx.offset;
1112 last->depth = e->depth;
1113 }
1114 return 0;
1115 }
1116
1117 static void truncate_pack(struct hashfile_checkpoint *checkpoint)
1118 {
1119 if (hashfile_truncate(pack_file, checkpoint))
1120 die_errno(_("cannot truncate pack to skip duplicate"));
1121 pack_size = checkpoint->offset;
1122 }
1123
1124 static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark)
1125 {
1126 size_t in_sz = 64 * 1024, out_sz = 64 * 1024;
1127 unsigned char *in_buf = xmalloc(in_sz);
1128 unsigned char *out_buf = xmalloc(out_sz);
1129 struct odb_source *source;
1130 struct object_entry *e;
1131 struct object_id oid;
1132 unsigned long hdrlen;
1133 off_t offset;
1134 struct git_hash_ctx c;
1135 git_zstream s;
1136 struct hashfile_checkpoint checkpoint;
1137 struct repo_config_values *cfg = repo_config_values(the_repository);
1138 int status = Z_OK;
1139
1140 /* Determine if we should auto-checkpoint. */
1141 if ((max_packsize
1142 && (pack_size + PACK_SIZE_THRESHOLD + len) > max_packsize)
1143 || (pack_size + PACK_SIZE_THRESHOLD + len) < pack_size)
1144 cycle_packfile();
1145
1146 hashfile_checkpoint_init(pack_file, &checkpoint);
1147 hashfile_checkpoint(pack_file, &checkpoint);
1148 offset = checkpoint.offset;
1149
1150 hdrlen = format_object_header((char *)out_buf, out_sz, OBJ_BLOB, len);
1151
1152 git_hash_init(&c, the_hash_algo);
1153 git_hash_update(&c, out_buf, hdrlen);
1154
1155 crc32_begin(pack_file);
1156
1157 git_deflate_init(&s, cfg->pack_compression_level);
1158
1159 hdrlen = encode_in_pack_object_header(out_buf, out_sz, OBJ_BLOB, len);
1160
1161 s.next_out = out_buf + hdrlen;
1162 s.avail_out = out_sz - hdrlen;
1163
1164 while (status != Z_STREAM_END) {
1165 if (0 < len && !s.avail_in) {
1166 size_t cnt = in_sz < len ? in_sz : (size_t)len;
1167 size_t n = fread(in_buf, 1, cnt, stdin);
1168 if (!n && feof(stdin))
1169 die(_("EOF in data (%" PRIuMAX " bytes remaining)"), len);
1170
1171 git_hash_update(&c, in_buf, n);
1172 s.next_in = in_buf;
1173 s.avail_in = n;
1174 len -= n;
1175 }
1176
1177 status = git_deflate(&s, len ? 0 : Z_FINISH);
1178
1179 if (!s.avail_out || status == Z_STREAM_END) {
1180 size_t n = s.next_out - out_buf;
1181 hashwrite(pack_file, out_buf, n);
1182 pack_size += n;
1183 s.next_out = out_buf;
1184 s.avail_out = out_sz;
1185 }
1186
1187 switch (status) {
1188 case Z_OK:
1189 case Z_BUF_ERROR:
1190 case Z_STREAM_END:
1191 continue;
1192 default:
1193 die(_("unexpected deflate failure: %d"), status);
1194 }
1195 }
1196 git_deflate_end(&s);
1197 git_hash_final_oid(&oid, &c);
1198
1199 if (oidout)
1200 oidcpy(oidout, &oid);
1201
1202 e = insert_object(&oid);
1203
1204 if (mark)
1205 insert_mark(&marks, mark, e);
1206
1207 if (e->idx.offset) {
1208 duplicate_count_by_type[OBJ_BLOB]++;
1209 truncate_pack(&checkpoint);
1210 goto out;
1211 }
1212
1213 for (source = the_repository->objects->sources; source; source = source->next) {
1214 struct odb_source_files *files = odb_source_files_downcast(source);
1215
1216 if (!packfile_list_find_oid(packfile_store_get_packs(files->packed), &oid))
1217 continue;
1218 e->type = OBJ_BLOB;
1219 e->pack_id = MAX_PACK_ID;
1220 e->idx.offset = 1; /* just not zero! */
1221 duplicate_count_by_type[OBJ_BLOB]++;
1222 truncate_pack(&checkpoint);
1223 goto out;
1224 }
1225
1226 e->depth = 0;
1227 e->type = OBJ_BLOB;
1228 e->pack_id = pack_id;
1229 e->idx.offset = offset;
1230 e->idx.crc32 = crc32_end(pack_file);
1231 object_count++;
1232 object_count_by_type[OBJ_BLOB]++;
1233
1234 out:
1235 free(in_buf);
1236 free(out_buf);
1237 hashfile_checkpoint_release(&checkpoint);
1238 }
1239
1240 /* All calls must be guarded by find_object() or find_mark() to
1241 * ensure the 'struct object_entry' passed was written by this
1242 * process instance. We unpack the entry by the offset, avoiding
1243 * the need for the corresponding .idx file. This unpacking rule
1244 * works because we only use OBJ_REF_DELTA within the packfiles
1245 * created by fast-import.
1246 *
1247 * oe must not be NULL. Such an oe usually comes from giving
1248 * an unknown SHA-1 to find_object() or an undefined mark to
1249 * find_mark(). Callers must test for this condition and use
1250 * the standard read_sha1_file() when it happens.
1251 *
1252 * oe->pack_id must not be MAX_PACK_ID. Such an oe is usually from
1253 * find_mark(), where the mark was reloaded from an existing marks
1254 * file and is referencing an object that this fast-import process
1255 * instance did not write out to a packfile. Callers must test for
1256 * this condition and use read_sha1_file() instead.
1257 */
1258 static void *gfi_unpack_entry(
1259 struct object_entry *oe,
1260 unsigned long *sizep)
1261 {
1262 enum object_type type;
1263 size_t size_st = 0;
1264 void *data;
1265 struct packed_git *p = all_packs[oe->pack_id];
1266 if (p == pack_data && p->pack_size < (pack_size + the_hash_algo->rawsz)) {
1267 /* The object is stored in the packfile we are writing to
1268 * and we have modified it since the last time we scanned
1269 * back to read a previously written object. If an old
1270 * window covered [p->pack_size, p->pack_size + rawsz) its
1271 * data is stale and is not valid. Closing all windows
1272 * and updating the packfile length ensures we can read
1273 * the newly written data.
1274 */
1275 close_pack_windows(p);
1276 hashflush(pack_file);
1277
1278 /* We have to offer rawsz bytes additional on the end of
1279 * the packfile as the core unpacker code assumes the
1280 * footer is present at the file end and must promise
1281 * at least rawsz bytes within any window it maps. But
1282 * we don't actually create the footer here.
1283 */
1284 p->pack_size = pack_size + the_hash_algo->rawsz;
1285 }
1286 data = unpack_entry(the_repository, p, oe->idx.offset, &type, &size_st);
1287 if (sizep)
1288 *sizep = cast_size_t_to_ulong(size_st);
1289 return data;
1290 }
1291
1292 static void load_tree(struct tree_entry *root)
1293 {
1294 struct object_id *oid = &root->versions[1].oid;
1295 struct object_entry *myoe;
1296 struct tree_content *t;
1297 unsigned long size;
1298 char *buf;
1299 const char *c;
1300
1301 root->tree = t = new_tree_content(8);
1302 if (is_null_oid(oid))
1303 return;
1304
1305 myoe = find_object(oid);
1306 if (myoe && myoe->pack_id != MAX_PACK_ID) {
1307 if (myoe->type != OBJ_TREE)
1308 die(_("not a tree: %s"), oid_to_hex(oid));
1309 t->delta_depth = myoe->depth;
1310 buf = gfi_unpack_entry(myoe, &size);
1311 if (!buf)
1312 die(_("can't load tree %s"), oid_to_hex(oid));
1313 } else {
1314 enum object_type type;
1315 size_t size_st = 0;
1316 buf = odb_read_object(the_repository->objects, oid, &type,
1317 &size_st);
1318 size = cast_size_t_to_ulong(size_st);
1319 if (!buf || type != OBJ_TREE)
1320 die(_("can't load tree %s"), oid_to_hex(oid));
1321 }
1322
1323 c = buf;
1324 while (c != (buf + size)) {
1325 struct tree_entry *e = new_tree_entry();
1326
1327 if (t->entry_count == t->entry_capacity)
1328 root->tree = t = grow_tree_content(t, t->entry_count);
1329 t->entries[t->entry_count++] = e;
1330
1331 e->tree = NULL;
1332 c = parse_mode(c, &e->versions[1].mode);
1333 if (!c)
1334 die(_("corrupt mode in %s"), oid_to_hex(oid));
1335 e->versions[0].mode = e->versions[1].mode;
1336 e->name = to_atom(c, strlen(c));
1337 c += e->name->str_len + 1;
1338 oidread(&e->versions[0].oid, (unsigned char *)c,
1339 the_repository->hash_algo);
1340 oidread(&e->versions[1].oid, (unsigned char *)c,
1341 the_repository->hash_algo);
1342 c += the_hash_algo->rawsz;
1343 }
1344 free(buf);
1345 }
1346
1347 static int tecmp0 (const void *_a, const void *_b)
1348 {
1349 struct tree_entry *a = *((struct tree_entry**)_a);
1350 struct tree_entry *b = *((struct tree_entry**)_b);
1351 return base_name_compare(
1352 a->name->str_dat, a->name->str_len, a->versions[0].mode,
1353 b->name->str_dat, b->name->str_len, b->versions[0].mode);
1354 }
1355
1356 static int tecmp1 (const void *_a, const void *_b)
1357 {
1358 struct tree_entry *a = *((struct tree_entry**)_a);
1359 struct tree_entry *b = *((struct tree_entry**)_b);
1360 return base_name_compare(
1361 a->name->str_dat, a->name->str_len, a->versions[1].mode,
1362 b->name->str_dat, b->name->str_len, b->versions[1].mode);
1363 }
1364
1365 static void mktree(struct tree_content *t, int v, struct strbuf *b)
1366 {
1367 size_t maxlen = 0;
1368 unsigned int i;
1369
1370 if (!v)
1371 QSORT(t->entries, t->entry_count, tecmp0);
1372 else
1373 QSORT(t->entries, t->entry_count, tecmp1);
1374
1375 for (i = 0; i < t->entry_count; i++) {
1376 if (t->entries[i]->versions[v].mode)
1377 maxlen += t->entries[i]->name->str_len + 34;
1378 }
1379
1380 strbuf_reset(b);
1381 strbuf_grow(b, maxlen);
1382 for (i = 0; i < t->entry_count; i++) {
1383 struct tree_entry *e = t->entries[i];
1384 if (!e->versions[v].mode)
1385 continue;
1386 strbuf_addf(b, "%o %s%c",
1387 (unsigned int)(e->versions[v].mode & ~NO_DELTA),
1388 e->name->str_dat, '\0');
1389 strbuf_add(b, e->versions[v].oid.hash, the_hash_algo->rawsz);
1390 }
1391 }
1392
1393 static void store_tree(struct tree_entry *root)
1394 {
1395 struct tree_content *t;
1396 unsigned int i, j, del;
1397 struct last_object lo = { STRBUF_INIT, 0, 0, /* no_swap */ 1 };
1398 struct object_entry *le = NULL;
1399
1400 if (!is_null_oid(&root->versions[1].oid))
1401 return;
1402
1403 if (!root->tree)
1404 load_tree(root);
1405 t = root->tree;
1406
1407 for (i = 0; i < t->entry_count; i++) {
1408 if (t->entries[i]->tree)
1409 store_tree(t->entries[i]);
1410 }
1411
1412 if (!(root->versions[0].mode & NO_DELTA))
1413 le = find_object(&root->versions[0].oid);
1414 if (S_ISDIR(root->versions[0].mode) && le && le->pack_id == pack_id) {
1415 mktree(t, 0, &old_tree);
1416 lo.data = old_tree;
1417 lo.offset = le->idx.offset;
1418 lo.depth = t->delta_depth;
1419 }
1420
1421 mktree(t, 1, &new_tree);
1422 store_object(OBJ_TREE, &new_tree, &lo, &root->versions[1].oid, 0);
1423
1424 t->delta_depth = lo.depth;
1425 for (i = 0, j = 0, del = 0; i < t->entry_count; i++) {
1426 struct tree_entry *e = t->entries[i];
1427 if (e->versions[1].mode) {
1428 e->versions[0].mode = e->versions[1].mode;
1429 oidcpy(&e->versions[0].oid, &e->versions[1].oid);
1430 t->entries[j++] = e;
1431 } else {
1432 release_tree_entry(e);
1433 del++;
1434 }
1435 }
1436 t->entry_count -= del;
1437 }
1438
1439 static void tree_content_replace(
1440 struct tree_entry *root,
1441 const struct object_id *oid,
1442 const uint16_t mode,
1443 struct tree_content *newtree)
1444 {
1445 if (!S_ISDIR(mode))
1446 die(_("root cannot be a non-directory"));
1447 oidclr(&root->versions[0].oid, the_repository->hash_algo);
1448 oidcpy(&root->versions[1].oid, oid);
1449 if (root->tree)
1450 release_tree_content_recursive(root->tree);
1451 root->tree = newtree;
1452 }
1453
1454 static int tree_content_set(
1455 struct tree_entry *root,
1456 const char *p,
1457 const struct object_id *oid,
1458 const uint16_t mode,
1459 struct tree_content *subtree)
1460 {
1461 struct tree_content *t;
1462 const char *slash1;
1463 unsigned int i, n;
1464 struct tree_entry *e;
1465
1466 slash1 = strchrnul(p, '/');
1467 n = slash1 - p;
1468 if (!n)
1469 die(_("empty path component found in input"));
1470 if (!*slash1 && !S_ISDIR(mode) && subtree)
1471 die(_("non-directories cannot have subtrees"));
1472
1473 if (!root->tree)
1474 load_tree(root);
1475 t = root->tree;
1476 for (i = 0; i < t->entry_count; i++) {
1477 e = t->entries[i];
1478 if (e->name->str_len == n && !fspathncmp(p, e->name->str_dat, n)) {
1479 if (!*slash1) {
1480 if (!S_ISDIR(mode)
1481 && e->versions[1].mode == mode
1482 && oideq(&e->versions[1].oid, oid))
1483 return 0;
1484 e->versions[1].mode = mode;
1485 oidcpy(&e->versions[1].oid, oid);
1486 if (e->tree)
1487 release_tree_content_recursive(e->tree);
1488 e->tree = subtree;
1489
1490 /*
1491 * We need to leave e->versions[0].sha1 alone
1492 * to avoid modifying the preimage tree used
1493 * when writing out the parent directory.
1494 * But after replacing the subdir with a
1495 * completely different one, it's not a good
1496 * delta base any more, and besides, we've
1497 * thrown away the tree entries needed to
1498 * make a delta against it.
1499 *
1500 * So let's just explicitly disable deltas
1501 * for the subtree.
1502 */
1503 if (S_ISDIR(e->versions[0].mode))
1504 e->versions[0].mode |= NO_DELTA;
1505
1506 oidclr(&root->versions[1].oid, the_repository->hash_algo);
1507 return 1;
1508 }
1509 if (!S_ISDIR(e->versions[1].mode)) {
1510 e->tree = new_tree_content(8);
1511 e->versions[1].mode = S_IFDIR;
1512 }
1513 if (!e->tree)
1514 load_tree(e);
1515 if (tree_content_set(e, slash1 + 1, oid, mode, subtree)) {
1516 oidclr(&root->versions[1].oid, the_repository->hash_algo);
1517 return 1;
1518 }
1519 return 0;
1520 }
1521 }
1522
1523 if (t->entry_count == t->entry_capacity)
1524 root->tree = t = grow_tree_content(t, t->entry_count);
1525 e = new_tree_entry();
1526 e->name = to_atom(p, n);
1527 e->versions[0].mode = 0;
1528 oidclr(&e->versions[0].oid, the_repository->hash_algo);
1529 t->entries[t->entry_count++] = e;
1530 if (*slash1) {
1531 e->tree = new_tree_content(8);
1532 e->versions[1].mode = S_IFDIR;
1533 tree_content_set(e, slash1 + 1, oid, mode, subtree);
1534 } else {
1535 e->tree = subtree;
1536 e->versions[1].mode = mode;
1537 oidcpy(&e->versions[1].oid, oid);
1538 }
1539 oidclr(&root->versions[1].oid, the_repository->hash_algo);
1540 return 1;
1541 }
1542
1543 static int tree_content_remove(
1544 struct tree_entry *root,
1545 const char *p,
1546 struct tree_entry *backup_leaf,
1547 int allow_root)
1548 {
1549 struct tree_content *t;
1550 const char *slash1;
1551 unsigned int i, n;
1552 struct tree_entry *e;
1553
1554 slash1 = strchrnul(p, '/');
1555 n = slash1 - p;
1556
1557 if (!root->tree)
1558 load_tree(root);
1559
1560 if (!*p && allow_root) {
1561 e = root;
1562 goto del_entry;
1563 }
1564
1565 t = root->tree;
1566 for (i = 0; i < t->entry_count; i++) {
1567 e = t->entries[i];
1568 if (e->name->str_len == n && !fspathncmp(p, e->name->str_dat, n)) {
1569 if (*slash1 && !S_ISDIR(e->versions[1].mode))
1570 /*
1571 * If p names a file in some subdirectory, and a
1572 * file or symlink matching the name of the
1573 * parent directory of p exists, then p cannot
1574 * exist and need not be deleted.
1575 */
1576 return 1;
1577 if (!*slash1 || !S_ISDIR(e->versions[1].mode))
1578 goto del_entry;
1579 if (!e->tree)
1580 load_tree(e);
1581 if (tree_content_remove(e, slash1 + 1, backup_leaf, 0)) {
1582 for (n = 0; n < e->tree->entry_count; n++) {
1583 if (e->tree->entries[n]->versions[1].mode) {
1584 oidclr(&root->versions[1].oid,
1585 the_repository->hash_algo);
1586 return 1;
1587 }
1588 }
1589 backup_leaf = NULL;
1590 goto del_entry;
1591 }
1592 return 0;
1593 }
1594 }
1595 return 0;
1596
1597 del_entry:
1598 if (backup_leaf)
1599 memcpy(backup_leaf, e, sizeof(*backup_leaf));
1600 else if (e->tree)
1601 release_tree_content_recursive(e->tree);
1602 e->tree = NULL;
1603 e->versions[1].mode = 0;
1604 oidclr(&e->versions[1].oid, the_repository->hash_algo);
1605 oidclr(&root->versions[1].oid, the_repository->hash_algo);
1606 return 1;
1607 }
1608
1609 static int tree_content_get(
1610 struct tree_entry *root,
1611 const char *p,
1612 struct tree_entry *leaf,
1613 int allow_root)
1614 {
1615 struct tree_content *t;
1616 const char *slash1;
1617 unsigned int i, n;
1618 struct tree_entry *e;
1619
1620 slash1 = strchrnul(p, '/');
1621 n = slash1 - p;
1622 if (!n && !allow_root)
1623 die(_("empty path component found in input"));
1624
1625 if (!root->tree)
1626 load_tree(root);
1627
1628 if (!n) {
1629 e = root;
1630 goto found_entry;
1631 }
1632
1633 t = root->tree;
1634 for (i = 0; i < t->entry_count; i++) {
1635 e = t->entries[i];
1636 if (e->name->str_len == n && !fspathncmp(p, e->name->str_dat, n)) {
1637 if (!*slash1)
1638 goto found_entry;
1639 if (!S_ISDIR(e->versions[1].mode))
1640 return 0;
1641 if (!e->tree)
1642 load_tree(e);
1643 return tree_content_get(e, slash1 + 1, leaf, 0);
1644 }
1645 }
1646 return 0;
1647
1648 found_entry:
1649 memcpy(leaf, e, sizeof(*leaf));
1650 if (e->tree && is_null_oid(&e->versions[1].oid))
1651 leaf->tree = dup_tree_content(e->tree);
1652 else
1653 leaf->tree = NULL;
1654 return 1;
1655 }
1656
1657 static int update_branch(struct branch *b)
1658 {
1659 static const char *msg = "fast-import";
1660 struct ref_transaction *transaction;
1661 struct object_id old_oid;
1662 struct strbuf err = STRBUF_INIT;
1663 static const char *replace_prefix = "refs/replace/";
1664
1665 if (starts_with(b->name, replace_prefix) &&
1666 !strcmp(b->name + strlen(replace_prefix),
1667 oid_to_hex(&b->oid))) {
1668 if (!quiet)
1669 warning(_("dropping %s since it would point to "
1670 "itself (i.e. to %s)"),
1671 b->name, oid_to_hex(&b->oid));
1672 refs_delete_ref(get_main_ref_store(the_repository),
1673 NULL, b->name, NULL, 0);
1674 return 0;
1675 }
1676 if (is_null_oid(&b->oid)) {
1677 if (b->delete)
1678 refs_delete_ref(get_main_ref_store(the_repository),
1679 NULL, b->name, NULL, 0);
1680 return 0;
1681 }
1682 if (refs_read_ref(get_main_ref_store(the_repository), b->name, &old_oid))
1683 oidclr(&old_oid, the_repository->hash_algo);
1684 if (!force_update && !is_null_oid(&old_oid)) {
1685 struct commit *old_cmit, *new_cmit;
1686 int ret;
1687
1688 old_cmit = lookup_commit_reference_gently(the_repository,
1689 &old_oid, 0);
1690 new_cmit = lookup_commit_reference_gently(the_repository,
1691 &b->oid, 0);
1692 if (!old_cmit || !new_cmit)
1693 return error(_("branch %s is missing commits."), b->name);
1694
1695 ret = repo_in_merge_bases(the_repository, old_cmit, new_cmit);
1696 if (ret < 0)
1697 exit(128);
1698 if (!ret) {
1699 warning(_("not updating %s"
1700 " (new tip %s does not contain %s)"),
1701 b->name, oid_to_hex(&b->oid),
1702 oid_to_hex(&old_oid));
1703 return -1;
1704 }
1705 }
1706 transaction = ref_store_transaction_begin(get_main_ref_store(the_repository),
1707 0, &err);
1708 if (!transaction ||
1709 ref_transaction_update(transaction, b->name, &b->oid, &old_oid,
1710 NULL, NULL, 0, msg, &err) ||
1711 ref_transaction_commit(transaction, &err)) {
1712 ref_transaction_free(transaction);
1713 error("%s", err.buf);
1714 strbuf_release(&err);
1715 return -1;
1716 }
1717 ref_transaction_free(transaction);
1718 strbuf_release(&err);
1719 return 0;
1720 }
1721
1722 static void dump_branches(void)
1723 {
1724 unsigned int i;
1725 struct branch *b;
1726
1727 for (i = 0; i < branch_table_sz; i++) {
1728 for (b = branch_table[i]; b; b = b->table_next_branch)
1729 failure |= update_branch(b);
1730 }
1731 }
1732
1733 static void dump_tags(void)
1734 {
1735 static const char *msg = "fast-import";
1736 struct tag *t;
1737 struct strbuf ref_name = STRBUF_INIT;
1738 struct strbuf err = STRBUF_INIT;
1739 struct ref_transaction *transaction;
1740
1741 transaction = ref_store_transaction_begin(get_main_ref_store(the_repository),
1742 0, &err);
1743 if (!transaction) {
1744 failure |= error("%s", err.buf);
1745 goto cleanup;
1746 }
1747 for (t = first_tag; t; t = t->next_tag) {
1748 strbuf_reset(&ref_name);
1749 strbuf_addf(&ref_name, "refs/tags/%s", t->name);
1750
1751 if (ref_transaction_update(transaction, ref_name.buf,
1752 &t->oid, NULL, NULL, NULL,
1753 0, msg, &err)) {
1754 failure |= error("%s", err.buf);
1755 goto cleanup;
1756 }
1757 }
1758 if (ref_transaction_commit(transaction, &err))
1759 failure |= error("%s", err.buf);
1760
1761 cleanup:
1762 ref_transaction_free(transaction);
1763 strbuf_release(&ref_name);
1764 strbuf_release(&err);
1765 }
1766
1767 static void dump_marks(void)
1768 {
1769 struct lock_file mark_lock = LOCK_INIT;
1770 FILE *f;
1771
1772 if (!export_marks_file || (import_marks_file && !import_marks_file_done))
1773 return;
1774
1775 if (safe_create_leading_directories_const(the_repository, export_marks_file)) {
1776 failure |= error_errno(_("unable to create leading directories of %s"),
1777 export_marks_file);
1778 return;
1779 }
1780
1781 if (hold_lock_file_for_update(&mark_lock, export_marks_file, 0) < 0) {
1782 failure |= error_errno(_("unable to write marks file %s"),
1783 export_marks_file);
1784 return;
1785 }
1786
1787 f = fdopen_lock_file(&mark_lock, "w");
1788 if (!f) {
1789 int saved_errno = errno;
1790 rollback_lock_file(&mark_lock);
1791 failure |= error(_("unable to write marks file %s: %s"),
1792 export_marks_file, strerror(saved_errno));
1793 return;
1794 }
1795
1796 for_each_mark(marks, 0, dump_marks_fn, f);
1797 if (commit_lock_file(&mark_lock)) {
1798 failure |= error_errno(_("unable to write file %s"),
1799 export_marks_file);
1800 return;
1801 }
1802 }
1803
1804 static void insert_object_entry(struct mark_set **s, struct object_id *oid, uintmax_t mark)
1805 {
1806 struct object_entry *e;
1807 e = find_object(oid);
1808 if (!e) {
1809 enum object_type type = odb_read_object_info(the_repository->objects,
1810 oid, NULL);
1811 if (type < 0)
1812 die(_("object not found: %s"), oid_to_hex(oid));
1813 e = insert_object(oid);
1814 e->type = type;
1815 e->pack_id = MAX_PACK_ID;
1816 e->idx.offset = 1; /* just not zero! */
1817 }
1818 insert_mark(s, mark, e);
1819 }
1820
1821 static void insert_oid_entry(struct mark_set **s, struct object_id *oid, uintmax_t mark)
1822 {
1823 insert_mark(s, mark, xmemdupz(oid, sizeof(*oid)));
1824 }
1825
1826 static void read_mark_file(struct mark_set **s, FILE *f, mark_set_inserter_t inserter)
1827 {
1828 char line[512];
1829 while (fgets(line, sizeof(line), f)) {
1830 uintmax_t mark;
1831 char *end;
1832 struct object_id oid;
1833
1834 /* Ensure SHA-1 objects are padded with zeros. */
1835 memset(oid.hash, 0, sizeof(oid.hash));
1836
1837 end = strchr(line, '\n');
1838 if (line[0] != ':' || !end)
1839 die(_("corrupt mark line: %s"), line);
1840 *end = 0;
1841 mark = strtoumax(line + 1, &end, 10);
1842 if (!mark || end == line + 1
1843 || *end != ' '
1844 || get_oid_hex_any(end + 1, &oid) == GIT_HASH_UNKNOWN)
1845 die(_("corrupt mark line: %s"), line);
1846 inserter(s, &oid, mark);
1847 }
1848 }
1849
1850 static void read_marks(void)
1851 {
1852 FILE *f = fopen(import_marks_file, "r");
1853 if (f)
1854 ;
1855 else if (import_marks_file_ignore_missing && errno == ENOENT)
1856 goto done; /* Marks file does not exist */
1857 else
1858 die_errno(_("cannot read '%s'"), import_marks_file);
1859 read_mark_file(&marks, f, insert_object_entry);
1860 fclose(f);
1861 done:
1862 import_marks_file_done = 1;
1863 }
1864
1865
1866 static int read_next_command(struct fast_import_state *state)
1867 {
1868 static int stdin_eof = 0;
1869
1870 if (stdin_eof) {
1871 unread_command_buf = 0;
1872 return EOF;
1873 }
1874
1875 for (;;) {
1876 if (unread_command_buf) {
1877 unread_command_buf = 0;
1878 } else {
1879 struct recent_command *rc;
1880
1881 stdin_eof = strbuf_getline_lf(&command_buf, stdin);
1882 if (stdin_eof)
1883 return EOF;
1884
1885 if (!state->seen_data_command
1886 && !starts_with(command_buf.buf, "feature ")
1887 && !starts_with(command_buf.buf, "option ")) {
1888 parse_argv(state);
1889 }
1890
1891 rc = rc_free;
1892 if (rc)
1893 rc_free = rc->next;
1894 else {
1895 rc = cmd_hist.next;
1896 cmd_hist.next = rc->next;
1897 cmd_hist.next->prev = &cmd_hist;
1898 free(rc->buf);
1899 }
1900
1901 rc->buf = xstrdup(command_buf.buf);
1902 rc->prev = cmd_tail;
1903 rc->next = cmd_hist.prev;
1904 rc->prev->next = rc;
1905 cmd_tail = rc;
1906 }
1907 if (command_buf.buf[0] == '#')
1908 continue;
1909 return 0;
1910 }
1911 }
1912
1913 static void skip_optional_lf(void)
1914 {
1915 int term_char = fgetc(stdin);
1916 if (term_char != '\n' && term_char != EOF)
1917 ungetc(term_char, stdin);
1918 }
1919
1920 static void parse_mark(struct fast_import_state *state)
1921 {
1922 const char *v;
1923 if (skip_prefix(command_buf.buf, "mark :", &v)) {
1924 next_mark = strtoumax(v, NULL, 10);
1925 read_next_command(state);
1926 }
1927 else
1928 next_mark = 0;
1929 }
1930
1931 static void parse_original_identifier(struct fast_import_state *state)
1932 {
1933 const char *v;
1934 if (skip_prefix(command_buf.buf, "original-oid ", &v))
1935 read_next_command(state);
1936 }
1937
1938 static int parse_data(struct strbuf *sb, uintmax_t limit, uintmax_t *len_res)
1939 {
1940 const char *data;
1941 strbuf_reset(sb);
1942
1943 if (!skip_prefix(command_buf.buf, "data ", &data))
1944 die(_("expected 'data n' command, found: %s"), command_buf.buf);
1945
1946 if (skip_prefix(data, "<<", &data)) {
1947 char *term = xstrdup(data);
1948 size_t term_len = command_buf.len - (data - command_buf.buf);
1949
1950 for (;;) {
1951 if (strbuf_getline_lf(&command_buf, stdin) == EOF)
1952 die(_("EOF in data (terminator '%s' not found)"), term);
1953 if (term_len == command_buf.len
1954 && !strcmp(term, command_buf.buf))
1955 break;
1956 strbuf_addbuf(sb, &command_buf);
1957 strbuf_addch(sb, '\n');
1958 }
1959 free(term);
1960 }
1961 else {
1962 uintmax_t len = strtoumax(data, NULL, 10);
1963 size_t n = 0, length = (size_t)len;
1964
1965 if (limit && limit < len) {
1966 *len_res = len;
1967 return 0;
1968 }
1969 if (length < len)
1970 die(_("data is too large to use in this context"));
1971
1972 while (n < length) {
1973 size_t s = strbuf_fread(sb, length - n, stdin);
1974 if (!s && feof(stdin))
1975 die(_("EOF in data (%lu bytes remaining)"),
1976 (unsigned long)(length - n));
1977 n += s;
1978 }
1979 }
1980
1981 skip_optional_lf();
1982 return 1;
1983 }
1984
1985 static int validate_raw_date(const char *src, struct strbuf *result, int strict)
1986 {
1987 const char *orig_src = src;
1988 char *endp;
1989 unsigned long num;
1990
1991 errno = 0;
1992
1993 num = strtoul(src, &endp, 10);
1994 /*
1995 * NEEDSWORK: perhaps check for reasonable values? For example, we
1996 * could error on values representing times more than a
1997 * day in the future.
1998 */
1999 if (errno || endp == src || *endp != ' ')
2000 return -1;
2001
2002 src = endp + 1;
2003 if (*src != '-' && *src != '+')
2004 return -1;
2005
2006 num = strtoul(src + 1, &endp, 10);
2007 /*
2008 * NEEDSWORK: check for brokenness other than num > 1400, such as
2009 * (num % 100) >= 60, or ((num % 100) % 15) != 0 ?
2010 */
2011 if (errno || endp == src + 1 || *endp || /* did not parse */
2012 (strict && (1400 < num)) /* parsed a broken timezone */
2013 )
2014 return -1;
2015
2016 strbuf_addstr(result, orig_src);
2017 return 0;
2018 }
2019
2020 static char *parse_ident(const char *buf)
2021 {
2022 const char *ltgt;
2023 size_t name_len;
2024 struct strbuf ident = STRBUF_INIT;
2025
2026 /* ensure there is a space delimiter even if there is no name */
2027 if (*buf == '<')
2028 --buf;
2029
2030 ltgt = buf + strcspn(buf, "<>");
2031 if (*ltgt != '<')
2032 die(_("missing < in ident string: %s"), buf);
2033 if (ltgt != buf && ltgt[-1] != ' ')
2034 die(_("missing space before < in ident string: %s"), buf);
2035 ltgt = ltgt + 1 + strcspn(ltgt + 1, "<>");
2036 if (*ltgt != '>')
2037 die(_("missing > in ident string: %s"), buf);
2038 ltgt++;
2039 if (*ltgt != ' ')
2040 die(_("missing space after > in ident string: %s"), buf);
2041 ltgt++;
2042 name_len = ltgt - buf;
2043 strbuf_add(&ident, buf, name_len);
2044
2045 switch (whenspec) {
2046 case WHENSPEC_RAW:
2047 if (validate_raw_date(ltgt, &ident, 1) < 0)
2048 die(_("invalid raw date \"%s\" in ident: %s"), ltgt, buf);
2049 break;
2050 case WHENSPEC_RAW_PERMISSIVE:
2051 if (validate_raw_date(ltgt, &ident, 0) < 0)
2052 die(_("invalid raw date \"%s\" in ident: %s"), ltgt, buf);
2053 break;
2054 case WHENSPEC_RFC2822:
2055 if (parse_date(ltgt, &ident) < 0)
2056 die(_("invalid rfc2822 date \"%s\" in ident: %s"), ltgt, buf);
2057 break;
2058 case WHENSPEC_NOW:
2059 if (strcmp("now", ltgt))
2060 die(_("date in ident must be 'now': %s"), buf);
2061 datestamp(&ident);
2062 break;
2063 }
2064
2065 return strbuf_detach(&ident, NULL);
2066 }
2067
2068 static void parse_and_store_blob(
2069 struct last_object *last,
2070 struct object_id *oidout,
2071 uintmax_t mark)
2072 {
2073 static struct strbuf buf = STRBUF_INIT;
2074 uintmax_t len;
2075
2076 if (parse_data(&buf, repo_settings_get_big_file_threshold(the_repository), &len))
2077 store_object(OBJ_BLOB, &buf, last, oidout, mark);
2078 else {
2079 if (last) {
2080 strbuf_release(&last->data);
2081 last->offset = 0;
2082 last->depth = 0;
2083 }
2084 stream_blob(len, oidout, mark);
2085 skip_optional_lf();
2086 }
2087 }
2088
2089 static void parse_new_blob(struct fast_import_state *state)
2090 {
2091 read_next_command(state);
2092 parse_mark(state);
2093 parse_original_identifier(state);
2094 parse_and_store_blob(&last_blob, NULL, next_mark);
2095 }
2096
2097 static void unload_one_branch(void)
2098 {
2099 while (cur_active_branches
2100 && cur_active_branches >= max_active_branches) {
2101 uintmax_t min_commit = ULONG_MAX;
2102 struct branch *e, *l = NULL, *p = NULL;
2103
2104 for (e = active_branches; e; e = e->active_next_branch) {
2105 if (e->last_commit < min_commit) {
2106 p = l;
2107 min_commit = e->last_commit;
2108 }
2109 l = e;
2110 }
2111
2112 if (p) {
2113 e = p->active_next_branch;
2114 p->active_next_branch = e->active_next_branch;
2115 } else {
2116 e = active_branches;
2117 active_branches = e->active_next_branch;
2118 }
2119 e->active = 0;
2120 e->active_next_branch = NULL;
2121 if (e->branch_tree.tree) {
2122 release_tree_content_recursive(e->branch_tree.tree);
2123 e->branch_tree.tree = NULL;
2124 }
2125 cur_active_branches--;
2126 }
2127 }
2128
2129 static void load_branch(struct branch *b)
2130 {
2131 load_tree(&b->branch_tree);
2132 if (!b->active) {
2133 b->active = 1;
2134 b->active_next_branch = active_branches;
2135 active_branches = b;
2136 cur_active_branches++;
2137 branch_load_count++;
2138 }
2139 }
2140
2141 static unsigned char convert_num_notes_to_fanout(uintmax_t num_notes)
2142 {
2143 unsigned char fanout = 0;
2144 while ((num_notes >>= 8))
2145 fanout++;
2146 return fanout;
2147 }
2148
2149 static void construct_path_with_fanout(const char *hex_sha1,
2150 unsigned char fanout, char *path)
2151 {
2152 unsigned int i = 0, j = 0;
2153 if (fanout >= the_hash_algo->rawsz)
2154 die(_("too large fanout (%u)"), fanout);
2155 while (fanout) {
2156 path[i++] = hex_sha1[j++];
2157 path[i++] = hex_sha1[j++];
2158 path[i++] = '/';
2159 fanout--;
2160 }
2161 memcpy(path + i, hex_sha1 + j, the_hash_algo->hexsz - j);
2162 path[i + the_hash_algo->hexsz - j] = '\0';
2163 }
2164
2165 static uintmax_t do_change_note_fanout(
2166 struct tree_entry *orig_root, struct tree_entry *root,
2167 char *hex_oid, unsigned int hex_oid_len,
2168 char *fullpath, unsigned int fullpath_len,
2169 unsigned char fanout)
2170 {
2171 struct tree_content *t;
2172 struct tree_entry *e, leaf;
2173 unsigned int i, tmp_hex_oid_len, tmp_fullpath_len;
2174 uintmax_t num_notes = 0;
2175 struct object_id oid;
2176 /* hex oid + '/' between each pair of hex digits + NUL */
2177 char realpath[GIT_MAX_HEXSZ + ((GIT_MAX_HEXSZ / 2) - 1) + 1];
2178 const unsigned hexsz = the_hash_algo->hexsz;
2179
2180 if (!root->tree)
2181 load_tree(root);
2182 t = root->tree;
2183
2184 for (i = 0; t && i < t->entry_count; i++) {
2185 e = t->entries[i];
2186 tmp_hex_oid_len = hex_oid_len + e->name->str_len;
2187 tmp_fullpath_len = fullpath_len;
2188
2189 /*
2190 * We're interested in EITHER existing note entries (entries
2191 * with exactly 40 hex chars in path, not including directory
2192 * separators), OR directory entries that may contain note
2193 * entries (with < 40 hex chars in path).
2194 * Also, each path component in a note entry must be a multiple
2195 * of 2 chars.
2196 */
2197 if (!e->versions[1].mode ||
2198 tmp_hex_oid_len > hexsz ||
2199 e->name->str_len % 2)
2200 continue;
2201
2202 /* This _may_ be a note entry, or a subdir containing notes */
2203 memcpy(hex_oid + hex_oid_len, e->name->str_dat,
2204 e->name->str_len);
2205 if (tmp_fullpath_len)
2206 fullpath[tmp_fullpath_len++] = '/';
2207 memcpy(fullpath + tmp_fullpath_len, e->name->str_dat,
2208 e->name->str_len);
2209 tmp_fullpath_len += e->name->str_len;
2210 fullpath[tmp_fullpath_len] = '\0';
2211
2212 if (tmp_hex_oid_len == hexsz && !get_oid_hex(hex_oid, &oid)) {
2213 /* This is a note entry */
2214 if (fanout == 0xff) {
2215 /* Counting mode, no rename */
2216 num_notes++;
2217 continue;
2218 }
2219 construct_path_with_fanout(hex_oid, fanout, realpath);
2220 if (!strcmp(fullpath, realpath)) {
2221 /* Note entry is in correct location */
2222 num_notes++;
2223 continue;
2224 }
2225
2226 /* Rename fullpath to realpath */
2227 if (!tree_content_remove(orig_root, fullpath, &leaf, 0))
2228 die(_("failed to remove path %s"), fullpath);
2229 tree_content_set(orig_root, realpath,
2230 &leaf.versions[1].oid,
2231 leaf.versions[1].mode,
2232 leaf.tree);
2233 } else if (S_ISDIR(e->versions[1].mode)) {
2234 /* This is a subdir that may contain note entries */
2235 num_notes += do_change_note_fanout(orig_root, e,
2236 hex_oid, tmp_hex_oid_len,
2237 fullpath, tmp_fullpath_len, fanout);
2238 }
2239
2240 /* The above may have reallocated the current tree_content */
2241 t = root->tree;
2242 }
2243 return num_notes;
2244 }
2245
2246 static uintmax_t change_note_fanout(struct tree_entry *root,
2247 unsigned char fanout)
2248 {
2249 /*
2250 * The size of path is due to one slash between every two hex digits,
2251 * plus the terminating NUL. Note that there is no slash at the end, so
2252 * the number of slashes is one less than half the number of hex
2253 * characters.
2254 */
2255 char hex_oid[GIT_MAX_HEXSZ], path[GIT_MAX_HEXSZ + (GIT_MAX_HEXSZ / 2) - 1 + 1];
2256 return do_change_note_fanout(root, root, hex_oid, 0, path, 0, fanout);
2257 }
2258
2259 static int parse_mapped_oid_hex(const char *hex, struct object_id *oid, const char **end)
2260 {
2261 int algo;
2262 khiter_t it;
2263
2264 /* Make SHA-1 object IDs have all-zero padding. */
2265 memset(oid->hash, 0, sizeof(oid->hash));
2266
2267 algo = parse_oid_hex_any(hex, oid, end);
2268 if (algo == GIT_HASH_UNKNOWN)
2269 return -1;
2270
2271 it = kh_get_oid_map(sub_oid_map, *oid);
2272 /* No such object? */
2273 if (it == kh_end(sub_oid_map)) {
2274 /* If we're using the same algorithm, pass it through. */
2275 if (hash_algos[algo].format_id == the_hash_algo->format_id)
2276 return 0;
2277 return -1;
2278 }
2279 oidcpy(oid, kh_value(sub_oid_map, it));
2280 return 0;
2281 }
2282
2283 /*
2284 * Given a pointer into a string, parse a mark reference:
2285 *
2286 * idnum ::= ':' bigint;
2287 *
2288 * Update *endptr to point to the first character after the value.
2289 *
2290 * Complain if the following character is not what is expected,
2291 * either a space or end of the string.
2292 */
2293 static uintmax_t parse_mark_ref(const char *p, char **endptr)
2294 {
2295 uintmax_t mark;
2296
2297 assert(*p == ':');
2298 p++;
2299 mark = strtoumax(p, endptr, 10);
2300 if (*endptr == p)
2301 die(_("no value after ':' in mark: %s"), command_buf.buf);
2302 return mark;
2303 }
2304
2305 /*
2306 * Parse the mark reference, and complain if this is not the end of
2307 * the string.
2308 */
2309 static uintmax_t parse_mark_ref_eol(const char *p)
2310 {
2311 char *end;
2312 uintmax_t mark;
2313
2314 mark = parse_mark_ref(p, &end);
2315 if (*end != '\0')
2316 die(_("garbage after mark: %s"), command_buf.buf);
2317 return mark;
2318 }
2319
2320 /*
2321 * Parse the mark reference, demanding a trailing space. Update *p to
2322 * point to the first character after the space.
2323 */
2324 static uintmax_t parse_mark_ref_space(const char **p)
2325 {
2326 uintmax_t mark;
2327 char *end;
2328
2329 mark = parse_mark_ref(*p, &end);
2330 if (*end++ != ' ')
2331 die(_("missing space after mark: %s"), command_buf.buf);
2332 *p = end;
2333 return mark;
2334 }
2335
2336 /*
2337 * Parse the path string into the strbuf. The path can either be quoted with
2338 * escape sequences or unquoted without escape sequences. Unquoted strings may
2339 * contain spaces only if `is_last_field` is nonzero; otherwise, it stops
2340 * parsing at the first space.
2341 */
2342 static void parse_path(struct strbuf *sb, const char *p, const char **endp,
2343 int is_last_field, const char *field)
2344 {
2345 if (*p == '"') {
2346 if (unquote_c_style(sb, p, endp))
2347 die(_("invalid %s: %s"), field, command_buf.buf);
2348 if (strlen(sb->buf) != sb->len)
2349 die(_("NUL in %s: %s"), field, command_buf.buf);
2350 } else {
2351 /*
2352 * Unless we are parsing the last field of a line,
2353 * SP is the end of this field.
2354 */
2355 *endp = is_last_field
2356 ? p + strlen(p)
2357 : strchrnul(p, ' ');
2358 strbuf_add(sb, p, *endp - p);
2359 }
2360 }
2361
2362 /*
2363 * Parse the path string into the strbuf, and complain if this is not the end of
2364 * the string. Unquoted strings may contain spaces.
2365 */
2366 static void parse_path_eol(struct strbuf *sb, const char *p, const char *field)
2367 {
2368 const char *end;
2369
2370 parse_path(sb, p, &end, 1, field);
2371 if (*end)
2372 die(_("garbage after %s: %s"), field, command_buf.buf);
2373 }
2374
2375 /*
2376 * Parse the path string into the strbuf, and ensure it is followed by a space.
2377 * Unquoted strings may not contain spaces. Update *endp to point to the first
2378 * character after the space.
2379 */
2380 static void parse_path_space(struct strbuf *sb, const char *p,
2381 const char **endp, const char *field)
2382 {
2383 parse_path(sb, p, endp, 0, field);
2384 if (**endp != ' ')
2385 die(_("missing space after %s: %s"), field, command_buf.buf);
2386 (*endp)++;
2387 }
2388
2389 static void file_change_m(struct fast_import_state *state, const char *p, struct branch *b)
2390 {
2391 static struct strbuf path = STRBUF_INIT;
2392 struct object_entry *oe;
2393 struct object_id oid;
2394 uint16_t mode, inline_data = 0;
2395
2396 p = parse_mode(p, &mode);
2397 if (!p)
2398 die(_("corrupt mode: %s"), command_buf.buf);
2399 switch (mode) {
2400 case 0644:
2401 case 0755:
2402 mode |= S_IFREG;
2403 case S_IFREG | 0644:
2404 case S_IFREG | 0755:
2405 case S_IFLNK:
2406 case S_IFDIR:
2407 case S_IFGITLINK:
2408 /* ok */
2409 break;
2410 default:
2411 die(_("corrupt mode: %s"), command_buf.buf);
2412 }
2413
2414 if (*p == ':') {
2415 oe = find_mark(marks, parse_mark_ref_space(&p));
2416 oidcpy(&oid, &oe->idx.oid);
2417 } else if (skip_prefix(p, "inline ", &p)) {
2418 inline_data = 1;
2419 oe = NULL; /* not used with inline_data, but makes gcc happy */
2420 } else {
2421 if (parse_mapped_oid_hex(p, &oid, &p))
2422 die(_("invalid dataref: %s"), command_buf.buf);
2423 oe = find_object(&oid);
2424 if (*p++ != ' ')
2425 die(_("missing space after SHA1: %s"), command_buf.buf);
2426 }
2427
2428 strbuf_reset(&path);
2429 parse_path_eol(&path, p, "path");
2430
2431 /* Git does not track empty, non-toplevel directories. */
2432 if (S_ISDIR(mode) &&
2433 is_empty_tree_oid(&oid, the_repository->hash_algo) &&
2434 *path.buf) {
2435 tree_content_remove(&b->branch_tree, path.buf, NULL, 0);
2436 return;
2437 }
2438
2439 if (S_ISGITLINK(mode)) {
2440 if (inline_data)
2441 die(_("Git links cannot be specified 'inline': %s"),
2442 command_buf.buf);
2443 else if (oe) {
2444 if (oe->type != OBJ_COMMIT)
2445 die(_("not a commit (actually a %s): %s"),
2446 type_name(oe->type), command_buf.buf);
2447 }
2448 /*
2449 * Accept the sha1 without checking; it expected to be in
2450 * another repository.
2451 */
2452 } else if (inline_data) {
2453 if (S_ISDIR(mode))
2454 die(_("directories cannot be specified 'inline': %s"),
2455 command_buf.buf);
2456 while (read_next_command(state) != EOF) {
2457 const char *v;
2458 if (skip_prefix(command_buf.buf, "cat-blob ", &v))
2459 parse_cat_blob(state, v);
2460 else {
2461 parse_and_store_blob(&last_blob, &oid, 0);
2462 break;
2463 }
2464 }
2465 } else {
2466 enum object_type expected = S_ISDIR(mode) ?
2467 OBJ_TREE: OBJ_BLOB;
2468 enum object_type type = oe ? oe->type :
2469 odb_read_object_info(the_repository->objects,
2470 &oid, NULL);
2471 if (type < 0)
2472 die(_("%s not found: %s"),
2473 S_ISDIR(mode) ? _("tree") : _("blob"),
2474 command_buf.buf);
2475 if (type != expected)
2476 die(_("not a %s (actually a %s): %s"),
2477 type_name(expected), type_name(type),
2478 command_buf.buf);
2479 }
2480
2481 if (!*path.buf) {
2482 tree_content_replace(&b->branch_tree, &oid, mode, NULL);
2483 return;
2484 }
2485
2486 if (!verify_path(path.buf, mode))
2487 die(_("invalid path '%s'"), path.buf);
2488 tree_content_set(&b->branch_tree, path.buf, &oid, mode, NULL);
2489 }
2490
2491 static void file_change_d(const char *p, struct branch *b)
2492 {
2493 static struct strbuf path = STRBUF_INIT;
2494
2495 strbuf_reset(&path);
2496 parse_path_eol(&path, p, "path");
2497 tree_content_remove(&b->branch_tree, path.buf, NULL, 1);
2498 }
2499
2500 static void file_change_cr(const char *p, struct branch *b, int rename)
2501 {
2502 static struct strbuf source = STRBUF_INIT;
2503 static struct strbuf dest = STRBUF_INIT;
2504 struct tree_entry leaf;
2505
2506 strbuf_reset(&source);
2507 parse_path_space(&source, p, &p, "source");
2508 strbuf_reset(&dest);
2509 parse_path_eol(&dest, p, "dest");
2510
2511 memset(&leaf, 0, sizeof(leaf));
2512 if (rename)
2513 tree_content_remove(&b->branch_tree, source.buf, &leaf, 1);
2514 else
2515 tree_content_get(&b->branch_tree, source.buf, &leaf, 1);
2516 if (!leaf.versions[1].mode)
2517 die(_("path %s not in branch"), source.buf);
2518 if (!*dest.buf) { /* C "path/to/subdir" "" */
2519 tree_content_replace(&b->branch_tree,
2520 &leaf.versions[1].oid,
2521 leaf.versions[1].mode,
2522 leaf.tree);
2523 return;
2524 }
2525 if (!verify_path(dest.buf, leaf.versions[1].mode))
2526 die(_("invalid path '%s'"), dest.buf);
2527 tree_content_set(&b->branch_tree, dest.buf,
2528 &leaf.versions[1].oid,
2529 leaf.versions[1].mode,
2530 leaf.tree);
2531 }
2532
2533 static void note_change_n(struct fast_import_state *state, const char *p, struct branch *b, unsigned char *old_fanout)
2534 {
2535 struct object_entry *oe;
2536 struct branch *s;
2537 struct object_id oid, commit_oid;
2538 char path[GIT_MAX_RAWSZ * 3];
2539 uint16_t inline_data = 0;
2540 unsigned char new_fanout;
2541
2542 /*
2543 * When loading a branch, we don't traverse its tree to count the real
2544 * number of notes (too expensive to do this for all non-note refs).
2545 * This means that recently loaded notes refs might incorrectly have
2546 * b->num_notes == 0, and consequently, old_fanout might be wrong.
2547 *
2548 * Fix this by traversing the tree and counting the number of notes
2549 * when b->num_notes == 0. If the notes tree is truly empty, the
2550 * calculation should not take long.
2551 */
2552 if (b->num_notes == 0 && *old_fanout == 0) {
2553 /* Invoke change_note_fanout() in "counting mode". */
2554 b->num_notes = change_note_fanout(&b->branch_tree, 0xff);
2555 *old_fanout = convert_num_notes_to_fanout(b->num_notes);
2556 }
2557
2558 /* Now parse the notemodify command. */
2559 /* <dataref> or 'inline' */
2560 if (*p == ':') {
2561 oe = find_mark(marks, parse_mark_ref_space(&p));
2562 oidcpy(&oid, &oe->idx.oid);
2563 } else if (skip_prefix(p, "inline ", &p)) {
2564 inline_data = 1;
2565 oe = NULL; /* not used with inline_data, but makes gcc happy */
2566 } else {
2567 if (parse_mapped_oid_hex(p, &oid, &p))
2568 die(_("invalid dataref: %s"), command_buf.buf);
2569 oe = find_object(&oid);
2570 if (*p++ != ' ')
2571 die(_("missing space after SHA1: %s"), command_buf.buf);
2572 }
2573
2574 /* <commit-ish> */
2575 s = lookup_branch(p);
2576 if (s) {
2577 if (is_null_oid(&s->oid))
2578 die(_("can't add a note on empty branch."));
2579 oidcpy(&commit_oid, &s->oid);
2580 } else if (*p == ':') {
2581 uintmax_t commit_mark = parse_mark_ref_eol(p);
2582 struct object_entry *commit_oe = find_mark(marks, commit_mark);
2583 if (commit_oe->type != OBJ_COMMIT)
2584 die(_("mark :%" PRIuMAX " not a commit"), commit_mark);
2585 oidcpy(&commit_oid, &commit_oe->idx.oid);
2586 } else if (!repo_get_oid(the_repository, p, &commit_oid)) {
2587 size_t size;
2588 char *buf = odb_read_object_peeled(the_repository->objects,
2589 &commit_oid, OBJ_COMMIT, &size,
2590 &commit_oid);
2591 if (!buf || size < the_hash_algo->hexsz + 6)
2592 die(_("not a valid commit: %s"), p);
2593 free(buf);
2594 } else
2595 die(_("invalid ref name or SHA1 expression: %s"), p);
2596
2597 if (inline_data) {
2598 read_next_command(state);
2599 parse_and_store_blob(&last_blob, &oid, 0);
2600 } else if (oe) {
2601 if (oe->type != OBJ_BLOB)
2602 die(_("not a blob (actually a %s): %s"),
2603 type_name(oe->type), command_buf.buf);
2604 } else if (!is_null_oid(&oid)) {
2605 enum object_type type = odb_read_object_info(the_repository->objects, &oid,
2606 NULL);
2607 if (type < 0)
2608 die(_("blob not found: %s"), command_buf.buf);
2609 if (type != OBJ_BLOB)
2610 die(_("not a blob (actually a %s): %s"),
2611 type_name(type), command_buf.buf);
2612 }
2613
2614 construct_path_with_fanout(oid_to_hex(&commit_oid), *old_fanout, path);
2615 if (tree_content_remove(&b->branch_tree, path, NULL, 0))
2616 b->num_notes--;
2617
2618 if (is_null_oid(&oid))
2619 return; /* nothing to insert */
2620
2621 b->num_notes++;
2622 new_fanout = convert_num_notes_to_fanout(b->num_notes);
2623 construct_path_with_fanout(oid_to_hex(&commit_oid), new_fanout, path);
2624 tree_content_set(&b->branch_tree, path, &oid, S_IFREG | 0644, NULL);
2625 }
2626
2627 static void file_change_deleteall(struct branch *b)
2628 {
2629 release_tree_content_recursive(b->branch_tree.tree);
2630 oidclr(&b->branch_tree.versions[0].oid, the_repository->hash_algo);
2631 oidclr(&b->branch_tree.versions[1].oid, the_repository->hash_algo);
2632 load_tree(&b->branch_tree);
2633 b->num_notes = 0;
2634 }
2635
2636 static void parse_from_commit(struct branch *b, char *buf, unsigned long size)
2637 {
2638 if (!buf || size < the_hash_algo->hexsz + 6)
2639 die(_("not a valid commit: %s"), oid_to_hex(&b->oid));
2640 if (memcmp("tree ", buf, 5)
2641 || get_oid_hex(buf + 5, &b->branch_tree.versions[1].oid))
2642 die(_("the commit %s is corrupt"), oid_to_hex(&b->oid));
2643 oidcpy(&b->branch_tree.versions[0].oid,
2644 &b->branch_tree.versions[1].oid);
2645 }
2646
2647 static void parse_from_existing(struct branch *b)
2648 {
2649 if (is_null_oid(&b->oid)) {
2650 oidclr(&b->branch_tree.versions[0].oid, the_repository->hash_algo);
2651 oidclr(&b->branch_tree.versions[1].oid, the_repository->hash_algo);
2652 } else {
2653 unsigned long size;
2654 size_t size_st = 0;
2655 char *buf;
2656
2657 buf = odb_read_object_peeled(the_repository->objects, &b->oid,
2658 OBJ_COMMIT, &size_st, &b->oid);
2659 size = cast_size_t_to_ulong(size_st);
2660 parse_from_commit(b, buf, size);
2661 free(buf);
2662 }
2663 }
2664
2665 static int parse_objectish(struct fast_import_state *state, struct branch *b, const char *objectish)
2666 {
2667 struct branch *s;
2668 struct object_id oid;
2669
2670 oidcpy(&oid, &b->branch_tree.versions[1].oid);
2671
2672 s = lookup_branch(objectish);
2673 if (b == s)
2674 die(_("can't create a branch from itself: %s"), b->name);
2675 else if (s) {
2676 struct object_id *t = &s->branch_tree.versions[1].oid;
2677 oidcpy(&b->oid, &s->oid);
2678 oidcpy(&b->branch_tree.versions[0].oid, t);
2679 oidcpy(&b->branch_tree.versions[1].oid, t);
2680 } else if (*objectish == ':') {
2681 uintmax_t idnum = parse_mark_ref_eol(objectish);
2682 struct object_entry *oe = find_mark(marks, idnum);
2683 if (oe->type != OBJ_COMMIT)
2684 die(_("mark :%" PRIuMAX " not a commit"), idnum);
2685 if (!oideq(&b->oid, &oe->idx.oid)) {
2686 oidcpy(&b->oid, &oe->idx.oid);
2687 if (oe->pack_id != MAX_PACK_ID) {
2688 unsigned long size;
2689 char *buf = gfi_unpack_entry(oe, &size);
2690 parse_from_commit(b, buf, size);
2691 free(buf);
2692 } else
2693 parse_from_existing(b);
2694 }
2695 } else if (!repo_get_oid(the_repository, objectish, &b->oid)) {
2696 parse_from_existing(b);
2697 if (is_null_oid(&b->oid))
2698 b->delete = 1;
2699 }
2700 else
2701 die(_("invalid ref name or SHA1 expression: %s"), objectish);
2702
2703 if (b->branch_tree.tree && !oideq(&oid, &b->branch_tree.versions[1].oid)) {
2704 release_tree_content_recursive(b->branch_tree.tree);
2705 b->branch_tree.tree = NULL;
2706 }
2707
2708 read_next_command(state);
2709 return 1;
2710 }
2711
2712 static int parse_from(struct fast_import_state *state, struct branch *b)
2713 {
2714 const char *from;
2715
2716 if (!skip_prefix(command_buf.buf, "from ", &from))
2717 return 0;
2718
2719 return parse_objectish(state, b, from);
2720 }
2721
2722 static int parse_objectish_with_prefix(struct fast_import_state *state, struct branch *b, const char *prefix)
2723 {
2724 const char *base;
2725
2726 if (!skip_prefix(command_buf.buf, prefix, &base))
2727 return 0;
2728
2729 return parse_objectish(state, b, base);
2730 }
2731
2732 static struct hash_list *parse_merge(struct fast_import_state *state, unsigned int *count)
2733 {
2734 struct hash_list *list = NULL, **tail = &list, *n;
2735 const char *from;
2736 struct branch *s;
2737
2738 *count = 0;
2739 while (skip_prefix(command_buf.buf, "merge ", &from)) {
2740 n = xmalloc(sizeof(*n));
2741 s = lookup_branch(from);
2742 if (s)
2743 oidcpy(&n->oid, &s->oid);
2744 else if (*from == ':') {
2745 uintmax_t idnum = parse_mark_ref_eol(from);
2746 struct object_entry *oe = find_mark(marks, idnum);
2747 if (oe->type != OBJ_COMMIT)
2748 die(_("mark :%" PRIuMAX " not a commit"), idnum);
2749 oidcpy(&n->oid, &oe->idx.oid);
2750 } else if (!repo_get_oid(the_repository, from, &n->oid)) {
2751 size_t size;
2752 char *buf = odb_read_object_peeled(the_repository->objects,
2753 &n->oid, OBJ_COMMIT,
2754 &size, &n->oid);
2755 if (!buf || size < the_hash_algo->hexsz + 6)
2756 die(_("not a valid commit: %s"), from);
2757 free(buf);
2758 } else
2759 die(_("invalid ref name or SHA1 expression: %s"), from);
2760
2761 n->next = NULL;
2762 *tail = n;
2763 tail = &n->next;
2764
2765 (*count)++;
2766 read_next_command(state);
2767 }
2768 return list;
2769 }
2770
2771 struct signature_data {
2772 char *hash_algo; /* "sha1" or "sha256" */
2773 char *sig_format; /* "openpgp", "x509", "ssh", or "unknown" */
2774 struct strbuf data; /* The actual signature data */
2775 };
2776
2777 static void parse_one_signature(struct fast_import_state *state, struct signature_data *sig, const char *v)
2778 {
2779 char *args = xstrdup(v); /* Will be freed when sig->hash_algo is freed */
2780 char *space = strchr(args, ' ');
2781
2782 if (!space)
2783 die(_("expected gpgsig format: 'gpgsig <hash-algo> <signature-format>', "
2784 "got 'gpgsig %s'"), args);
2785 *space = '\0';
2786
2787 sig->hash_algo = args;
2788 sig->sig_format = space + 1;
2789
2790 /* Validate hash algorithm */
2791 if (strcmp(sig->hash_algo, "sha1") &&
2792 strcmp(sig->hash_algo, "sha256"))
2793 die(_("unknown git hash algorithm in gpgsig: '%s'"), sig->hash_algo);
2794
2795 /* Validate signature format */
2796 if (!valid_signature_format(sig->sig_format))
2797 die(_("invalid signature format in gpgsig: '%s'"), sig->sig_format);
2798 if (!strcmp(sig->sig_format, "unknown"))
2799 warning(_("'unknown' signature format in gpgsig"));
2800
2801 /* Read signature data */
2802 read_next_command(state);
2803 parse_data(&sig->data, 0, NULL);
2804 }
2805
2806 static void discard_one_signature(struct fast_import_state *state)
2807 {
2808 struct strbuf data = STRBUF_INIT;
2809
2810 read_next_command(state);
2811 parse_data(&data, 0, NULL);
2812 strbuf_release(&data);
2813 }
2814
2815 static void add_gpgsig_to_commit(struct strbuf *commit_data,
2816 const char *header,
2817 struct signature_data *sig)
2818 {
2819 struct string_list siglines = STRING_LIST_INIT_NODUP;
2820
2821 if (!sig || !sig->hash_algo)
2822 return;
2823
2824 strbuf_addstr(commit_data, header);
2825 string_list_split_in_place(&siglines, sig->data.buf, "\n", -1);
2826 strbuf_add_separated_string_list(commit_data, "\n ", &siglines);
2827 strbuf_addch(commit_data, '\n');
2828 string_list_clear(&siglines, 1);
2829 strbuf_release(&sig->data);
2830 free(sig->hash_algo);
2831 }
2832
2833 static void store_signature(struct signature_data *stored_sig,
2834 struct signature_data *new_sig,
2835 const char *hash_type)
2836 {
2837 if (stored_sig->hash_algo) {
2838 warning(_("multiple %s signatures found, "
2839 "ignoring additional signature"),
2840 hash_type);
2841 strbuf_release(&new_sig->data);
2842 free(new_sig->hash_algo);
2843 } else {
2844 *stored_sig = *new_sig;
2845 }
2846 }
2847
2848 static void import_one_signature(struct fast_import_state *state,
2849 struct signature_data *sig_sha1,
2850 struct signature_data *sig_sha256,
2851 const char *v)
2852 {
2853 struct signature_data sig = { NULL, NULL, STRBUF_INIT };
2854
2855 parse_one_signature(state, &sig, v);
2856
2857 if (!strcmp(sig.hash_algo, "sha1"))
2858 store_signature(sig_sha1, &sig, "SHA-1");
2859 else if (!strcmp(sig.hash_algo, "sha256"))
2860 store_signature(sig_sha256, &sig, "SHA-256");
2861 else
2862 die(_("parse_one_signature() returned unknown hash algo"));
2863 }
2864
2865 static void finalize_commit_buffer(struct strbuf *new_data,
2866 struct signature_data *sig_sha1,
2867 struct signature_data *sig_sha256,
2868 struct strbuf *msg)
2869 {
2870 add_gpgsig_to_commit(new_data, "gpgsig ", sig_sha1);
2871 add_gpgsig_to_commit(new_data, "gpgsig-sha256 ", sig_sha256);
2872
2873 strbuf_addch(new_data, '\n');
2874 strbuf_addbuf(new_data, msg);
2875 }
2876
2877 static void warn_invalid_signature(struct signature_check *check,
2878 const char *msg, enum sign_mode mode)
2879 {
2880 const char *signer = check->signer ? check->signer : _("unknown");
2881 const char *subject;
2882 int subject_len = find_commit_subject(msg, &subject);
2883
2884 switch (mode) {
2885 case SIGN_STRIP_IF_INVALID:
2886 if (subject_len > 100)
2887 warning(_("stripping invalid signature for commit '%.100s...'\n"
2888 " allegedly by %s"), subject, signer);
2889 else if (subject_len > 0)
2890 warning(_("stripping invalid signature for commit '%.*s'\n"
2891 " allegedly by %s"), subject_len, subject, signer);
2892 else
2893 warning(_("stripping invalid signature for commit\n"
2894 " allegedly by %s"), signer);
2895 break;
2896 case SIGN_SIGN_IF_INVALID:
2897 if (subject_len > 100)
2898 warning(_("replacing invalid signature for commit '%.100s...'\n"
2899 " allegedly by %s"), subject, signer);
2900 else if (subject_len > 0)
2901 warning(_("replacing invalid signature for commit '%.*s'\n"
2902 " allegedly by %s"), subject_len, subject, signer);
2903 else
2904 warning(_("replacing invalid signature for commit\n"
2905 " allegedly by %s"), signer);
2906 break;
2907 default:
2908 BUG("unsupported signing mode");
2909 }
2910 }
2911
2912 static void handle_signature_if_invalid(struct strbuf *new_data,
2913 struct signature_data *sig_sha1,
2914 struct signature_data *sig_sha256,
2915 struct strbuf *msg,
2916 enum sign_mode mode)
2917 {
2918 struct strbuf tmp_buf = STRBUF_INIT;
2919 struct signature_check signature_check = { 0 };
2920 int ret;
2921
2922 /* Check signature in a temporary commit buffer */
2923 strbuf_addbuf(&tmp_buf, new_data);
2924 finalize_commit_buffer(&tmp_buf, sig_sha1, sig_sha256, msg);
2925 ret = verify_commit_buffer(tmp_buf.buf, tmp_buf.len, &signature_check);
2926
2927 if (ret) {
2928 if (mode == SIGN_ABORT_IF_INVALID)
2929 die(_("aborting due to invalid signature"));
2930
2931 warn_invalid_signature(&signature_check, msg->buf, mode);
2932
2933 if (mode == SIGN_SIGN_IF_INVALID) {
2934 struct strbuf signature = STRBUF_INIT;
2935 struct strbuf payload = STRBUF_INIT;
2936
2937 /*
2938 * NEEDSWORK: To properly support interoperability mode
2939 * when signing commit signatures, the commit buffer
2940 * must be provided in both the repository and
2941 * compatibility object formats. As currently
2942 * implemented, only the repository object format is
2943 * considered meaning compatibility signatures cannot be
2944 * generated. Thus, attempting to sign commit signatures
2945 * in interoperability mode is currently unsupported.
2946 */
2947 if (the_repository->compat_hash_algo)
2948 die(_("signing commits in interoperability mode is unsupported"));
2949
2950 strbuf_addstr(&payload, signature_check.payload);
2951 if (sign_buffer(&payload, &signature, signed_commit_keyid,
2952 SIGN_BUFFER_USE_DEFAULT_KEY))
2953 die(_("failed to sign commit object"));
2954 add_header_signature(new_data, &signature, the_hash_algo);
2955
2956 strbuf_release(&signature);
2957 strbuf_release(&payload);
2958 }
2959
2960 finalize_commit_buffer(new_data, NULL, NULL, msg);
2961 } else {
2962 strbuf_swap(new_data, &tmp_buf);
2963 }
2964
2965 signature_check_clear(&signature_check);
2966 strbuf_release(&tmp_buf);
2967 }
2968
2969 static void parse_new_commit(struct fast_import_state *state, const char *arg)
2970 {
2971 static struct strbuf msg = STRBUF_INIT;
2972 struct signature_data sig_sha1 = { NULL, NULL, STRBUF_INIT };
2973 struct signature_data sig_sha256 = { NULL, NULL, STRBUF_INIT };
2974 struct branch *b;
2975 char *author = NULL;
2976 char *committer = NULL;
2977 char *encoding = NULL;
2978 struct hash_list *merge_list = NULL;
2979 unsigned int merge_count;
2980 unsigned char prev_fanout, new_fanout;
2981 const char *v;
2982
2983 b = lookup_branch(arg);
2984 if (!b)
2985 b = new_branch(arg);
2986
2987 read_next_command(state);
2988 parse_mark(state);
2989 parse_original_identifier(state);
2990 if (skip_prefix(command_buf.buf, "author ", &v)) {
2991 author = parse_ident(v);
2992 read_next_command(state);
2993 }
2994 if (skip_prefix(command_buf.buf, "committer ", &v)) {
2995 committer = parse_ident(v);
2996 read_next_command(state);
2997 }
2998 if (!committer)
2999 die(_("expected committer but didn't get one"));
3000
3001 while (skip_prefix(command_buf.buf, "gpgsig ", &v)) {
3002 switch (signed_commit_mode) {
3003
3004 /* First, modes that don't need the signature to be parsed */
3005 case SIGN_ABORT:
3006 die(_("encountered signed commit; use "
3007 "--signed-commits=<mode> to handle it"));
3008 case SIGN_WARN_STRIP:
3009 warning(_("stripping a commit signature"));
3010 /* fallthru */
3011 case SIGN_STRIP:
3012 discard_one_signature(state);
3013 break;
3014
3015 /* Second, modes that parse the signature */
3016 case SIGN_WARN_VERBATIM:
3017 warning(_("importing a commit signature verbatim"));
3018 /* fallthru */
3019 case SIGN_VERBATIM:
3020 case SIGN_STRIP_IF_INVALID:
3021 case SIGN_SIGN_IF_INVALID:
3022 case SIGN_ABORT_IF_INVALID:
3023 import_one_signature(state, &sig_sha1, &sig_sha256, v);
3024 break;
3025
3026 /* Third, BUG */
3027 default:
3028 BUG("invalid signed_commit_mode value %d", signed_commit_mode);
3029 }
3030 read_next_command(state);
3031 }
3032
3033 if (skip_prefix(command_buf.buf, "encoding ", &v)) {
3034 encoding = xstrdup(v);
3035 read_next_command(state);
3036 }
3037 parse_data(&msg, 0, NULL);
3038 read_next_command(state);
3039 parse_from(state, b);
3040 merge_list = parse_merge(state, &merge_count);
3041
3042 /* ensure the branch is active/loaded */
3043 if (!b->branch_tree.tree || !max_active_branches) {
3044 unload_one_branch();
3045 load_branch(b);
3046 }
3047
3048 prev_fanout = convert_num_notes_to_fanout(b->num_notes);
3049
3050 /* file_change* */
3051 while (command_buf.len > 0) {
3052 if (skip_prefix(command_buf.buf, "M ", &v))
3053 file_change_m(state, v, b);
3054 else if (skip_prefix(command_buf.buf, "D ", &v))
3055 file_change_d(v, b);
3056 else if (skip_prefix(command_buf.buf, "R ", &v))
3057 file_change_cr(v, b, 1);
3058 else if (skip_prefix(command_buf.buf, "C ", &v))
3059 file_change_cr(v, b, 0);
3060 else if (skip_prefix(command_buf.buf, "N ", &v))
3061 note_change_n(state, v, b, &prev_fanout);
3062 else if (!strcmp("deleteall", command_buf.buf))
3063 file_change_deleteall(b);
3064 else if (skip_prefix(command_buf.buf, "ls ", &v))
3065 parse_ls(state, v, b);
3066 else if (skip_prefix(command_buf.buf, "cat-blob ", &v))
3067 parse_cat_blob(state, v);
3068 else {
3069 unread_command_buf = 1;
3070 break;
3071 }
3072 if (read_next_command(state) == EOF)
3073 break;
3074 }
3075
3076 new_fanout = convert_num_notes_to_fanout(b->num_notes);
3077 if (new_fanout != prev_fanout)
3078 b->num_notes = change_note_fanout(&b->branch_tree, new_fanout);
3079
3080 /* build the tree and the commit */
3081 store_tree(&b->branch_tree);
3082 oidcpy(&b->branch_tree.versions[0].oid,
3083 &b->branch_tree.versions[1].oid);
3084
3085 strbuf_reset(&new_data);
3086 strbuf_addf(&new_data, "tree %s\n",
3087 oid_to_hex(&b->branch_tree.versions[1].oid));
3088 if (!is_null_oid(&b->oid))
3089 strbuf_addf(&new_data, "parent %s\n",
3090 oid_to_hex(&b->oid));
3091 while (merge_list) {
3092 struct hash_list *next = merge_list->next;
3093 strbuf_addf(&new_data, "parent %s\n",
3094 oid_to_hex(&merge_list->oid));
3095 free(merge_list);
3096 merge_list = next;
3097 }
3098 strbuf_addf(&new_data,
3099 "author %s\n"
3100 "committer %s\n",
3101 author ? author : committer, committer);
3102 if (encoding)
3103 strbuf_addf(&new_data,
3104 "encoding %s\n",
3105 encoding);
3106
3107 if ((signed_commit_mode == SIGN_STRIP_IF_INVALID ||
3108 signed_commit_mode == SIGN_SIGN_IF_INVALID ||
3109 signed_commit_mode == SIGN_ABORT_IF_INVALID) &&
3110 (sig_sha1.hash_algo || sig_sha256.hash_algo))
3111 handle_signature_if_invalid(&new_data, &sig_sha1, &sig_sha256,
3112 &msg, signed_commit_mode);
3113 else
3114 finalize_commit_buffer(&new_data, &sig_sha1, &sig_sha256, &msg);
3115
3116 free(author);
3117 free(committer);
3118 free(encoding);
3119
3120 if (!store_object(OBJ_COMMIT, &new_data, NULL, &b->oid, next_mark))
3121 b->pack_id = pack_id;
3122 b->last_commit = object_count_by_type[OBJ_COMMIT];
3123 }
3124
3125 static void handle_tag_signature_if_invalid(struct strbuf *buf,
3126 struct strbuf *msg,
3127 size_t sig_offset)
3128 {
3129 struct strbuf signature = STRBUF_INIT;
3130 struct strbuf payload = STRBUF_INIT;
3131 struct signature_check sigc = { 0 };
3132
3133 strbuf_addbuf(&payload, buf);
3134 strbuf_addch(&payload, '\n');
3135 strbuf_add(&payload, msg->buf, sig_offset);
3136 strbuf_add(&signature, msg->buf + sig_offset, msg->len - sig_offset);
3137
3138 sigc.payload_type = SIGNATURE_PAYLOAD_TAG;
3139 sigc.payload = strbuf_detach(&payload, &sigc.payload_len);
3140
3141 if (!check_signature(&sigc, signature.buf, signature.len))
3142 goto out;
3143
3144 if (signed_tag_mode == SIGN_ABORT_IF_INVALID)
3145 die(_("aborting due to invalid signature"));
3146
3147 strbuf_setlen(msg, sig_offset);
3148
3149 if (signed_tag_mode == SIGN_SIGN_IF_INVALID) {
3150 strbuf_attach(&payload, sigc.payload, sigc.payload_len,
3151 sigc.payload_len + 1);
3152 sigc.payload = NULL;
3153 strbuf_reset(&signature);
3154
3155 if (sign_buffer(&payload, &signature, signed_tag_keyid,
3156 SIGN_BUFFER_USE_DEFAULT_KEY))
3157 die(_("failed to sign tag object"));
3158
3159 strbuf_addbuf(msg, &signature);
3160 }
3161
3162 out:
3163 signature_check_clear(&sigc);
3164 strbuf_release(&signature);
3165 strbuf_release(&payload);
3166 }
3167
3168 static void handle_tag_signature(struct strbuf *buf, struct strbuf *msg, const char *name)
3169 {
3170 size_t sig_offset = parse_signed_buffer(msg->buf, msg->len);
3171
3172 /* If there is no signature, there is nothing to do. */
3173 if (sig_offset >= msg->len)
3174 return;
3175
3176 switch (signed_tag_mode) {
3177
3178 /* First, modes that don't change anything */
3179 case SIGN_WARN_VERBATIM:
3180 warning(_("importing a tag signature verbatim for tag '%s'"), name);
3181 /* fallthru */
3182 case SIGN_VERBATIM:
3183 /* Nothing to do, the signature will be put into the imported tag. */
3184 break;
3185
3186 /* Second, modes that remove the signature */
3187 case SIGN_WARN_STRIP:
3188 warning(_("stripping a tag signature for tag '%s'"), name);
3189 /* fallthru */
3190 case SIGN_STRIP:
3191 /* Truncate the buffer to remove the signature */
3192 strbuf_setlen(msg, sig_offset);
3193 break;
3194 case SIGN_ABORT_IF_INVALID:
3195 case SIGN_SIGN_IF_INVALID:
3196 case SIGN_STRIP_IF_INVALID:
3197 handle_tag_signature_if_invalid(buf, msg, sig_offset);
3198 break;
3199
3200 /* Third, aborting modes */
3201 case SIGN_ABORT:
3202 die(_("encountered signed tag; use "
3203 "--signed-tags=<mode> to handle it"));
3204 default:
3205 BUG("invalid signed_tag_mode value %d from tag '%s'",
3206 signed_tag_mode, name);
3207 }
3208 }
3209
3210 static void parse_new_tag(struct fast_import_state *state, const char *arg)
3211 {
3212 static struct strbuf msg = STRBUF_INIT;
3213 const char *from;
3214 char *tagger;
3215 struct branch *s;
3216 struct tag *t;
3217 uintmax_t from_mark = 0;
3218 struct object_id oid;
3219 enum object_type type;
3220 const char *v;
3221
3222 t = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct tag));
3223 t->name = mem_pool_strdup(&fi_mem_pool, arg);
3224 if (last_tag)
3225 last_tag->next_tag = t;
3226 else
3227 first_tag = t;
3228 last_tag = t;
3229 read_next_command(state);
3230 parse_mark(state);
3231
3232 /* from ... */
3233 if (!skip_prefix(command_buf.buf, "from ", &from))
3234 die(_("expected 'from' command, got '%s'"), command_buf.buf);
3235 s = lookup_branch(from);
3236 if (s) {
3237 if (is_null_oid(&s->oid))
3238 die(_("can't tag an empty branch."));
3239 oidcpy(&oid, &s->oid);
3240 type = OBJ_COMMIT;
3241 } else if (*from == ':') {
3242 struct object_entry *oe;
3243 from_mark = parse_mark_ref_eol(from);
3244 oe = find_mark(marks, from_mark);
3245 type = oe->type;
3246 oidcpy(&oid, &oe->idx.oid);
3247 } else if (!repo_get_oid(the_repository, from, &oid)) {
3248 struct object_entry *oe = find_object(&oid);
3249 if (!oe) {
3250 type = odb_read_object_info(the_repository->objects,
3251 &oid, NULL);
3252 if (type < 0)
3253 die(_("not a valid object: %s"), from);
3254 } else
3255 type = oe->type;
3256 } else
3257 die(_("invalid ref name or SHA1 expression: %s"), from);
3258 read_next_command(state);
3259
3260 /* original-oid ... */
3261 parse_original_identifier(state);
3262
3263 /* tagger ... */
3264 if (skip_prefix(command_buf.buf, "tagger ", &v)) {
3265 tagger = parse_ident(v);
3266 read_next_command(state);
3267 } else
3268 tagger = NULL;
3269
3270 /* tag payload/message */
3271 parse_data(&msg, 0, NULL);
3272
3273 /* build the tag object */
3274 strbuf_reset(&new_data);
3275
3276 strbuf_addf(&new_data,
3277 "object %s\n"
3278 "type %s\n"
3279 "tag %s\n",
3280 oid_to_hex(&oid), type_name(type), t->name);
3281 if (tagger)
3282 strbuf_addf(&new_data,
3283 "tagger %s\n", tagger);
3284
3285 handle_tag_signature(&new_data, &msg, t->name);
3286
3287 strbuf_addch(&new_data, '\n');
3288 strbuf_addbuf(&new_data, &msg);
3289 free(tagger);
3290
3291 if (store_object(OBJ_TAG, &new_data, NULL, &t->oid, next_mark))
3292 t->pack_id = MAX_PACK_ID;
3293 else
3294 t->pack_id = pack_id;
3295 }
3296
3297 static void parse_reset_branch(struct fast_import_state *state, const char *arg)
3298 {
3299 struct branch *b;
3300 const char *tag_name;
3301
3302 b = lookup_branch(arg);
3303 if (b) {
3304 oidclr(&b->oid, the_repository->hash_algo);
3305 oidclr(&b->branch_tree.versions[0].oid, the_repository->hash_algo);
3306 oidclr(&b->branch_tree.versions[1].oid, the_repository->hash_algo);
3307 if (b->branch_tree.tree) {
3308 release_tree_content_recursive(b->branch_tree.tree);
3309 b->branch_tree.tree = NULL;
3310 }
3311 }
3312 else
3313 b = new_branch(arg);
3314 read_next_command(state);
3315 parse_from(state, b);
3316 if (b->delete && skip_prefix(b->name, "refs/tags/", &tag_name)) {
3317 /*
3318 * Elsewhere, we call dump_branches() before dump_tags(),
3319 * and dump_branches() will handle ref deletions first, so
3320 * in order to make sure the deletion actually takes effect,
3321 * we need to remove the tag from our list of tags to update.
3322 *
3323 * NEEDSWORK: replace list of tags with hashmap for faster
3324 * deletion?
3325 */
3326 struct tag *t, *prev = NULL;
3327 for (t = first_tag; t; t = t->next_tag) {
3328 if (!strcmp(t->name, tag_name))
3329 break;
3330 prev = t;
3331 }
3332 if (t) {
3333 if (prev)
3334 prev->next_tag = t->next_tag;
3335 else
3336 first_tag = t->next_tag;
3337 if (!t->next_tag)
3338 last_tag = prev;
3339 /* There is no mem_pool_free(t) function to call. */
3340 }
3341 }
3342 if (command_buf.len > 0)
3343 unread_command_buf = 1;
3344 }
3345
3346 static void cat_blob_write(const char *buf, unsigned long size)
3347 {
3348 if (write_in_full(cat_blob_fd, buf, size) < 0)
3349 die_errno(_("write to frontend failed"));
3350 }
3351
3352 static void cat_blob(struct object_entry *oe, struct object_id *oid)
3353 {
3354 struct strbuf line = STRBUF_INIT;
3355 struct iovec iov[3];
3356 unsigned long size;
3357 enum object_type type = 0;
3358 char *buf;
3359
3360 if (!oe || oe->pack_id == MAX_PACK_ID) {
3361 size_t size_st = 0;
3362 buf = odb_read_object(the_repository->objects, oid, &type,
3363 &size_st);
3364 size = cast_size_t_to_ulong(size_st);
3365 } else {
3366 type = oe->type;
3367 buf = gfi_unpack_entry(oe, &size);
3368 }
3369
3370 /*
3371 * Output based on batch_one_object() from cat-file.c.
3372 */
3373 if (type <= 0) {
3374 strbuf_reset(&line);
3375 strbuf_addf(&line, "%s missing\n", oid_to_hex(oid));
3376 cat_blob_write(line.buf, line.len);
3377 strbuf_release(&line);
3378 free(buf);
3379 return;
3380 }
3381 if (!buf)
3382 die(_("can't read object %s"), oid_to_hex(oid));
3383 if (type != OBJ_BLOB)
3384 die(_("object %s is a %s but a blob was expected."),
3385 oid_to_hex(oid), type_name(type));
3386 strbuf_reset(&line);
3387 strbuf_addf(&line, "%s %s %"PRIuMAX"\n", oid_to_hex(oid),
3388 type_name(type), (uintmax_t)size);
3389
3390 /*
3391 * Write the header, the payload and the trailing newline with a
3392 * single writev(3p) call instead of three separate write(3p) calls.
3393 */
3394 iov[0].iov_base = line.buf;
3395 iov[0].iov_len = line.len;
3396 iov[1].iov_base = buf;
3397 iov[1].iov_len = size;
3398 iov[2].iov_base = (void *) "\n";
3399 iov[2].iov_len = 1;
3400
3401 if (writev_in_full(cat_blob_fd, iov, ARRAY_SIZE(iov)) < 0)
3402 die_errno(_("write to frontend failed"));
3403 strbuf_release(&line);
3404 if (oe && oe->pack_id == pack_id) {
3405 last_blob.offset = oe->idx.offset;
3406 strbuf_attach(&last_blob.data, buf, size, size + 1);
3407 last_blob.depth = oe->depth;
3408 } else
3409 free(buf);
3410 }
3411
3412 static void parse_get_mark(struct fast_import_state *state UNUSED, const char *p)
3413 {
3414 struct object_entry *oe;
3415 char output[GIT_MAX_HEXSZ + 2];
3416
3417 /* get-mark SP <object> LF */
3418 if (*p != ':')
3419 die(_("not a mark: %s"), p);
3420
3421 oe = find_mark(marks, parse_mark_ref_eol(p));
3422 if (!oe)
3423 die(_("unknown mark: %s"), command_buf.buf);
3424
3425 xsnprintf(output, sizeof(output), "%s\n", oid_to_hex(&oe->idx.oid));
3426 cat_blob_write(output, the_hash_algo->hexsz + 1);
3427 }
3428
3429 static void parse_cat_blob(struct fast_import_state *state UNUSED, const char *p)
3430 {
3431 struct object_entry *oe;
3432 struct object_id oid;
3433
3434 /* cat-blob SP <object> LF */
3435 if (*p == ':') {
3436 oe = find_mark(marks, parse_mark_ref_eol(p));
3437 if (!oe)
3438 die(_("unknown mark: %s"), command_buf.buf);
3439 oidcpy(&oid, &oe->idx.oid);
3440 } else {
3441 if (parse_mapped_oid_hex(p, &oid, &p))
3442 die(_("invalid dataref: %s"), command_buf.buf);
3443 if (*p)
3444 die(_("garbage after SHA1: %s"), command_buf.buf);
3445 oe = find_object(&oid);
3446 }
3447
3448 cat_blob(oe, &oid);
3449 }
3450
3451 static struct object_entry *dereference(struct object_entry *oe,
3452 struct object_id *oid)
3453 {
3454 unsigned long size;
3455 char *buf = NULL;
3456 const unsigned hexsz = the_hash_algo->hexsz;
3457
3458 if (!oe) {
3459 enum object_type type = odb_read_object_info(the_repository->objects,
3460 oid, NULL);
3461 if (type < 0)
3462 die(_("object not found: %s"), oid_to_hex(oid));
3463 /* cache it! */
3464 oe = insert_object(oid);
3465 oe->type = type;
3466 oe->pack_id = MAX_PACK_ID;
3467 oe->idx.offset = 1;
3468 }
3469 switch (oe->type) {
3470 case OBJ_TREE: /* easy case. */
3471 return oe;
3472 case OBJ_COMMIT:
3473 case OBJ_TAG:
3474 break;
3475 default:
3476 die(_("not a tree-ish: %s"), command_buf.buf);
3477 }
3478
3479 if (oe->pack_id != MAX_PACK_ID) { /* in a pack being written */
3480 buf = gfi_unpack_entry(oe, &size);
3481 } else {
3482 enum object_type unused;
3483 size_t size_st = 0;
3484 buf = odb_read_object(the_repository->objects, oid,
3485 &unused, &size_st);
3486 size = cast_size_t_to_ulong(size_st);
3487 }
3488 if (!buf)
3489 die(_("can't load object %s"), oid_to_hex(oid));
3490
3491 /* Peel one layer. */
3492 switch (oe->type) {
3493 case OBJ_TAG:
3494 if (size < hexsz + strlen("object ") ||
3495 get_oid_hex(buf + strlen("object "), oid))
3496 die(_("invalid SHA1 in tag: %s"), command_buf.buf);
3497 break;
3498 case OBJ_COMMIT:
3499 if (size < hexsz + strlen("tree ") ||
3500 get_oid_hex(buf + strlen("tree "), oid))
3501 die(_("invalid SHA1 in commit: %s"), command_buf.buf);
3502 }
3503
3504 free(buf);
3505 return find_object(oid);
3506 }
3507
3508 static void insert_mapped_mark(uintmax_t mark, void *object, void *cbp)
3509 {
3510 struct object_id *fromoid = object;
3511 struct object_id *tooid = find_mark(cbp, mark);
3512 int ret;
3513 khiter_t it;
3514
3515 it = kh_put_oid_map(sub_oid_map, *fromoid, &ret);
3516 /* We've already seen this object. */
3517 if (ret == 0)
3518 return;
3519 kh_value(sub_oid_map, it) = tooid;
3520 }
3521
3522 static void build_mark_map_one(struct mark_set *from, struct mark_set *to)
3523 {
3524 for_each_mark(from, 0, insert_mapped_mark, to);
3525 }
3526
3527 static void build_mark_map(struct string_list *from, struct string_list *to)
3528 {
3529 struct string_list_item *fromp, *top;
3530
3531 sub_oid_map = kh_init_oid_map();
3532
3533 for_each_string_list_item(fromp, from) {
3534 top = string_list_lookup(to, fromp->string);
3535 if (!fromp->util) {
3536 die(_("missing from marks for submodule '%s'"), fromp->string);
3537 } else if (!top || !top->util) {
3538 die(_("missing to marks for submodule '%s'"), fromp->string);
3539 }
3540 build_mark_map_one(fromp->util, top->util);
3541 }
3542 }
3543
3544 static struct object_entry *parse_treeish_dataref(const char **p)
3545 {
3546 struct object_id oid;
3547 struct object_entry *e;
3548
3549 if (**p == ':') { /* <mark> */
3550 e = find_mark(marks, parse_mark_ref_space(p));
3551 if (!e)
3552 die(_("unknown mark: %s"), command_buf.buf);
3553 oidcpy(&oid, &e->idx.oid);
3554 } else { /* <sha1> */
3555 if (parse_mapped_oid_hex(*p, &oid, p))
3556 die(_("invalid dataref: %s"), command_buf.buf);
3557 e = find_object(&oid);
3558 if (*(*p)++ != ' ')
3559 die(_("missing space after tree-ish: %s"), command_buf.buf);
3560 }
3561
3562 while (!e || e->type != OBJ_TREE)
3563 e = dereference(e, &oid);
3564 return e;
3565 }
3566
3567 static void print_ls(int mode, const unsigned char *hash, const char *path)
3568 {
3569 static struct strbuf line = STRBUF_INIT;
3570
3571 /* See show_tree(). */
3572 const char *type =
3573 S_ISGITLINK(mode) ? commit_type :
3574 S_ISDIR(mode) ? tree_type :
3575 blob_type;
3576
3577 if (!mode) {
3578 /* missing SP path LF */
3579 strbuf_reset(&line);
3580 strbuf_addstr(&line, "missing ");
3581 quote_c_style(path, &line, NULL, 0);
3582 strbuf_addch(&line, '\n');
3583 } else {
3584 /* mode SP type SP object_name TAB path LF */
3585 strbuf_reset(&line);
3586 strbuf_addf(&line, "%06o %s %s\t",
3587 mode & ~NO_DELTA, type, hash_to_hex(hash));
3588 quote_c_style(path, &line, NULL, 0);
3589 strbuf_addch(&line, '\n');
3590 }
3591 cat_blob_write(line.buf, line.len);
3592 }
3593
3594 static void parse_ls(struct fast_import_state *state UNUSED, const char *p, struct branch *b)
3595 {
3596 static struct strbuf path = STRBUF_INIT;
3597 struct tree_entry *root = NULL;
3598 struct tree_entry leaf = {NULL};
3599
3600 /* ls SP (<tree-ish> SP)? <path> */
3601 if (*p == '"') {
3602 if (!b)
3603 die(_("not in a commit: %s"), command_buf.buf);
3604 root = &b->branch_tree;
3605 } else {
3606 struct object_entry *e = parse_treeish_dataref(&p);
3607 root = new_tree_entry();
3608 oidcpy(&root->versions[1].oid, &e->idx.oid);
3609 if (!is_null_oid(&root->versions[1].oid))
3610 root->versions[1].mode = S_IFDIR;
3611 load_tree(root);
3612 }
3613 strbuf_reset(&path);
3614 parse_path_eol(&path, p, "path");
3615 tree_content_get(root, path.buf, &leaf, 1);
3616 /*
3617 * A directory in preparation would have a sha1 of zero
3618 * until it is saved. Save, for simplicity.
3619 */
3620 if (S_ISDIR(leaf.versions[1].mode))
3621 store_tree(&leaf);
3622
3623 print_ls(leaf.versions[1].mode, leaf.versions[1].oid.hash, path.buf);
3624 if (leaf.tree)
3625 release_tree_content_recursive(leaf.tree);
3626 if (!b || root != &b->branch_tree)
3627 release_tree_entry(root);
3628 }
3629
3630 static void checkpoint(void)
3631 {
3632 checkpoint_requested = 0;
3633 if (object_count) {
3634 cycle_packfile();
3635 }
3636 dump_branches();
3637 dump_tags();
3638 dump_marks();
3639 }
3640
3641 static void parse_checkpoint(struct fast_import_state *state UNUSED)
3642 {
3643 checkpoint_requested = 1;
3644 skip_optional_lf();
3645 }
3646
3647 static void parse_progress(struct fast_import_state *state UNUSED)
3648 {
3649 fwrite(command_buf.buf, 1, command_buf.len, stdout);
3650 fputc('\n', stdout);
3651 fflush(stdout);
3652 skip_optional_lf();
3653 }
3654
3655 static void parse_alias(struct fast_import_state *state)
3656 {
3657 struct object_entry *e;
3658 struct branch b;
3659
3660 skip_optional_lf();
3661 read_next_command(state);
3662
3663 /* mark ... */
3664 parse_mark(state);
3665 if (!next_mark)
3666 die(_("expected 'mark' command, got %s"), command_buf.buf);
3667
3668 /* to ... */
3669 memset(&b, 0, sizeof(b));
3670 if (!parse_objectish_with_prefix(state, &b, "to "))
3671 die(_("expected 'to' command, got %s"), command_buf.buf);
3672 e = find_object(&b.oid);
3673 assert(e);
3674 insert_mark(&marks, next_mark, e);
3675 }
3676
3677 static char* make_fast_import_path(struct fast_import_state *state, const char *path)
3678 {
3679 if (!relative_marks_paths || is_absolute_path(path))
3680 return prefix_filename(state->prefix, path);
3681 return repo_git_path(the_repository, "info/fast-import/%s", path);
3682 }
3683
3684 static void option_import_marks(struct fast_import_state *state, const char *marks,
3685 int from_stream, int ignore_missing)
3686 {
3687 if (import_marks_file) {
3688 if (from_stream)
3689 die(_("only one import-marks command allowed per stream"));
3690
3691 /* read previous mark file */
3692 if(!import_marks_file_from_stream)
3693 read_marks();
3694 }
3695
3696 free(import_marks_file);
3697 import_marks_file = make_fast_import_path(state, marks);
3698 import_marks_file_from_stream = from_stream;
3699 import_marks_file_ignore_missing = ignore_missing;
3700 }
3701
3702 static void option_date_format(const char *fmt)
3703 {
3704 if (!strcmp(fmt, "raw"))
3705 whenspec = WHENSPEC_RAW;
3706 else if (!strcmp(fmt, "raw-permissive"))
3707 whenspec = WHENSPEC_RAW_PERMISSIVE;
3708 else if (!strcmp(fmt, "rfc2822"))
3709 whenspec = WHENSPEC_RFC2822;
3710 else if (!strcmp(fmt, "now"))
3711 whenspec = WHENSPEC_NOW;
3712 else
3713 die(_("unknown --date-format argument %s"), fmt);
3714 }
3715
3716 static unsigned long ulong_arg(const char *option, const char *arg)
3717 {
3718 char *endptr;
3719 unsigned long rv = strtoul(arg, &endptr, 0);
3720 if (strchr(arg, '-') || endptr == arg || *endptr)
3721 die(_("%s: argument must be a non-negative integer"), option);
3722 return rv;
3723 }
3724
3725 static void option_depth(const char *depth)
3726 {
3727 max_depth = ulong_arg("--depth", depth);
3728 if (max_depth > MAX_DEPTH)
3729 die(_("--depth cannot exceed %u"), MAX_DEPTH);
3730 }
3731
3732 static void option_active_branches(const char *branches)
3733 {
3734 max_active_branches = ulong_arg("--active-branches", branches);
3735 }
3736
3737 static void option_export_marks(struct fast_import_state *state, const char *marks)
3738 {
3739 free(export_marks_file);
3740 export_marks_file = make_fast_import_path(state, marks);
3741 }
3742
3743 static void option_cat_blob_fd(struct fast_import_state *state UNUSED, const char *fd)
3744 {
3745 unsigned long n = ulong_arg("--cat-blob-fd", fd);
3746 if (n > (unsigned long) INT_MAX)
3747 die(_("--cat-blob-fd cannot exceed %d"), INT_MAX);
3748 cat_blob_fd = (int) n;
3749 }
3750
3751 static void option_export_pack_edges(struct fast_import_state *state, const char *edges)
3752 {
3753 char *fn = prefix_filename(state->prefix, edges);
3754 if (pack_edges)
3755 fclose(pack_edges);
3756 pack_edges = xfopen(fn, "a");
3757 free(fn);
3758 }
3759
3760 static void option_rewrite_submodules(struct fast_import_state *state, const char *arg, struct string_list *list)
3761 {
3762 struct mark_set *ms;
3763 FILE *fp;
3764 char *s = xstrdup(arg);
3765 char *f = strchr(s, ':');
3766 if (!f)
3767 die(_("expected format name:filename for submodule rewrite option"));
3768 *f = '\0';
3769 f++;
3770 CALLOC_ARRAY(ms, 1);
3771
3772 f = prefix_filename(state->prefix, f);
3773 fp = fopen(f, "r");
3774 if (!fp)
3775 die_errno(_("cannot read '%s'"), f);
3776 read_mark_file(&ms, fp, insert_oid_entry);
3777 fclose(fp);
3778 free(f);
3779
3780 string_list_insert(list, s)->util = ms;
3781
3782 free(s);
3783 }
3784
3785 static int parse_one_option(struct fast_import_state *state, const char *option)
3786 {
3787 if (skip_prefix(option, "max-pack-size=", &option)) {
3788 unsigned long v;
3789 if (!git_parse_ulong(option, &v))
3790 return 0;
3791 if (v < 8192) {
3792 warning(_("max-pack-size is now in bytes, assuming --max-pack-size=%lum"), v);
3793 v *= 1024 * 1024;
3794 } else if (v < 1024 * 1024) {
3795 warning(_("minimum max-pack-size is 1 MiB"));
3796 v = 1024 * 1024;
3797 }
3798 max_packsize = v;
3799 } else if (skip_prefix(option, "big-file-threshold=", &option)) {
3800 unsigned long v;
3801 if (!git_parse_ulong(option, &v))
3802 return 0;
3803 repo_settings_set_big_file_threshold(the_repository, v);
3804 } else if (skip_prefix(option, "depth=", &option)) {
3805 option_depth(option);
3806 } else if (skip_prefix(option, "active-branches=", &option)) {
3807 option_active_branches(option);
3808 } else if (skip_prefix(option, "export-pack-edges=", &option)) {
3809 option_export_pack_edges(state, option);
3810 } else if (skip_prefix(option, "signed-commits=", &option)) {
3811 if (parse_sign_mode(option, &signed_commit_mode, &signed_commit_keyid))
3812 usagef(_("unknown --signed-commits mode '%s'"), option);
3813 } else if (skip_prefix(option, "signed-tags=", &option)) {
3814 if (parse_sign_mode(option, &signed_tag_mode, &signed_tag_keyid))
3815 usagef(_("unknown --signed-tags mode '%s'"), option);
3816 } else if (!strcmp(option, "quiet")) {
3817 show_stats = 0;
3818 quiet = 1;
3819 } else if (!strcmp(option, "stats")) {
3820 show_stats = 1;
3821 } else if (!strcmp(option, "allow-unsafe-features")) {
3822 ; /* already handled during early option parsing */
3823 } else {
3824 return 0;
3825 }
3826
3827 return 1;
3828 }
3829
3830 static void check_unsafe_feature(struct fast_import_state *state, const char *feature, int from_stream)
3831 {
3832 if (from_stream && !state->allow_unsafe_features)
3833 die(_("feature '%s' forbidden in input without --allow-unsafe-features"),
3834 feature);
3835 }
3836
3837 static int parse_one_feature(struct fast_import_state *state, const char *feature, int from_stream)
3838 {
3839 const char *arg;
3840
3841 if (skip_prefix(feature, "date-format=", &arg)) {
3842 option_date_format(arg);
3843 } else if (skip_prefix(feature, "import-marks=", &arg)) {
3844 check_unsafe_feature(state, "import-marks", from_stream);
3845 option_import_marks(state, arg, from_stream, 0);
3846 } else if (skip_prefix(feature, "import-marks-if-exists=", &arg)) {
3847 check_unsafe_feature(state, "import-marks-if-exists", from_stream);
3848 option_import_marks(state, arg, from_stream, 1);
3849 } else if (skip_prefix(feature, "export-marks=", &arg)) {
3850 check_unsafe_feature(state, feature, from_stream);
3851 option_export_marks(state, arg);
3852 } else if (!strcmp(feature, "alias")) {
3853 ; /* Don't die - this feature is supported */
3854 } else if (skip_prefix(feature, "rewrite-submodules-to=", &arg)) {
3855 option_rewrite_submodules(state, arg, &sub_marks_to);
3856 } else if (skip_prefix(feature, "rewrite-submodules-from=", &arg)) {
3857 option_rewrite_submodules(state, arg, &sub_marks_from);
3858 } else if (!strcmp(feature, "get-mark")) {
3859 ; /* Don't die - this feature is supported */
3860 } else if (!strcmp(feature, "cat-blob")) {
3861 ; /* Don't die - this feature is supported */
3862 } else if (!strcmp(feature, "relative-marks")) {
3863 relative_marks_paths = 1;
3864 } else if (!strcmp(feature, "no-relative-marks")) {
3865 relative_marks_paths = 0;
3866 } else if (!strcmp(feature, "done")) {
3867 require_explicit_termination = 1;
3868 } else if (!strcmp(feature, "force")) {
3869 force_update = 1;
3870 } else if (!strcmp(feature, "notes") || !strcmp(feature, "ls")) {
3871 ; /* do nothing; we have the feature */
3872 } else {
3873 return 0;
3874 }
3875
3876 return 1;
3877 }
3878
3879 static void parse_feature(struct fast_import_state *state, const char *feature)
3880 {
3881 if (state->seen_data_command)
3882 die(_("got feature command '%s' after data command"), feature);
3883
3884 if (parse_one_feature(state, feature, 1))
3885 return;
3886
3887 die(_("this version of fast-import does not support feature %s."), feature);
3888 }
3889
3890 static void parse_option(struct fast_import_state *state, const char *option)
3891 {
3892 if (state->seen_data_command)
3893 die(_("got option command '%s' after data command"), option);
3894
3895 if (parse_one_option(state, option))
3896 return;
3897
3898 die(_("this version of fast-import does not support option: %s"), option);
3899 }
3900
3901 static void git_pack_config(void)
3902 {
3903 int indexversion_value;
3904 int limit;
3905 unsigned long packsizelimit_value;
3906
3907 if (!repo_config_get_ulong(the_repository, "pack.depth", &max_depth)) {
3908 if (max_depth > MAX_DEPTH)
3909 max_depth = MAX_DEPTH;
3910 }
3911 if (!repo_config_get_int(the_repository, "pack.indexversion", &indexversion_value)) {
3912 pack_idx_opts.version = indexversion_value;
3913 if (pack_idx_opts.version > 2)
3914 git_die_config(the_repository, "pack.indexversion",
3915 "bad pack.indexVersion=%"PRIu32, pack_idx_opts.version);
3916 }
3917 if (!repo_config_get_ulong(the_repository, "pack.packsizelimit", &packsizelimit_value))
3918 max_packsize = packsizelimit_value;
3919
3920 if (!repo_config_get_int(the_repository, "fastimport.unpacklimit", &limit))
3921 unpack_limit = limit;
3922 else if (!repo_config_get_int(the_repository, "transfer.unpacklimit", &limit))
3923 unpack_limit = limit;
3924
3925 repo_config(the_repository, git_default_config, NULL);
3926 }
3927
3928 static const char *const fast_import_usage[] = {
3929 N_("git fast-import [<options>]"),
3930 NULL
3931 };
3932
3933 static void parse_argv(struct fast_import_state *state)
3934 {
3935 unsigned int i;
3936
3937 for (i = 1; i < state->argc; i++) {
3938 const char *a = state->argv[i];
3939
3940 if (*a != '-' || !strcmp(a, "--"))
3941 break;
3942
3943 if (!skip_prefix(a, "--", &a))
3944 die(_("unknown option %s"), a);
3945
3946 if (parse_one_option(state, a))
3947 continue;
3948
3949 if (parse_one_feature(state, a, 0))
3950 continue;
3951
3952 if (skip_prefix(a, "cat-blob-fd=", &a)) {
3953 option_cat_blob_fd(state, a);
3954 continue;
3955 }
3956
3957 die(_("unknown option --%s"), a);
3958 }
3959 if (i != state->argc)
3960 usage_with_options(fast_import_usage, state->option);
3961
3962 state->seen_data_command = 1;
3963 if (import_marks_file)
3964 read_marks();
3965 build_mark_map(&sub_marks_from, &sub_marks_to);
3966 }
3967
3968 int cmd_fast_import(int argc,
3969 const char **argv,
3970 const char *prefix,
3971 struct repository *repo)
3972 {
3973 struct fast_import_state state;
3974
3975 unsigned long pack_size_limit, big_file_threshold, depth, active_branches;
3976 char *edges, *signed_commits, *signed_tags, *date_format, *import_marks;
3977 char *import_marks_if_exists, *export_marks, *submodules_from, *submodules_to;
3978 int opt_quiet, opt_show_stats, opt_relative_marks, opt_force, opt_done;
3979 int opt_allow_unsafe;
3980 int cat_blob;
3981
3982 /*
3983 * NEEDSWORK: For now this is used only to render
3984 * `-h`/`--help-all` usage messages. The actual parsing is
3985 * done by parse_one_option()/parse_one_feature().
3986 */
3987 struct option fast_import_options[] = {
3988 OPT_GROUP(N_("Common")),
3989 OPT_STRING_F(0, "date-format", &date_format, N_("fmt"),
3990 N_("format of the commit/tag dates"), PARSE_OPT_NONEG),
3991 OPT_BOOL_F(0, "stats", &opt_show_stats,
3992 N_("display some basic statistics (objects, packfiles and memory)"),
3993 PARSE_OPT_NONEG),
3994 OPT_BOOL_F(0, "quiet", &opt_quiet,
3995 N_("disable the output shown by --stats"), PARSE_OPT_NONEG),
3996 OPT_BOOL_F(0, "force", &opt_force,
3997 N_("force updating modified existing branches"), PARSE_OPT_NONEG),
3998 OPT_BOOL_F(0, "done", &opt_done,
3999 N_("require a terminating 'done' command"), PARSE_OPT_NONEG),
4000 OPT_UNSIGNED(0, "max-pack-size", &pack_size_limit,
4001 N_("maximum size of each output pack file")),
4002 OPT_UNSIGNED(0, "big-file-threshold", &big_file_threshold,
4003 N_("maximum size of a blob that will be deltified")),
4004 OPT_UNSIGNED(0, "depth", &depth,
4005 N_("maximum delta depth")),
4006 OPT_UNSIGNED(0, "active-branches", &active_branches,
4007 N_("maximum number of branches to maintain active")),
4008 OPT_GROUP(N_("Marks")),
4009 OPT_STRING_F(0, "import-marks", &import_marks, N_("file"),
4010 N_("import marks from <file>"), PARSE_OPT_NONEG),
4011 OPT_STRING_F(0, "import-marks-if-exists", &import_marks_if_exists, N_("file"),
4012 N_("import marks from <file> if it exists"), PARSE_OPT_NONEG),
4013 OPT_STRING_F(0, "export-marks", &export_marks, N_("file"),
4014 N_("dump marks to <file>"), PARSE_OPT_NONEG),
4015 OPT_BOOL(0, "relative-marks", &opt_relative_marks,
4016 N_("are --(import|export)-marks= paths relative to '.git/info/fast-import'?")),
4017 OPT_GROUP(N_("Submodule rewrite")),
4018 OPT_STRING_F(0, "rewrite-submodules-from", &submodules_from, N_("name:filename"),
4019 N_("rewrite object IDs for submodule <name> from <filename>"),
4020 PARSE_OPT_NONEG),
4021 OPT_STRING_F(0, "rewrite-submodules-to", &submodules_to, N_("name:filename"),
4022 N_("rewrite object IDs for submodule <name> to <filename>"),
4023 PARSE_OPT_NONEG),
4024 OPT_GROUP(N_("Signing")),
4025 OPT_STRING_F(0, "signed-commits", &signed_commits, N_("mode"),
4026 N_("how to handle signed commits"),
4027 PARSE_OPT_NONEG),
4028 OPT_STRING_F(0, "signed-tags", &signed_tags, N_("mode"),
4029 N_("how to handle signed tags"),
4030 PARSE_OPT_NONEG),
4031 OPT_HIDDEN_GROUP(N_("Advanced")),
4032 OPT_BOOL_F(0, "allow-unsafe-features", &opt_allow_unsafe,
4033 N_("allow unsafe mark commands from the stream"),
4034 PARSE_OPT_HIDDEN | PARSE_OPT_NONEG),
4035 OPT_STRING_F(0, "export-pack-edges", &edges, N_("file"),
4036 N_("dump edge commits to <file>"),
4037 PARSE_OPT_HIDDEN | PARSE_OPT_NONEG),
4038 OPT_INTEGER_F(0, "cat-blob-fd", &cat_blob,
4039 N_("write some responses to <fd> instead of stdout"),
4040 PARSE_OPT_HIDDEN | PARSE_OPT_NONEG),
4041 OPT_END()
4042 };
4043
4044 show_usage_with_options_if_asked(argc, argv, fast_import_usage, fast_import_options);
4045
4046 fast_import_state_init(&state, argc, argv, prefix, fast_import_options);
4047
4048 reset_pack_idx_option(&pack_idx_opts);
4049 git_pack_config();
4050
4051 alloc_objects(object_entry_alloc);
4052 strbuf_init(&command_buf, 0);
4053 CALLOC_ARRAY(atom_table, atom_table_sz);
4054 CALLOC_ARRAY(branch_table, branch_table_sz);
4055 CALLOC_ARRAY(avail_tree_table, avail_tree_table_sz);
4056 marks = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set));
4057
4058 hashmap_init(&object_table, object_entry_hashcmp, NULL, 0);
4059
4060 /*
4061 * We don't parse most options until after we've seen the set of
4062 * "feature" lines at the start of the stream (which allows the command
4063 * line to override stream data). But we must do an early parse of any
4064 * command-line options that impact how we interpret the feature lines.
4065 */
4066 for (int i = 1; i < argc; i++) {
4067 const char *arg = argv[i];
4068 if (*arg != '-' || !strcmp(arg, "--"))
4069 break;
4070 if (!strcmp(arg, "--allow-unsafe-features"))
4071 state.allow_unsafe_features = 1;
4072 }
4073
4074 rc_free = mem_pool_alloc(&fi_mem_pool, cmd_save * sizeof(*rc_free));
4075 for (unsigned int i = 0; i < (cmd_save - 1); i++)
4076 rc_free[i].next = &rc_free[i + 1];
4077 rc_free[cmd_save - 1].next = NULL;
4078
4079 start_packfile();
4080 set_die_routine(die_nicely);
4081 set_checkpoint_signal();
4082 while (read_next_command(&state) != EOF) {
4083 const char *v;
4084 if (!strcmp("blob", command_buf.buf))
4085 parse_new_blob(&state);
4086 else if (skip_prefix(command_buf.buf, "commit ", &v))
4087 parse_new_commit(&state, v);
4088 else if (skip_prefix(command_buf.buf, "tag ", &v))
4089 parse_new_tag(&state, v);
4090 else if (skip_prefix(command_buf.buf, "reset ", &v))
4091 parse_reset_branch(&state, v);
4092 else if (skip_prefix(command_buf.buf, "ls ", &v))
4093 parse_ls(&state, v, NULL);
4094 else if (skip_prefix(command_buf.buf, "cat-blob ", &v))
4095 parse_cat_blob(&state, v);
4096 else if (skip_prefix(command_buf.buf, "get-mark ", &v))
4097 parse_get_mark(&state, v);
4098 else if (!strcmp("checkpoint", command_buf.buf))
4099 parse_checkpoint(&state);
4100 else if (!strcmp("done", command_buf.buf))
4101 break;
4102 else if (!strcmp("alias", command_buf.buf))
4103 parse_alias(&state);
4104 else if (starts_with(command_buf.buf, "progress "))
4105 parse_progress(&state);
4106 else if (skip_prefix(command_buf.buf, "feature ", &v))
4107 parse_feature(&state, v);
4108 else if (skip_prefix(command_buf.buf, "option git ", &v))
4109 parse_option(&state, v);
4110 else if (starts_with(command_buf.buf, "option "))
4111 /* ignore non-git options*/;
4112 else
4113 die(_("unsupported command: %s"), command_buf.buf);
4114
4115 if (checkpoint_requested)
4116 checkpoint();
4117 }
4118
4119 /* argv hasn't been parsed yet, do so */
4120 if (!state.seen_data_command)
4121 parse_argv(&state);
4122
4123 if (require_explicit_termination && feof(stdin))
4124 die(_("stream ends early"));
4125
4126 end_packfile();
4127
4128 dump_branches();
4129 dump_tags();
4130 unkeep_all_packs();
4131 dump_marks();
4132
4133 if (pack_edges)
4134 fclose(pack_edges);
4135
4136 if (show_stats) {
4137 uintmax_t total_count = 0, duplicate_count = 0;
4138 for (size_t i = 0; i < ARRAY_SIZE(object_count_by_type); i++)
4139 total_count += object_count_by_type[i];
4140 for (size_t i = 0; i < ARRAY_SIZE(duplicate_count_by_type); i++)
4141 duplicate_count += duplicate_count_by_type[i];
4142
4143 fprintf(stderr, "%s statistics:\n", argv[0]);
4144 fprintf(stderr, "---------------------------------------------------------------------\n");
4145 fprintf(stderr, "Alloc'd objects: %10" PRIuMAX "\n", alloc_count);
4146 fprintf(stderr, "Total objects: %10" PRIuMAX " (%10" PRIuMAX " duplicates )\n", total_count, duplicate_count);
4147 fprintf(stderr, " blobs : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB], delta_count_attempts_by_type[OBJ_BLOB]);
4148 fprintf(stderr, " trees : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE], delta_count_attempts_by_type[OBJ_TREE]);
4149 fprintf(stderr, " commits: %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT], delta_count_attempts_by_type[OBJ_COMMIT]);
4150 fprintf(stderr, " tags : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG], delta_count_attempts_by_type[OBJ_TAG]);
4151 fprintf(stderr, "Total branches: %10lu (%10lu loads )\n", branch_count, branch_load_count);
4152 fprintf(stderr, " marks: %10" PRIuMAX " (%10" PRIuMAX " unique )\n", (((uintmax_t)1) << marks->shift) * 1024, marks_set_count);
4153 fprintf(stderr, " atoms: %10u\n", atom_cnt);
4154 fprintf(stderr, "Memory total: %10" PRIuMAX " KiB\n", (tree_entry_allocd + fi_mem_pool.pool_alloc + alloc_count*sizeof(struct object_entry))/1024);
4155 fprintf(stderr, " pools: %10lu KiB\n", (unsigned long)((tree_entry_allocd + fi_mem_pool.pool_alloc) /1024));
4156 fprintf(stderr, " objects: %10" PRIuMAX " KiB\n", (alloc_count*sizeof(struct object_entry))/1024);
4157 fprintf(stderr, "---------------------------------------------------------------------\n");
4158 pack_report(repo);
4159 fprintf(stderr, "---------------------------------------------------------------------\n");
4160 fprintf(stderr, "\n");
4161 }
4162
4163 return failure ? 1 : 0;
4164 }