@matheusbsbs / matheus-bagliano / commits / aa7e44bf57
Fork 0

Unify write_index_file functions

This patch unifies the write_index_file functions in builtin-pack-objects.c and index-pack.c. As the name "index" is overloaded in git, move in the direction of using "idx" and "pack idx" when refering to the pack index. There should be no change in functionality. Signed-off-by: Geert Bosch <bosch@gnat.com> Acked-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Geert Bosch committed Jun 1, 2007 at 15:18 UTC aa7e44bf571717aeb4435e6605391d51f3117047
4 files changed +245 -341
builtin-pack-objects.c
+52 -170
@@ -23,10 +23,9 @@ git-pack-objects [{ -q | --progress | --all-progress }] [--max-pack-size=N] \n\
23 [--stdout | base-name] [<ref-list | <object-list]";
24
25 struct object_entry {
26 - unsigned char sha1[20];
27 - uint32_t crc32; /* crc of raw pack data for this object */
28 - off_t offset; /* offset into the final pack file */
26 + struct pack_idx_entry idx;
27 unsigned long size; /* uncompressed size */
28 +
29 unsigned int hash; /* name hint hash */
30 unsigned int depth; /* delta depth */
31 struct packed_git *in_pack; /* already in pack */
@@ -66,7 +65,6 @@ static int allow_ofs_delta;
65 static const char *pack_tmp_name, *idx_tmp_name;
66 static char tmpname[PATH_MAX];
67 static const char *base_name;
69 -static unsigned char pack_file_sha1[20];
68 static int progress = 1;
69 static int window = 10;
70 static uint32_t pack_size_limit;
@@ -246,11 +244,11 @@ static void *delta_against(void *buf, unsigned long size, struct object_entry *e
244 {
245 unsigned long othersize, delta_size;
246 enum object_type type;
249 - void *otherbuf = read_sha1_file(entry->delta->sha1, &type, &othersize);
247 + void *otherbuf = read_sha1_file(entry->delta->idx.sha1, &type, &othersize);
248 void *delta_buf;
249
250 if (!otherbuf)
253 - die("unable to read %s", sha1_to_hex(entry->delta->sha1));
251 + die("unable to read %s", sha1_to_hex(entry->delta->idx.sha1));
252 delta_buf = diff_delta(otherbuf, othersize,
253 buf, size, &delta_size, 0);
254 if (!delta_buf || delta_size != entry->delta_size)
@@ -379,11 +377,11 @@ static unsigned long write_object(struct sha1file *f,
377 /* yes if unlimited packfile */
378 !pack_size_limit ? 1 :
379 /* no if base written to previous pack */
382 - entry->delta->offset == (off_t)-1 ? 0 :
380 + entry->delta->idx.offset == (off_t)-1 ? 0 :
381 /* otherwise double-check written to this
382 * pack, like we do below
383 */
386 - entry->delta->offset ? 1 : 0;
384 + entry->delta->idx.offset ? 1 : 0;
385
386 if (!pack_to_stdout)
387 crc32_begin(f);
@@ -411,22 +409,22 @@ static unsigned long write_object(struct sha1file *f,
409 unsigned long maxsize;
410 void *out;
411 if (!usable_delta) {
414 - buf = read_sha1_file(entry->sha1, &obj_type, &size);
412 + buf = read_sha1_file(entry->idx.sha1, &obj_type, &size);
413 if (!buf)
416 - die("unable to read %s", sha1_to_hex(entry->sha1));
414 + die("unable to read %s", sha1_to_hex(entry->idx.sha1));
415 } else if (entry->delta_data) {
416 size = entry->delta_size;
417 buf = entry->delta_data;
418 entry->delta_data = NULL;
421 - obj_type = (allow_ofs_delta && entry->delta->offset) ?
419 + obj_type = (allow_ofs_delta && entry->delta->idx.offset) ?
420 OBJ_OFS_DELTA : OBJ_REF_DELTA;
421 } else {
424 - buf = read_sha1_file(entry->sha1, &type, &size);
422 + buf = read_sha1_file(entry->idx.sha1, &type, &size);
423 if (!buf)
426 - die("unable to read %s", sha1_to_hex(entry->sha1));
424 + die("unable to read %s", sha1_to_hex(entry->idx.sha1));
425 buf = delta_against(buf, size, entry);
426 size = entry->delta_size;
429 - obj_type = (allow_ofs_delta && entry->delta->offset) ?
427 + obj_type = (allow_ofs_delta && entry->delta->idx.offset) ?
428 OBJ_OFS_DELTA : OBJ_REF_DELTA;
429 }
430 /* compress the data to store and put compressed length in datalen */
@@ -456,7 +454,7 @@ static unsigned long write_object(struct sha1file *f,
454 * encoding of the relative offset for the delta
455 * base from this object's position in the pack.
456 */
459 - off_t ofs = entry->offset - entry->delta->offset;
457 + off_t ofs = entry->idx.offset - entry->delta->idx.offset;
458 unsigned pos = sizeof(dheader) - 1;
459 dheader[pos] = ofs & 127;
460 while (ofs >>= 7)
@@ -480,7 +478,7 @@ static unsigned long write_object(struct sha1file *f,
478 return 0;
479 }
480 sha1write(f, header, hdrlen);
483 - sha1write(f, entry->delta->sha1, 20);
481 + sha1write(f, entry->delta->idx.sha1, 20);
482 hdrlen += 20;
483 } else {
484 if (limit && hdrlen + datalen + 20 >= limit) {
@@ -501,7 +499,7 @@ static unsigned long write_object(struct sha1file *f,
499 off_t offset;
500
501 if (entry->delta) {
504 - obj_type = (allow_ofs_delta && entry->delta->offset) ?
502 + obj_type = (allow_ofs_delta && entry->delta->idx.offset) ?
503 OBJ_OFS_DELTA : OBJ_REF_DELTA;
504 reused_delta++;
505 }
@@ -511,11 +509,11 @@ static unsigned long write_object(struct sha1file *f,
509 datalen = revidx[1].offset - offset;
510 if (!pack_to_stdout && p->index_version > 1 &&
511 check_pack_crc(p, &w_curs, offset, datalen, revidx->nr))
514 - die("bad packed object CRC for %s", sha1_to_hex(entry->sha1));
512 + die("bad packed object CRC for %s", sha1_to_hex(entry->idx.sha1));
513 offset += entry->in_pack_header_size;
514 datalen -= entry->in_pack_header_size;
515 if (obj_type == OBJ_OFS_DELTA) {
518 - off_t ofs = entry->offset - entry->delta->offset;
516 + off_t ofs = entry->idx.offset - entry->delta->idx.offset;
517 unsigned pos = sizeof(dheader) - 1;
518 dheader[pos] = ofs & 127;
519 while (ofs >>= 7)
@@ -529,7 +527,7 @@ static unsigned long write_object(struct sha1file *f,
527 if (limit && hdrlen + 20 + datalen + 20 >= limit)
528 return 0;
529 sha1write(f, header, hdrlen);
532 - sha1write(f, entry->delta->sha1, 20);
530 + sha1write(f, entry->delta->idx.sha1, 20);
531 hdrlen += 20;
532 } else {
533 if (limit && hdrlen + datalen + 20 >= limit)
@@ -539,7 +537,7 @@ static unsigned long write_object(struct sha1file *f,
537
538 if (!pack_to_stdout && p->index_version == 1 &&
539 check_pack_inflate(p, &w_curs, offset, datalen, entry->size))
542 - die("corrupt packed object for %s", sha1_to_hex(entry->sha1));
540 + die("corrupt packed object for %s", sha1_to_hex(entry->idx.sha1));
541 copy_pack_data(f, p, &w_curs, offset, datalen);
542 unuse_pack(&w_curs);
543 reused++;
@@ -548,7 +546,7 @@ static unsigned long write_object(struct sha1file *f,
546 written_delta++;
547 written++;
548 if (!pack_to_stdout)
551 - entry->crc32 = crc32_end(f);
549 + entry->idx.crc32 = crc32_end(f);
550 return hdrlen + datalen;
551 }
552
@@ -559,7 +557,7 @@ static off_t write_one(struct sha1file *f,
557 unsigned long size;
558
559 /* offset is non zero if object is written already. */
562 - if (e->offset || e->preferred_base)
560 + if (e->idx.offset || e->preferred_base)
561 return offset;
562
563 /* if we are deltified, write out base object first. */
@@ -569,10 +567,10 @@ static off_t write_one(struct sha1file *f,
567 return 0;
568 }
569
572 - e->offset = offset;
570 + e->idx.offset = offset;
571 size = write_object(f, e, offset);
572 if (!size) {
575 - e->offset = 0;
573 + e->idx.offset = 0;
574 return 0;
575 }
576 written_list[nr_written++] = e;
@@ -589,8 +587,7 @@ static int open_object_dir_tmp(const char *path)
587 return mkstemp(tmpname);
588 }
589
592 -/* forward declarations for write_pack_file */
593 -static void write_index_file(off_t last_obj_offset, unsigned char *sha1);
590 +/* forward declaration for write_pack_file */
591 static int adjust_perm(const char *path, mode_t mode);
592
593 static void write_pack_file(void)
@@ -607,6 +604,8 @@ static void write_pack_file(void)
604 written_list = xmalloc(nr_objects * sizeof(struct object_entry *));
605
606 do {
607 + unsigned char sha1[20];
608 +
609 if (pack_to_stdout) {
610 f = sha1fd(1, "<stdout>");
611 } else {
@@ -638,23 +637,23 @@ static void write_pack_file(void)
637 * If so, rewrite it like in fast-import
638 */
639 if (pack_to_stdout || nr_written == nr_remaining) {
641 - sha1close(f, pack_file_sha1, 1);
640 + sha1close(f, sha1, 1);
641 } else {
643 - sha1close(f, pack_file_sha1, 0);
644 - fixup_pack_header_footer(f->fd, pack_file_sha1, pack_tmp_name, nr_written);
642 + sha1close(f, sha1, 0);
643 + fixup_pack_header_footer(f->fd, sha1, pack_tmp_name, nr_written);
644 close(f->fd);
645 }
646
647 if (!pack_to_stdout) {
649 - unsigned char object_list_sha1[20];
648 mode_t mode = umask(0);
649
650 umask(mode);
651 mode = 0444 & ~mode;
652
655 - write_index_file(last_obj_offset, object_list_sha1);
653 + idx_tmp_name = write_idx_file(NULL,
654 + (struct pack_idx_entry **) written_list, nr_written, sha1);
655 snprintf(tmpname, sizeof(tmpname), "%s-%s.pack",
657 - base_name, sha1_to_hex(object_list_sha1));
656 + base_name, sha1_to_hex(sha1));
657 if (adjust_perm(pack_tmp_name, mode))
658 die("unable to make temporary pack file readable: %s",
659 strerror(errno));
@@ -662,19 +661,19 @@ static void write_pack_file(void)
661 die("unable to rename temporary pack file: %s",
662 strerror(errno));
663 snprintf(tmpname, sizeof(tmpname), "%s-%s.idx",
665 - base_name, sha1_to_hex(object_list_sha1));
664 + base_name, sha1_to_hex(sha1));
665 if (adjust_perm(idx_tmp_name, mode))
666 die("unable to make temporary index file readable: %s",
667 strerror(errno));
668 if (rename(idx_tmp_name, tmpname))
669 die("unable to rename temporary index file: %s",
670 strerror(errno));
672 - puts(sha1_to_hex(object_list_sha1));
671 + puts(sha1_to_hex(sha1));
672 }
673
674 /* mark written objects as written to previous pack */
675 for (j = 0; j < nr_written; j++) {
677 - written_list[j]->offset = (off_t)-1;
676 + written_list[j]->idx.offset = (off_t)-1;
677 }
678 nr_remaining -= nr_written;
679 } while (nr_remaining && i < nr_objects);
@@ -692,129 +691,12 @@ static void write_pack_file(void)
691 */
692 for (j = 0; i < nr_objects; i++) {
693 struct object_entry *e = objects + i;
695 - j += !e->offset && !e->preferred_base;
694 + j += !e->idx.offset && !e->preferred_base;
695 }
696 if (j)
697 die("wrote %u objects as expected but %u unwritten", written, j);
698 }
699
701 -static int sha1_sort(const void *_a, const void *_b)
702 -{
703 - const struct object_entry *a = *(struct object_entry **)_a;
704 - const struct object_entry *b = *(struct object_entry **)_b;
705 - return hashcmp(a->sha1, b->sha1);
706 -}
707 -
708 -static uint32_t index_default_version = 1;
709 -static uint32_t index_off32_limit = 0x7fffffff;
710 -
711 -static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
712 -{
713 - struct sha1file *f;
714 - struct object_entry **sorted_by_sha, **list, **last;
715 - uint32_t array[256];
716 - uint32_t i, index_version;
717 - SHA_CTX ctx;
718 -
719 - int fd = open_object_dir_tmp("tmp_idx_XXXXXX");
720 - if (fd < 0)
721 - die("unable to create %s: %s\n", tmpname, strerror(errno));
722 - idx_tmp_name = xstrdup(tmpname);
723 - f = sha1fd(fd, idx_tmp_name);
724 -
725 - if (nr_written) {
726 - sorted_by_sha = written_list;
727 - qsort(sorted_by_sha, nr_written, sizeof(*sorted_by_sha), sha1_sort);
728 - list = sorted_by_sha;
729 - last = sorted_by_sha + nr_written;
730 - } else
731 - sorted_by_sha = list = last = NULL;
732 -
733 - /* if last object's offset is >= 2^31 we should use index V2 */
734 - index_version = (last_obj_offset >> 31) ? 2 : index_default_version;
735 -
736 - /* index versions 2 and above need a header */
737 - if (index_version >= 2) {
738 - struct pack_idx_header hdr;
739 - hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
740 - hdr.idx_version = htonl(index_version);
741 - sha1write(f, &hdr, sizeof(hdr));
742 - }
743 -
744 - /*
745 - * Write the first-level table (the list is sorted,
746 - * but we use a 256-entry lookup to be able to avoid
747 - * having to do eight extra binary search iterations).
748 - */
749 - for (i = 0; i < 256; i++) {
750 - struct object_entry **next = list;
751 - while (next < last) {
752 - struct object_entry *entry = *next;
753 - if (entry->sha1[0] != i)
754 - break;
755 - next++;
756 - }
757 - array[i] = htonl(next - sorted_by_sha);
758 - list = next;
759 - }
760 - sha1write(f, array, 256 * 4);
761 -
762 - /* Compute the SHA1 hash of sorted object names. */
763 - SHA1_Init(&ctx);
764 -
765 - /* Write the actual SHA1 entries. */
766 - list = sorted_by_sha;
767 - for (i = 0; i < nr_written; i++) {
768 - struct object_entry *entry = *list++;
769 - if (index_version < 2) {
770 - uint32_t offset = htonl(entry->offset);
771 - sha1write(f, &offset, 4);
772 - }
773 - sha1write(f, entry->sha1, 20);
774 - SHA1_Update(&ctx, entry->sha1, 20);
775 - }
776 -
777 - if (index_version >= 2) {
778 - unsigned int nr_large_offset = 0;
779 -
780 - /* write the crc32 table */
781 - list = sorted_by_sha;
782 - for (i = 0; i < nr_written; i++) {
783 - struct object_entry *entry = *list++;
784 - uint32_t crc32_val = htonl(entry->crc32);
785 - sha1write(f, &crc32_val, 4);
786 - }
787 -
788 - /* write the 32-bit offset table */
789 - list = sorted_by_sha;
790 - for (i = 0; i < nr_written; i++) {
791 - struct object_entry *entry = *list++;
792 - uint32_t offset = (entry->offset <= index_off32_limit) ?
793 - entry->offset : (0x80000000 | nr_large_offset++);
794 - offset = htonl(offset);
795 - sha1write(f, &offset, 4);
796 - }
797 -
798 - /* write the large offset table */
799 - list = sorted_by_sha;
800 - while (nr_large_offset) {
801 - struct object_entry *entry = *list++;
802 - uint64_t offset = entry->offset;
803 - if (offset > index_off32_limit) {
804 - uint32_t split[2];
805 - split[0] = htonl(offset >> 32);
806 - split[1] = htonl(offset & 0xffffffff);
807 - sha1write(f, split, 8);
808 - nr_large_offset--;
809 - }
810 - }
811 - }
812 -
813 - sha1write(f, pack_file_sha1, 20);
814 - sha1close(f, NULL, 1);
815 - SHA1_Final(sha1, &ctx);
816 -}
817 -
700 static int locate_object_entry_hash(const unsigned char *sha1)
701 {
702 int i;
@@ -822,7 +704,7 @@ static int locate_object_entry_hash(const unsigned char *sha1)
704 memcpy(&ui, sha1, sizeof(unsigned int));
705 i = ui % object_ix_hashsz;
706 while (0 < object_ix[i]) {
825 - if (!hashcmp(sha1, objects[object_ix[i] - 1].sha1))
707 + if (!hashcmp(sha1, objects[object_ix[i] - 1].idx.sha1))
708 return i;
709 if (++i == object_ix_hashsz)
710 i = 0;
@@ -854,7 +736,7 @@ static void rehash_objects(void)
736 object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz);
737 memset(object_ix, 0, sizeof(int) * object_ix_hashsz);
738 for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
857 - int ix = locate_object_entry_hash(oe->sha1);
739 + int ix = locate_object_entry_hash(oe->idx.sha1);
740 if (0 <= ix)
741 continue;
742 ix = -1 - ix;
@@ -948,7 +830,7 @@ static int add_object_entry(const unsigned char *sha1, enum object_type type,
830
831 entry = objects + nr_objects++;
832 memset(entry, 0, sizeof(*entry));
951 - hashcpy(entry->sha1, sha1);
833 + hashcpy(entry->idx.sha1, sha1);
834 entry->hash = hash;
835 if (type)
836 entry->type = type;
@@ -1268,13 +1150,13 @@ static void check_object(struct object_entry *entry)
1150 ofs += 1;
1151 if (!ofs || MSB(ofs, 7))
1152 die("delta base offset overflow in pack for %s",
1271 - sha1_to_hex(entry->sha1));
1153 + sha1_to_hex(entry->idx.sha1));
1154 c = buf[used_0++];
1155 ofs = (ofs << 7) + (c & 127);
1156 }
1157 if (ofs >= entry->in_pack_offset)
1158 die("delta base offset out of bound for %s",
1277 - sha1_to_hex(entry->sha1));
1159 + sha1_to_hex(entry->idx.sha1));
1160 ofs = entry->in_pack_offset - ofs;
1161 if (!no_reuse_delta && !entry->preferred_base)
1162 base_ref = find_packed_object_name(p, ofs);
@@ -1321,10 +1203,10 @@ static void check_object(struct object_entry *entry)
1203 unuse_pack(&w_curs);
1204 }
1205
1324 - entry->type = sha1_object_info(entry->sha1, &entry->size);
1206 + entry->type = sha1_object_info(entry->idx.sha1, &entry->size);
1207 if (entry->type < 0)
1208 die("unable to get type of object %s",
1327 - sha1_to_hex(entry->sha1));
1209 + sha1_to_hex(entry->idx.sha1));
1210 }
1211
1212 static int pack_offset_sort(const void *_a, const void *_b)
@@ -1334,7 +1216,7 @@ static int pack_offset_sort(const void *_a, const void *_b)
1216
1217 /* avoid filesystem trashing with loose objects */
1218 if (!a->in_pack && !b->in_pack)
1337 - return hashcmp(a->sha1, b->sha1);
1219 + return hashcmp(a->idx.sha1, b->idx.sha1);
1220
1221 if (a->in_pack < b->in_pack)
1222 return -1;
@@ -1463,16 +1345,16 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
1345
1346 /* Load data if not already done */
1347 if (!trg->data) {
1466 - trg->data = read_sha1_file(trg_entry->sha1, &type, &sz);
1348 + trg->data = read_sha1_file(trg_entry->idx.sha1, &type, &sz);
1349 if (sz != trg_size)
1350 die("object %s inconsistent object length (%lu vs %lu)",
1469 - sha1_to_hex(trg_entry->sha1), sz, trg_size);
1351 + sha1_to_hex(trg_entry->idx.sha1), sz, trg_size);
1352 }
1353 if (!src->data) {
1472 - src->data = read_sha1_file(src_entry->sha1, &type, &sz);
1354 + src->data = read_sha1_file(src_entry->idx.sha1, &type, &sz);
1355 if (sz != src_size)
1356 die("object %s inconsistent object length (%lu vs %lu)",
1475 - sha1_to_hex(src_entry->sha1), sz, src_size);
1357 + sha1_to_hex(src_entry->idx.sha1), sz, src_size);
1358 }
1359 if (!src->index) {
1360 src->index = create_delta_index(src->data, src_size);
@@ -1869,12 +1751,12 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
1751 }
1752 if (!prefixcmp(arg, "--index-version=")) {
1753 char *c;
1872 - index_default_version = strtoul(arg + 16, &c, 10);
1873 - if (index_default_version > 2)
1754 + pack_idx_default_version = strtoul(arg + 16, &c, 10);
1755 + if (pack_idx_default_version > 2)
1756 die("bad %s", arg);
1757 if (*c == ',')
1876 - index_off32_limit = strtoul(c+1, &c, 0);
1877 - if (*c || index_off32_limit & 0x80000000)
1758 + pack_idx_off32_limit = strtoul(c+1, &c, 0);
1759 + if (*c || pack_idx_off32_limit & 0x80000000)
1760 die("bad %s", arg);
1761 continue;
1762 }
index-pack.c
+37 -171
@@ -13,13 +13,11 @@ static const char index_pack_usage[] =
13
14 struct object_entry
15 {
16 - off_t offset;
16 + struct pack_idx_entry idx;
17 unsigned long size;
18 unsigned int hdr_size;
19 - uint32_t crc32;
19 enum object_type type;
20 enum object_type real_type;
22 - unsigned char sha1[20];
21 };
22
23 union delta_base {
@@ -197,7 +195,7 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
195 unsigned shift;
196 void *data;
197
200 - obj->offset = consumed_bytes;
198 + obj->idx.offset = consumed_bytes;
199 input_crc32 = crc32(0, Z_NULL, 0);
200
201 p = fill(1);
@@ -229,15 +227,15 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
227 while (c & 128) {
228 base_offset += 1;
229 if (!base_offset || MSB(base_offset, 7))
232 - bad_object(obj->offset, "offset value overflow for delta base object");
230 + bad_object(obj->idx.offset, "offset value overflow for delta base object");
231 p = fill(1);
232 c = *p;
233 use(1);
234 base_offset = (base_offset << 7) + (c & 127);
235 }
238 - delta_base->offset = obj->offset - base_offset;
239 - if (delta_base->offset >= obj->offset)
240 - bad_object(obj->offset, "delta base offset is out of bound");
236 + delta_base->offset = obj->idx.offset - base_offset;
237 + if (delta_base->offset >= obj->idx.offset)
238 + bad_object(obj->idx.offset, "delta base offset is out of bound");
239 break;
240 case OBJ_COMMIT:
241 case OBJ_TREE:
@@ -245,19 +243,19 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
243 case OBJ_TAG:
244 break;
245 default:
248 - bad_object(obj->offset, "unknown object type %d", obj->type);
246 + bad_object(obj->idx.offset, "unknown object type %d", obj->type);
247 }
250 - obj->hdr_size = consumed_bytes - obj->offset;
248 + obj->hdr_size = consumed_bytes - obj->idx.offset;
249
252 - data = unpack_entry_data(obj->offset, obj->size);
253 - obj->crc32 = input_crc32;
250 + data = unpack_entry_data(obj->idx.offset, obj->size);
251 + obj->idx.crc32 = input_crc32;
252 return data;
253 }
254
255 static void *get_data_from_pack(struct object_entry *obj)
256 {
259 - unsigned long from = obj[0].offset + obj[0].hdr_size;
260 - unsigned long len = obj[1].offset - from;
257 + unsigned long from = obj[0].idx.offset + obj[0].hdr_size;
258 + unsigned long len = obj[1].idx.offset - from;
259 unsigned long rdy = 0;
260 unsigned char *src, *data;
261 z_stream stream;
@@ -360,11 +358,11 @@ static void resolve_delta(struct object_entry *delta_obj, void *base_data,
358 &result_size);
359 free(delta_data);
360 if (!result)
363 - bad_object(delta_obj->offset, "failed to apply delta");
364 - sha1_object(result, result_size, type, delta_obj->sha1);
361 + bad_object(delta_obj->idx.offset, "failed to apply delta");
362 + sha1_object(result, result_size, type, delta_obj->idx.sha1);
363 nr_resolved_deltas++;
364
367 - hashcpy(delta_base.sha1, delta_obj->sha1);
365 + hashcpy(delta_base.sha1, delta_obj->idx.sha1);
366 if (!find_delta_children(&delta_base, &first, &last)) {
367 for (j = first; j <= last; j++) {
368 struct object_entry *child = objects + deltas[j].obj_no;
@@ -374,7 +372,7 @@ static void resolve_delta(struct object_entry *delta_obj, void *base_data,
372 }
373
374 memset(&delta_base, 0, sizeof(delta_base));
377 - delta_base.offset = delta_obj->offset;
375 + delta_base.offset = delta_obj->idx.offset;
376 if (!find_delta_children(&delta_base, &first, &last)) {
377 for (j = first; j <= last; j++) {
378 struct object_entry *child = objects + deltas[j].obj_no;
@@ -418,12 +416,12 @@ static void parse_pack_objects(unsigned char *sha1)
416 delta->obj_no = i;
417 delta++;
418 } else
421 - sha1_object(data, obj->size, obj->type, obj->sha1);
419 + sha1_object(data, obj->size, obj->type, obj->idx.sha1);
420 free(data);
421 if (verbose)
422 display_progress(&progress, i+1);
423 }
426 - objects[i].offset = consumed_bytes;
424 + objects[i].idx.offset = consumed_bytes;
425 if (verbose)
426 stop_progress(&progress);
427
@@ -465,10 +463,10 @@ static void parse_pack_objects(unsigned char *sha1)
463
464 if (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA)
465 continue;
468 - hashcpy(base.sha1, obj->sha1);
466 + hashcpy(base.sha1, obj->idx.sha1);
467 ref = !find_delta_children(&base, &ref_first, &ref_last);
468 memset(&base, 0, sizeof(base));
471 - base.offset = obj->offset;
469 + base.offset = obj->idx.offset;
470 ofs = !find_delta_children(&base, &ofs_first, &ofs_last);
471 if (!ref && !ofs)
472 continue;
@@ -535,11 +533,11 @@ static void append_obj_to_pack(const unsigned char *sha1, void *buf,
533 }
534 header[n++] = c;
535 write_or_die(output_fd, header, n);
538 - obj[0].crc32 = crc32(0, Z_NULL, 0);
539 - obj[0].crc32 = crc32(obj[0].crc32, header, n);
540 - obj[1].offset = obj[0].offset + n;
541 - obj[1].offset += write_compressed(output_fd, buf, size, &obj[0].crc32);
542 - hashcpy(obj->sha1, sha1);
536 + obj[0].idx.crc32 = crc32(0, Z_NULL, 0);
537 + obj[0].idx.crc32 = crc32(obj[0].idx.crc32, header, n);
538 + obj[1].idx.offset = obj[0].idx.offset + n;
539 + obj[1].idx.offset += write_compressed(output_fd, buf, size, &obj[0].idx.crc32);
540 + hashcpy(obj->idx.sha1, sha1);
541 }
542
543 static int delta_pos_compare(const void *_a, const void *_b)
@@ -602,145 +600,6 @@ static void fix_unresolved_deltas(int nr_unresolved)
600 free(sorted_by_pos);
601 }
602
605 -static uint32_t index_default_version = 1;
606 -static uint32_t index_off32_limit = 0x7fffffff;
607 -
608 -static int sha1_compare(const void *_a, const void *_b)
609 -{
610 - struct object_entry *a = *(struct object_entry **)_a;
611 - struct object_entry *b = *(struct object_entry **)_b;
612 - return hashcmp(a->sha1, b->sha1);
613 -}
614 -
615 -/*
616 - * On entry *sha1 contains the pack content SHA1 hash, on exit it is
617 - * the SHA1 hash of sorted object names.
618 - */
619 -static const char *write_index_file(const char *index_name, unsigned char *sha1)
620 -{
621 - struct sha1file *f;
622 - struct object_entry **sorted_by_sha, **list, **last;
623 - uint32_t array[256];
624 - int i, fd;
625 - SHA_CTX ctx;
626 - uint32_t index_version;
627 -
628 - if (nr_objects) {
629 - sorted_by_sha =
630 - xcalloc(nr_objects, sizeof(struct object_entry *));
631 - list = sorted_by_sha;
632 - last = sorted_by_sha + nr_objects;
633 - for (i = 0; i < nr_objects; ++i)
634 - sorted_by_sha[i] = &objects[i];
635 - qsort(sorted_by_sha, nr_objects, sizeof(sorted_by_sha[0]),
636 - sha1_compare);
637 - }
638 - else
639 - sorted_by_sha = list = last = NULL;
640 -
641 - if (!index_name) {
642 - static char tmpfile[PATH_MAX];
643 - snprintf(tmpfile, sizeof(tmpfile),
644 - "%s/tmp_idx_XXXXXX", get_object_directory());
645 - fd = mkstemp(tmpfile);
646 - index_name = xstrdup(tmpfile);
647 - } else {
648 - unlink(index_name);
649 - fd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
650 - }
651 - if (fd < 0)
652 - die("unable to create %s: %s", index_name, strerror(errno));
653 - f = sha1fd(fd, index_name);
654 -
655 - /* if last object's offset is >= 2^31 we should use index V2 */
656 - index_version = (objects[nr_objects-1].offset >> 31) ? 2 : index_default_version;
657 -
658 - /* index versions 2 and above need a header */
659 - if (index_version >= 2) {
660 - struct pack_idx_header hdr;
661 - hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
662 - hdr.idx_version = htonl(index_version);
663 - sha1write(f, &hdr, sizeof(hdr));
664 - }
665 -
666 - /*
667 - * Write the first-level table (the list is sorted,
668 - * but we use a 256-entry lookup to be able to avoid
669 - * having to do eight extra binary search iterations).
670 - */
671 - for (i = 0; i < 256; i++) {
672 - struct object_entry **next = list;
673 - while (next < last) {
674 - struct object_entry *obj = *next;
675 - if (obj->sha1[0] != i)
676 - break;
677 - next++;
678 - }
679 - array[i] = htonl(next - sorted_by_sha);
680 - list = next;
681 - }
682 - sha1write(f, array, 256 * 4);
683 -
684 - /* compute the SHA1 hash of sorted object names. */
685 - SHA1_Init(&ctx);
686 -
687 - /*
688 - * Write the actual SHA1 entries..
689 - */
690 - list = sorted_by_sha;
691 - for (i = 0; i < nr_objects; i++) {
692 - struct object_entry *obj = *list++;
693 - if (index_version < 2) {
694 - uint32_t offset = htonl(obj->offset);
695 - sha1write(f, &offset, 4);
696 - }
697 - sha1write(f, obj->sha1, 20);
698 - SHA1_Update(&ctx, obj->sha1, 20);
699 - }
700 -
701 - if (index_version >= 2) {
702 - unsigned int nr_large_offset = 0;
703 -
704 - /* write the crc32 table */
705 - list = sorted_by_sha;
706 - for (i = 0; i < nr_objects; i++) {
707 - struct object_entry *obj = *list++;
708 - uint32_t crc32_val = htonl(obj->crc32);
709 - sha1write(f, &crc32_val, 4);
710 - }
711 -
712 - /* write the 32-bit offset table */
713 - list = sorted_by_sha;
714 - for (i = 0; i < nr_objects; i++) {
715 - struct object_entry *obj = *list++;
716 - uint32_t offset = (obj->offset <= index_off32_limit) ?
717 - obj->offset : (0x80000000 | nr_large_offset++);
718 - offset = htonl(offset);
719 - sha1write(f, &offset, 4);
720 - }
721 -
722 - /* write the large offset table */
723 - list = sorted_by_sha;
724 - while (nr_large_offset) {
725 - struct object_entry *obj = *list++;
726 - uint64_t offset = obj->offset;
727 - if (offset > index_off32_limit) {
728 - uint32_t split[2];
729 - split[0] = htonl(offset >> 32);
730 - split[1] = htonl(offset & 0xffffffff);
731 - sha1write(f, split, 8);
732 - nr_large_offset--;
733 - }
734 - }
735 - }
736 -
737 - sha1write(f, sha1, 20);
738 - sha1close(f, NULL, 1);
739 - free(sorted_by_sha);
740 - SHA1_Final(sha1, &ctx);
741 - return index_name;
742 -}
743 -
603 static void final(const char *final_pack_name, const char *curr_pack_name,
604 const char *final_index_name, const char *curr_index_name,
605 const char *keep_name, const char *keep_msg,
@@ -830,6 +689,7 @@ int main(int argc, char **argv)
689 const char *curr_index, *index_name = NULL;
690 const char *keep_name = NULL, *keep_msg = NULL;
691 char *index_name_buf = NULL, *keep_name_buf = NULL;
692 + struct pack_idx_entry **idx_objects;
693 unsigned char sha1[20];
694
695 for (i = 1; i < argc; i++) {
@@ -865,12 +725,12 @@ int main(int argc, char **argv)
725 index_name = argv[++i];
726 } else if (!prefixcmp(arg, "--index-version=")) {
727 char *c;
868 - index_default_version = strtoul(arg + 16, &c, 10);
869 - if (index_default_version > 2)
728 + pack_idx_default_version = strtoul(arg + 16, &c, 10);
729 + if (pack_idx_default_version > 2)
730 die("bad %s", arg);
731 if (*c == ',')
872 - index_off32_limit = strtoul(c+1, &c, 0);
873 - if (*c || index_off32_limit & 0x80000000)
732 + pack_idx_off32_limit = strtoul(c+1, &c, 0);
733 + if (*c || pack_idx_off32_limit & 0x80000000)
734 die("bad %s", arg);
735 } else
736 usage(index_pack_usage);
@@ -940,7 +800,13 @@ int main(int argc, char **argv)
800 nr_deltas - nr_resolved_deltas);
801 }
802 free(deltas);
943 - curr_index = write_index_file(index_name, sha1);
803 +
804 + idx_objects = xmalloc((nr_objects) * sizeof(struct pack_idx_entry *));
805 + for (i = 0; i < nr_objects; i++)
806 + idx_objects[i] = &objects[i].idx;
807 + curr_index = write_idx_file(index_name, idx_objects, nr_objects, sha1);
808 + free(idx_objects);
809 +
810 final(pack_name, curr_pack,
811 index_name, curr_index,
812 keep_name, keep_msg,
pack-write.c
+142
@@ -1,5 +1,147 @@
1 #include "cache.h"
2 #include "pack.h"
3 +#include "csum-file.h"
4 +
5 +uint32_t pack_idx_default_version = 1;
6 +uint32_t pack_idx_off32_limit = 0x7fffffff;
7 +
8 +static int sha1_compare(const void *_a, const void *_b)
9 +{
10 + struct pack_idx_entry *a = *(struct pack_idx_entry **)_a;
11 + struct pack_idx_entry *b = *(struct pack_idx_entry **)_b;
12 + return hashcmp(a->sha1, b->sha1);
13 +}
14 +
15 +/*
16 + * On entry *sha1 contains the pack content SHA1 hash, on exit it is
17 + * the SHA1 hash of sorted object names. The objects array passed in
18 + * will be sorted by SHA1 on exit.
19 + */
20 +const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1)
21 +{
22 + struct sha1file *f;
23 + struct pack_idx_entry **sorted_by_sha, **list, **last;
24 + off_t last_obj_offset = 0;
25 + uint32_t array[256];
26 + int i, fd;
27 + SHA_CTX ctx;
28 + uint32_t index_version;
29 +
30 + if (nr_objects) {
31 + sorted_by_sha = objects;
32 + list = sorted_by_sha;
33 + last = sorted_by_sha + nr_objects;
34 + for (i = 0; i < nr_objects; ++i) {
35 + if (objects[i]->offset > last_obj_offset)
36 + last_obj_offset = objects[i]->offset;
37 + }
38 + qsort(sorted_by_sha, nr_objects, sizeof(sorted_by_sha[0]),
39 + sha1_compare);
40 + }
41 + else
42 + sorted_by_sha = list = last = NULL;
43 +
44 + if (!index_name) {
45 + static char tmpfile[PATH_MAX];
46 + snprintf(tmpfile, sizeof(tmpfile),
47 + "%s/tmp_idx_XXXXXX", get_object_directory());
48 + fd = mkstemp(tmpfile);
49 + index_name = xstrdup(tmpfile);
50 + } else {
51 + unlink(index_name);
52 + fd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
53 + }
54 + if (fd < 0)
55 + die("unable to create %s: %s", index_name, strerror(errno));
56 + f = sha1fd(fd, index_name);
57 +
58 + /* if last object's offset is >= 2^31 we should use index V2 */
59 + index_version = (last_obj_offset >> 31) ? 2 : pack_idx_default_version;
60 +
61 + /* index versions 2 and above need a header */
62 + if (index_version >= 2) {
63 + struct pack_idx_header hdr;
64 + hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
65 + hdr.idx_version = htonl(index_version);
66 + sha1write(f, &hdr, sizeof(hdr));
67 + }
68 +
69 + /*
70 + * Write the first-level table (the list is sorted,
71 + * but we use a 256-entry lookup to be able to avoid
72 + * having to do eight extra binary search iterations).
73 + */
74 + for (i = 0; i < 256; i++) {
75 + struct pack_idx_entry **next = list;
76 + while (next < last) {
77 + struct pack_idx_entry *obj = *next;
78 + if (obj->sha1[0] != i)
79 + break;
80 + next++;
81 + }
82 + array[i] = htonl(next - sorted_by_sha);
83 + list = next;
84 + }
85 + sha1write(f, array, 256 * 4);
86 +
87 + /* compute the SHA1 hash of sorted object names. */
88 + SHA1_Init(&ctx);
89 +
90 + /*
91 + * Write the actual SHA1 entries..
92 + */
93 + list = sorted_by_sha;
94 + for (i = 0; i < nr_objects; i++) {
95 + struct pack_idx_entry *obj = *list++;
96 + if (index_version < 2) {
97 + uint32_t offset = htonl(obj->offset);
98 + sha1write(f, &offset, 4);
99 + }
100 + sha1write(f, obj->sha1, 20);
101 + SHA1_Update(&ctx, obj->sha1, 20);
102 + }
103 +
104 + if (index_version >= 2) {
105 + unsigned int nr_large_offset = 0;
106 +
107 + /* write the crc32 table */
108 + list = sorted_by_sha;
109 + for (i = 0; i < nr_objects; i++) {
110 + struct pack_idx_entry *obj = *list++;
111 + uint32_t crc32_val = htonl(obj->crc32);
112 + sha1write(f, &crc32_val, 4);
113 + }
114 +
115 + /* write the 32-bit offset table */
116 + list = sorted_by_sha;
117 + for (i = 0; i < nr_objects; i++) {
118 + struct pack_idx_entry *obj = *list++;
119 + uint32_t offset = (obj->offset <= pack_idx_off32_limit) ?
120 + obj->offset : (0x80000000 | nr_large_offset++);
121 + offset = htonl(offset);
122 + sha1write(f, &offset, 4);
123 + }
124 +
125 + /* write the large offset table */
126 + list = sorted_by_sha;
127 + while (nr_large_offset) {
128 + struct pack_idx_entry *obj = *list++;
129 + uint64_t offset = obj->offset;
130 + if (offset > pack_idx_off32_limit) {
131 + uint32_t split[2];
132 + split[0] = htonl(offset >> 32);
133 + split[1] = htonl(offset & 0xffffffff);
134 + sha1write(f, split, 8);
135 + nr_large_offset--;
136 + }
137 + }
138 + }
139 +
140 + sha1write(f, sha1, 20);
141 + sha1close(f, NULL, 1);
142 + SHA1_Final(sha1, &ctx);
143 + return index_name;
144 +}
145
146 void fixup_pack_header_footer(int pack_fd,
147 unsigned char *pack_file_sha1,
pack.h
+14
@@ -34,6 +34,10 @@ struct pack_header {
34 */
35 #define PACK_IDX_SIGNATURE 0xff744f63 /* "\377tOc" */
36
37 +/* These may be overridden by command-line parameters */
38 +extern uint32_t pack_idx_default_version;
39 +extern uint32_t pack_idx_off32_limit;
40 +
41 /*
42 * Packed object index header
43 */
@@ -42,6 +46,16 @@ struct pack_idx_header {
46 uint32_t idx_version;
47 };
48
49 +/*
50 + * Common part of object structure used for write_idx_file
51 + */
52 +struct pack_idx_entry {
53 + unsigned char sha1[20];
54 + uint32_t crc32;
55 + off_t offset;
56 +};
57 +
58 +extern const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1);
59
60 extern int verify_pack(struct packed_git *, int);
61 extern void fixup_pack_header_footer(int, unsigned char *, const char *, uint32_t);