@matheusbsbs / matheus-bagliano / commits / c553ca25bd
Fork 0

pack-objects: learn about pack index version 2

Pack index version 2 goes as follows: - 8 bytes of header with signature and version. - 256 entries of 4-byte first-level fan-out table. - Table of sorted 20-byte SHA1 records for each object in pack. - Table of 4-byte CRC32 entries for raw pack object data. - Table of 4-byte offset entries for objects in the pack if offset is representable with 31 bits or less, otherwise it is an index in the next table with top bit set. - Table of 8-byte offset entries indexed from previous table for offsets which are 32 bits or more (optional). - 20-byte SHA1 checksum of sorted object names. - 20-byte SHA1 checksum of the above. The object SHA1 table is all contiguous so future pack format that would contain this table directly won't require big changes to the code. It is also tighter for slightly better cache locality when looking up entries. Support for large packs exceeding 31 bits in size won't impose an index size bloat for packs within that range that don't need a 64-bit offset. And because newer objects which are likely to be the most frequently used are located at the beginning of the pack, they won't pay the 64-bit offset lookup at run time either even if the pack is large. Right now an index version 2 is created only when the biggest offset in a pack reaches 31 bits. It might be a good idea to always use index version 2 eventually to benefit from the CRC32 it contains when reusing pack data while repacking. [jc: with the "oops" fix to keep track of the last offset correctly] Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Nicolas Pitre committed Apr 9, 2007 at 01:06 UTC c553ca25bd60dc9fd50b8bc7bd329601b81cee66
1 file changed +87 -12
builtin-pack-objects.c
+87 -12
@@ -169,13 +169,33 @@ static void prepare_pack_revindex(struct pack_revindex *rix)
169 int i;
170 const char *index = p->index_data;
171
172 - index += 4 * 256;
172 rix->revindex = xmalloc(sizeof(*rix->revindex) * (num_ent + 1));
174 - for (i = 0; i < num_ent; i++) {
175 - uint32_t hl = *((uint32_t *)(index + 24 * i));
176 - rix->revindex[i].offset = ntohl(hl);
177 - rix->revindex[i].nr = i;
173 + index += 4 * 256;
174 +
175 + if (p->index_version > 1) {
176 + const uint32_t *off_32 =
177 + (uint32_t *)(index + 8 + p->num_objects * (20 + 4));
178 + const uint32_t *off_64 = off_32 + p->num_objects;
179 + for (i = 0; i < num_ent; i++) {
180 + uint32_t off = ntohl(*off_32++);
181 + if (!(off & 0x80000000)) {
182 + rix->revindex[i].offset = off;
183 + } else {
184 + rix->revindex[i].offset =
185 + ((uint64_t)ntohl(*off_64++)) << 32;
186 + rix->revindex[i].offset |=
187 + ntohl(*off_64++);
188 + }
189 + rix->revindex[i].nr = i;
190 + }
191 + } else {
192 + for (i = 0; i < num_ent; i++) {
193 + uint32_t hl = *((uint32_t *)(index + 24 * i));
194 + rix->revindex[i].offset = ntohl(hl);
195 + rix->revindex[i].nr = i;
196 + }
197 }
198 +
199 /* This knows the pack format -- the 20-byte trailer
200 * follows immediately after the last object data.
201 */
@@ -528,11 +548,11 @@ static off_t write_one(struct sha1file *f,
548 return offset + size;
549 }
550
531 -static void write_pack_file(void)
551 +static off_t write_pack_file(void)
552 {
553 uint32_t i;
554 struct sha1file *f;
535 - off_t offset;
555 + off_t offset, last_obj_offset = 0;
556 struct pack_header hdr;
557 unsigned last_percent = 999;
558 int do_progress = progress;
@@ -555,6 +575,7 @@ static void write_pack_file(void)
575 if (!nr_result)
576 goto done;
577 for (i = 0; i < nr_objects; i++) {
578 + last_obj_offset = offset;
579 offset = write_one(f, objects + i, offset);
580 if (do_progress) {
581 unsigned percent = written * 100 / nr_result;
@@ -572,9 +593,11 @@ static void write_pack_file(void)
593 if (written != nr_result)
594 die("wrote %u objects while expecting %u", written, nr_result);
595 sha1close(f, pack_file_sha1, 1);
596 +
597 + return last_obj_offset;
598 }
599
577 -static void write_index_file(void)
600 +static void write_index_file(off_t last_obj_offset)
601 {
602 uint32_t i;
603 struct sha1file *f = sha1create("%s-%s.%s", base_name,
@@ -582,6 +605,18 @@ static void write_index_file(void)
605 struct object_entry **list = sorted_by_sha;
606 struct object_entry **last = list + nr_result;
607 uint32_t array[256];
608 + uint32_t index_version;
609 +
610 + /* if last object's offset is >= 2^31 we should use index V2 */
611 + index_version = (last_obj_offset >> 31) ? 2 : 1;
612 +
613 + /* index versions 2 and above need a header */
614 + if (index_version >= 2) {
615 + struct pack_idx_header hdr;
616 + hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
617 + hdr.idx_version = htonl(index_version);
618 + sha1write(f, &hdr, sizeof(hdr));
619 + }
620
621 /*
622 * Write the first-level table (the list is sorted,
@@ -607,10 +642,49 @@ static void write_index_file(void)
642 list = sorted_by_sha;
643 for (i = 0; i < nr_result; i++) {
644 struct object_entry *entry = *list++;
610 - uint32_t offset = htonl(entry->offset);
611 - sha1write(f, &offset, 4);
645 + if (index_version < 2) {
646 + uint32_t offset = htonl(entry->offset);
647 + sha1write(f, &offset, 4);
648 + }
649 sha1write(f, entry->sha1, 20);
650 }
651 +
652 + if (index_version >= 2) {
653 + unsigned int nr_large_offset = 0;
654 +
655 + /* write the crc32 table */
656 + list = sorted_by_sha;
657 + for (i = 0; i < nr_objects; i++) {
658 + struct object_entry *entry = *list++;
659 + uint32_t crc32_val = htonl(entry->crc32);
660 + sha1write(f, &crc32_val, 4);
661 + }
662 +
663 + /* write the 32-bit offset table */
664 + list = sorted_by_sha;
665 + for (i = 0; i < nr_objects; i++) {
666 + struct object_entry *entry = *list++;
667 + uint32_t offset = (entry->offset <= 0x7fffffff) ?
668 + entry->offset : (0x80000000 | nr_large_offset++);
669 + offset = htonl(offset);
670 + sha1write(f, &offset, 4);
671 + }
672 +
673 + /* write the large offset table */
674 + list = sorted_by_sha;
675 + while (nr_large_offset) {
676 + struct object_entry *entry = *list++;
677 + uint64_t offset = entry->offset;
678 + if (offset > 0x7fffffff) {
679 + uint32_t split[2];
680 + split[0] = htonl(offset >> 32);
681 + split[1] = htonl(offset & 0xffffffff);
682 + sha1write(f, split, 8);
683 + nr_large_offset--;
684 + }
685 + }
686 + }
687 +
688 sha1write(f, pack_file_sha1, 20);
689 sha1close(f, NULL, 1);
690 }
@@ -1698,6 +1772,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
1772 if (reuse_cached_pack(object_list_sha1))
1773 ;
1774 else {
1775 + off_t last_obj_offset;
1776 if (nr_result)
1777 prepare_pack(window, depth);
1778 if (progress == 1 && pack_to_stdout) {
@@ -1707,9 +1782,9 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
1782 signal(SIGALRM, SIG_IGN );
1783 progress_update = 0;
1784 }
1710 - write_pack_file();
1785 + last_obj_offset = write_pack_file();
1786 if (!pack_to_stdout) {
1712 - write_index_file();
1787 + write_index_file(last_obj_offset);
1788 puts(sha1_to_hex(object_list_sha1));
1789 }
1790 }