@matheusbsbs / matheus-bagliano / commits / 74e34e1fca
Fork 0

sha1_file.c: learn about index version 2

With this patch, packs larger than 4GB are usable, even on a 32-bit machine (at least on Linux). If off_t is not large enough to deal with a large pack then die() is called instead of attempting to use the pack and producing garbage. This was tested with a 8GB pack specially created for the occasion on a 32-bit machine. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Nicolas Pitre committed Apr 9, 2007 at 01:06 UTC 74e34e1fca2ed9998581cc94073bc2dd28bbb8f3
1 file changed +89 -29
sha1_file.c
+89 -29
@@ -437,7 +437,7 @@ static int check_packed_git_idx(const char *path, struct packed_git *p)
437 void *idx_map;
438 struct pack_idx_header *hdr;
439 size_t idx_size;
440 - uint32_t nr, i, *index;
440 + uint32_t version, nr, i, *index;
441 int fd = open(path, O_RDONLY);
442 struct stat st;
443
@@ -455,21 +455,23 @@ static int check_packed_git_idx(const char *path, struct packed_git *p)
455 idx_map = xmmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
456 close(fd);
457
458 - /* a future index format would start with this, as older git
459 - * binaries would fail the non-monotonic index check below.
460 - * give a nicer warning to the user if we can.
461 - */
458 hdr = idx_map;
459 if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) {
464 - munmap(idx_map, idx_size);
465 - return error("index file %s is a newer version"
466 - " and is not supported by this binary"
467 - " (try upgrading GIT to a newer version)",
468 - path);
469 - }
460 + version = ntohl(hdr->idx_version);
461 + if (version < 2 || version > 2) {
462 + munmap(idx_map, idx_size);
463 + return error("index file %s is version %d"
464 + " and is not supported by this binary"
465 + " (try upgrading GIT to a newer version)",
466 + path, version);
467 + }
468 + } else
469 + version = 1;
470
471 nr = 0;
472 index = idx_map;
473 + if (version > 1)
474 + index += 2; /* skip index header */
475 for (i = 0; i < 256; i++) {
476 uint32_t n = ntohl(index[i]);
477 if (n < nr) {
@@ -479,19 +481,48 @@ static int check_packed_git_idx(const char *path, struct packed_git *p)
481 nr = n;
482 }
483
482 - /*
483 - * Total size:
484 - * - 256 index entries 4 bytes each
485 - * - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
486 - * - 20-byte SHA1 of the packfile
487 - * - 20-byte SHA1 file checksum
488 - */
489 - if (idx_size != 4*256 + nr * 24 + 20 + 20) {
490 - munmap(idx_map, idx_size);
491 - return error("wrong index file size in %s", path);
484 + if (version == 1) {
485 + /*
486 + * Total size:
487 + * - 256 index entries 4 bytes each
488 + * - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
489 + * - 20-byte SHA1 of the packfile
490 + * - 20-byte SHA1 file checksum
491 + */
492 + if (idx_size != 4*256 + nr * 24 + 20 + 20) {
493 + munmap(idx_map, idx_size);
494 + return error("wrong index file size in %s", path);
495 + }
496 + } else if (version == 2) {
497 + /*
498 + * Minimum size:
499 + * - 8 bytes of header
500 + * - 256 index entries 4 bytes each
501 + * - 20-byte sha1 entry * nr
502 + * - 4-byte crc entry * nr
503 + * - 4-byte offset entry * nr
504 + * - 20-byte SHA1 of the packfile
505 + * - 20-byte SHA1 file checksum
506 + * And after the 4-byte offset table might be a
507 + * variable sized table containing 8-byte entries
508 + * for offsets larger than 2^31.
509 + */
510 + unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20;
511 + if (idx_size < min_size || idx_size > min_size + (nr - 1)*8) {
512 + munmap(idx_map, idx_size);
513 + return error("wrong index file size in %s", path);
514 + }
515 + if (idx_size != min_size) {
516 + /* make sure we can deal with large pack offsets */
517 + off_t x = 0x7fffffffUL, y = 0xffffffffUL;
518 + if (x > (x + 1) || y > (y + 1)) {
519 + munmap(idx_map, idx_size);
520 + return error("pack too large for current definition of off_t in %s", path);
521 + }
522 + }
523 }
524
494 - p->index_version = 1;
525 + p->index_version = version;
526 p->index_data = idx_map;
527 p->index_size = idx_size;
528 p->num_objects = nr;
@@ -1531,27 +1562,56 @@ const unsigned char *nth_packed_object_sha1(const struct packed_git *p,
1562 uint32_t n)
1563 {
1564 const unsigned char *index = p->index_data;
1534 - index += 4 * 256;
1565 if (n >= p->num_objects)
1566 return NULL;
1537 - return index + 24 * n + 4;
1567 + index += 4 * 256;
1568 + if (p->index_version == 1) {
1569 + return index + 24 * n + 4;
1570 + } else {
1571 + index += 8;
1572 + return index + 20 * n;
1573 + }
1574 +}
1575 +
1576 +static off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n)
1577 +{
1578 + const unsigned char *index = p->index_data;
1579 + index += 4 * 256;
1580 + if (p->index_version == 1) {
1581 + return ntohl(*((uint32_t *)(index + 24 * n)));
1582 + } else {
1583 + uint32_t off;
1584 + index += 8 + p->num_objects * (20 + 4);
1585 + off = ntohl(*((uint32_t *)(index + 4 * n)));
1586 + if (!(off & 0x80000000))
1587 + return off;
1588 + index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
1589 + return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
1590 + ntohl(*((uint32_t *)(index + 4)));
1591 + }
1592 }
1593
1594 off_t find_pack_entry_one(const unsigned char *sha1,
1595 struct packed_git *p)
1596 {
1597 const uint32_t *level1_ofs = p->index_data;
1544 - int hi = ntohl(level1_ofs[*sha1]);
1545 - int lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
1598 const unsigned char *index = p->index_data;
1599 + unsigned hi, lo;
1600
1601 + if (p->index_version > 1) {
1602 + level1_ofs += 2;
1603 + index += 8;
1604 + }
1605 index += 4 * 256;
1606 + hi = ntohl(level1_ofs[*sha1]);
1607 + lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
1608
1609 do {
1551 - int mi = (lo + hi) / 2;
1552 - int cmp = hashcmp(index + 24 * mi + 4, sha1);
1610 + unsigned mi = (lo + hi) / 2;
1611 + unsigned x = (p->index_version > 1) ? (mi * 20) : (mi * 24 + 4);
1612 + int cmp = hashcmp(index + x, sha1);
1613 if (!cmp)
1554 - return ntohl(*((uint32_t *)((char *)index + (24 * mi))));
1614 + return nth_packed_object_offset(p, mi);
1615 if (cmp > 0)
1616 hi = mi;
1617 else