@matheusbsbs / matheus-bagliano / commits / d4c92e2ac9

object-file: avoid fd seekback by checking object size upfront

In certain scenarios, Git handles writing blobs that exceed "core.bigFileThreshold" differently by streaming the object directly into a packfile. When there is an active ODB transaction, these blobs are streamed to the same packfile instead of using a separate packfile for each. If "pack.packSizeLimit" is configured and streaming another object causes the packfile to exceed the configured limit, the packfile is truncated back to the previous object and the object write is restarted in a new packfile. This works fine, but requires the fd being read from to save a checkpoint so it becomes possible to rewind the input source via seeking back to a known offset at the beginning. In a subsequent commit, blob streaming is converted to use `struct odb_write_stream` as a more generic input source instead of an fd which doesn't provide a mechanism for rewinding. For this use case though, rewinding the fd is not strictly necessary because the inflated size of the object is known and can be used to approximate whether writing the object would cause the packfile to exceed the configured limit prior to writing anything. These blobs written to the packfile are never deltified thus the size difference between what is written versus the inflated size is due to zlib compression. While this does prevent packfiles from being filled to the potential maximum is some cases, it should be good enough and still prevents the packfile from exceeding any configured limit. Use the inflated blob size to determine whether writing an object to a packfile will exceed the configured "pack.packSizeLimit". Signed-off-by: Justin Tobler <jltobler@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Justin Tobler committed May 14, 2026 at 13:37 UTC d4c92e2ac975f256ccc207c65bf46e3be75a2115

1 file changed +25 -61

object-file.c

+25 -61

@@ -1448,29 +1448,17 @@ static int hash_blob_stream(struct odb_write_stream *stream,

1448

1449 /*

1450 * Read the contents from fd for size bytes, streaming it to the

1451 - * packfile in state while updating the hash in ctx. Signal a failure

1452 - * by returning a negative value when the resulting pack would exceed

1453 - * the pack size limit and this is not the first object in the pack,

1454 - * so that the caller can discard what we wrote from the current pack

1455 - * by truncating it and opening a new one. The caller will then call

1456 - * us again after rewinding the input fd.

1457 - *

1458 - * The already_hashed_to pointer is kept untouched by the caller to

1459 - * make sure we do not hash the same byte when we are called

1460 - * again. This way, the caller does not have to checkpoint its hash

1461 - * status before calling us just in case we ask it to call us again

1462 - * with a new pack.

1451 + * packfile in state while updating the hash in ctx.

1452 */

1464 -static int stream_blob_to_pack(struct transaction_packfile *state,

1465 - struct git_hash_ctx *ctx, off_t *already_hashed_to,

1466 - int fd, size_t size, const char *path)

1453 +static void stream_blob_to_pack(struct transaction_packfile *state,

1454 + struct git_hash_ctx *ctx, int fd, size_t size,

1455 + const char *path)

1456 {

1457 git_zstream s;

1458 unsigned char ibuf[16384];

1459 unsigned char obuf[16384];

1460 unsigned hdrlen;

1461 int status = Z_OK;

1473 - off_t offset = 0;

1462

1463 git_deflate_init(&s, pack_compression_level);

1464

@@ -1487,15 +1475,9 @@ static int stream_blob_to_pack(struct transaction_packfile *state,

1475 if ((size_t)read_result != rsize)

1476 die("failed to read %u bytes from '%s'",

1477 (unsigned)rsize, path);

1490 - offset += rsize;

1491 - if (*already_hashed_to < offset) {

1492 - size_t hsize = offset - *already_hashed_to;

1493 - if (rsize < hsize)

1494 - hsize = rsize;

1495 - if (hsize)

1496 - git_hash_update(ctx, ibuf, hsize);

1497 - *already_hashed_to = offset;

1498 - }

1478 +

1479 + git_hash_update(ctx, ibuf, rsize);

1480 +

1481 s.next_in = ibuf;

1482 s.avail_in = rsize;

1483 size -= rsize;

@@ -1506,14 +1488,6 @@ static int stream_blob_to_pack(struct transaction_packfile *state,

1488 if (!s.avail_out || status == Z_STREAM_END) {

1489 size_t written = s.next_out - obuf;

1490

1509 - /* would we bust the size limit? */

1510 - if (state->nr_written &&

1511 - pack_size_limit_cfg &&

1512 - pack_size_limit_cfg < state->offset + written) {

1513 - git_deflate_abort(&s);

1514 - return -1;

1515 - }

1516 -

1491 hashwrite(state->f, obuf, written);

1492 state->offset += written;

1493 s.next_out = obuf;

@@ -1530,7 +1504,6 @@ static int stream_blob_to_pack(struct transaction_packfile *state,

1504 }

1505 }

1506 git_deflate_end(&s);

1533 - return 0;

1507 }

1508

1509 static void flush_packfile_transaction(struct odb_transaction_files *transaction)

@@ -1606,48 +1579,39 @@ static int index_blob_packfile_transaction(struct odb_transaction_files *transac

1579 size_t size, const char *path)

1580 {

1581 struct transaction_packfile *state = &transaction->packfile;

1609 - off_t seekback, already_hashed_to;

1582 struct git_hash_ctx ctx;

1583 unsigned char obuf[16384];

1584 unsigned header_len;

1585 struct hashfile_checkpoint checkpoint;

1586 struct pack_idx_entry *idx;

1587

1616 - seekback = lseek(fd, 0, SEEK_CUR);

1617 - if (seekback == (off_t)-1)

1618 - return error("cannot find the current offset");

1619 -

1588 header_len = format_object_header((char *)obuf, sizeof(obuf),

1589 OBJ_BLOB, size);

1590 transaction->base.source->odb->repo->hash_algo->init_fn(&ctx);

1591 git_hash_update(&ctx, obuf, header_len);

1592

1593 + /*

1594 + * If writing another object to the packfile could result in it

1595 + * exceeding the configured size limit, flush the current packfile

1596 + * transaction.

1597 + *

1598 + * Note that this uses the inflated object size as an approximation.

1599 + * Blob objects written in this manner are not delta-compressed, so

1600 + * the difference between the inflated and on-disk size is limited

1601 + * to zlib compression and is sufficient for this check.

1602 + */

1603 + if (state->nr_written && pack_size_limit_cfg &&

1604 + pack_size_limit_cfg < state->offset + size)

1605 + flush_packfile_transaction(transaction);

1606 +

1607 CALLOC_ARRAY(idx, 1);

1608 prepare_packfile_transaction(transaction);

1609 hashfile_checkpoint_init(state->f, &checkpoint);

1610

1629 - already_hashed_to = 0;

1630 -

1631 - while (1) {

1632 - prepare_packfile_transaction(transaction);

1633 - hashfile_checkpoint(state->f, &checkpoint);

1634 - idx->offset = state->offset;

1635 - crc32_begin(state->f);

1636 -

1637 - if (!stream_blob_to_pack(state, &ctx, &already_hashed_to,

1638 - fd, size, path))

1639 - break;

1640 - /*

1641 - * Writing this object to the current pack will make

1642 - * it too big; we need to truncate it, start a new

1643 - * pack, and write into it.

1644 - */

1645 - hashfile_truncate(state->f, &checkpoint);

1646 - state->offset = checkpoint.offset;

1647 - flush_packfile_transaction(transaction);

1648 - if (lseek(fd, seekback, SEEK_SET) == (off_t)-1)

1649 - return error("cannot seek back");

1650 - }

1611 + hashfile_checkpoint(state->f, &checkpoint);

1612 + idx->offset = state->offset;

1613 + crc32_begin(state->f);

1614 + stream_blob_to_pack(state, &ctx, fd, size, path);

1615 git_hash_final_oid(result_oid, &ctx);

1616

1617 idx->crc32 = crc32_end(state->f);