@matheusbsbs / matheus-bagliano / commits / 606c192380

odb, packfile: use size_t for streaming object sizes

The odb_read_stream structure uses unsigned long for the size field, which is 32-bit on Windows even in 64-bit builds. When streaming objects larger than 4GB, the size would be truncated to zero or an incorrect value, resulting in empty files being written to disk. Change the size field in odb_read_stream to size_t and introduce unpack_object_header_sz() to return sizes via size_t pointer. Since object_info.sizep remains unsigned long for API compatibility, use temporary variables where the types differ, with comments noting the truncation limitation for code paths that still use unsigned long. Widening the producers to size_t in this way introduces a handful of silent size_t -> unsigned long narrowings on Windows, all in builtin/pack-objects.c, where the consumers are still typed unsigned long. Make those narrowings explicit with cast_size_t_to_ulong() so they assert loudly the moment an object actually exceeds ULONG_MAX bytes: - oe_get_size_slow() returns unsigned long but holds a size_t locally; cast at the return. - write_reuse_object() passes a size_t into check_pack_inflate(), whose expect parameter is unsigned long; cast at the call. - check_object() routes a size_t through SET_SIZE() and SET_DELTA_SIZE(), both of which take unsigned long via oe_set_size() / oe_set_delta_size(); cast at the three call sites in the OBJ_OFS_DELTA / OBJ_REF_DELTA branches and in the non-delta default arm. The cast-only treatment is deliberately a stop-gap. Properly widening oe_set_size, oe_get_size_slow's return type, check_pack_inflate's expect parameter, object_info.sizep, patch_delta, and the OE_SIZE_BITS bit-fields cascades into a series that is too large to be reviewable, so the proper widening is deferred to a follow-up topic. Until then, cast_size_t_to_ulong() at least makes the truncation explicit at the source: it documents the boundary, and on a 64-bit non-Windows platform it is a no-op. This was originally authored by LordKiRon <https://github.com/LordKiRon>, who preferred not to reveal their real name and therefore agreed that I take over authorship. Helped-by: Torsten Bögershausen <tboegi@web.de> Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Johannes Schindelin committed May 8, 2026 at 08:16 UTC 606c1923803c6d49df45178041c486431ff7cb4b

9 files changed +67 -30

builtin/pack-objects.c

+22 -12

@@ -629,14 +629,21 @@ static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry,

629 struct packed_git *p = IN_PACK(entry);

630 struct pack_window *w_curs = NULL;

631 uint32_t pos;

632 - off_t offset;

632 + off_t offset, cur;

633 enum object_type type = oe_type(entry);

634 + enum object_type in_pack_type;

635 off_t datalen;

636 unsigned char header[MAX_PACK_OBJECT_HEADER],

637 dheader[MAX_PACK_OBJECT_HEADER];

638 unsigned hdrlen;

639 const unsigned hashsz = the_hash_algo->rawsz;

639 - unsigned long entry_size = SIZE(entry);

640 + size_t entry_size;

641 +

642 + cur = entry->in_pack_offset;

643 + in_pack_type = unpack_object_header(p, &w_curs, &cur, &entry_size);

644 + if (in_pack_type < 0)

645 + die(_("write_reuse_object: unable to parse object header of %s"),

646 + oid_to_hex(&entry->idx.oid));

647

648 if (DELTA(entry))

649 type = (allow_ofs_delta && DELTA(entry)->idx.offset) ?

@@ -664,7 +671,8 @@ static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry,

671 datalen -= entry->in_pack_header_size;

672

673 if (!pack_to_stdout && p->index_version == 1 &&

667 - check_pack_inflate(p, &w_curs, offset, datalen, entry_size)) {

674 + check_pack_inflate(p, &w_curs, offset, datalen,

675 + cast_size_t_to_ulong(entry_size))) {

676 error(_("corrupt packed object for %s"),

677 oid_to_hex(&entry->idx.oid));

678 unuse_pack(&w_curs);

@@ -1087,7 +1095,7 @@ static void write_reused_pack_one(struct packed_git *reuse_packfile,

1095 {

1096 off_t offset, next, cur;

1097 enum object_type type;

1090 - unsigned long size;

1098 + size_t size;

1099

1100 offset = pack_pos_to_offset(reuse_packfile, pos);

1101 next = pack_pos_to_offset(reuse_packfile, pos + 1);

@@ -2243,7 +2251,7 @@ static void check_object(struct object_entry *entry, uint32_t object_index)

2251 off_t ofs;

2252 unsigned char *buf, c;

2253 enum object_type type;

2246 - unsigned long in_pack_size;

2254 + size_t in_pack_size;

2255

2256 buf = use_pack(p, &w_curs, entry->in_pack_offset, &avail);

2257

@@ -2270,7 +2278,7 @@ static void check_object(struct object_entry *entry, uint32_t object_index)

2278 default:

2279 /* Not a delta hence we've already got all we need. */

2280 oe_set_type(entry, entry->in_pack_type);

2273 - SET_SIZE(entry, in_pack_size);

2281 + SET_SIZE(entry, cast_size_t_to_ulong(in_pack_size));

2282 entry->in_pack_header_size = used;

2283 if (oe_type(entry) < OBJ_COMMIT || oe_type(entry) > OBJ_BLOB)

2284 goto give_up;

@@ -2324,8 +2332,8 @@ static void check_object(struct object_entry *entry, uint32_t object_index)

2332 if (have_base &&

2333 can_reuse_delta(&base_ref, entry, &base_entry)) {

2334 oe_set_type(entry, entry->in_pack_type);

2327 - SET_SIZE(entry, in_pack_size); /* delta size */

2328 - SET_DELTA_SIZE(entry, in_pack_size);

2335 + SET_SIZE(entry, cast_size_t_to_ulong(in_pack_size)); /* delta size */

2336 + SET_DELTA_SIZE(entry, cast_size_t_to_ulong(in_pack_size));

2337

2338 if (base_entry) {

2339 SET_DELTA(entry, base_entry);

@@ -2734,16 +2742,18 @@ unsigned long oe_get_size_slow(struct packing_data *pack,

2742 struct pack_window *w_curs;

2743 unsigned char *buf;

2744 enum object_type type;

2737 - unsigned long used, avail, size;

2745 + unsigned long used, avail;

2746 + size_t size;

2747

2748 if (e->type_ != OBJ_OFS_DELTA && e->type_ != OBJ_REF_DELTA) {

2749 + unsigned long sz;

2750 packing_data_lock(&to_pack);

2751 if (odb_read_object_info(the_repository->objects,

2742 - &e->idx.oid, &size) < 0)

2752 + &e->idx.oid, &sz) < 0)

2753 die(_("unable to get size of %s"),

2754 oid_to_hex(&e->idx.oid));

2755 packing_data_unlock(&to_pack);

2746 - return size;

2756 + return sz;

2757 }

2758

2759 p = oe_in_pack(pack, e);

@@ -2760,7 +2770,7 @@ unsigned long oe_get_size_slow(struct packing_data *pack,

2770

2771 unuse_pack(&w_curs);

2772 packing_data_unlock(&to_pack);

2763 - return size;

2773 + return cast_size_t_to_ulong(size);

2774 }

2775

2776 static int try_delta(struct unpacked *trg, struct unpacked *src,

object-file.c

+9 -1

@@ -2326,6 +2326,7 @@ int odb_source_loose_read_object_stream(struct odb_read_stream **out,

2326 struct object_info oi = OBJECT_INFO_INIT;

2327 struct odb_loose_read_stream *st;

2328 unsigned long mapsize;

2329 + unsigned long size_ul;

2330 void *mapped;

2331

2332 mapped = odb_source_loose_map_object(source, oid, &mapsize);

@@ -2349,11 +2350,18 @@ int odb_source_loose_read_object_stream(struct odb_read_stream **out,

2350 goto error;

2351 }

2352

2352 - oi.sizep = &st->base.size;

2353 + /*

2354 + * object_info.sizep is unsigned long* (32-bit on Windows), but

2355 + * st->base.size is size_t (64-bit). Use temporary variable.

2356 + * Note: loose objects >4GB would still truncate here, but such

2357 + * large loose objects are uncommon (they'd normally be packed).

2358 + */

2359 + oi.sizep = &size_ul;

2360 oi.typep = &st->base.type;

2361

2362 if (parse_loose_header(st->hdr, &oi) < 0 || st->base.type < 0)

2363 goto error;

2364 + st->base.size = size_ul;

2365

2366 st->mapped = mapped;

2367 st->mapsize = mapsize;

odb/streaming.c

+12 -1

@@ -157,15 +157,26 @@ static int open_istream_incore(struct odb_read_stream **out,

157 .base.read = read_istream_incore,

158 };

159 struct odb_incore_read_stream *st;

160 + unsigned long size_ul;

161 int ret;

162

163 oi.typep = &stream.base.type;

163 - oi.sizep = &stream.base.size;

164 + /*

165 + * object_info.sizep is unsigned long* (32-bit on Windows), but

166 + * stream.base.size is size_t (64-bit). We use a temporary variable

167 + * because the types are incompatible. Note: this path still truncates

168 + * for >4GB objects, but large objects should use pack streaming

169 + * (packfile_store_read_object_stream) which handles size_t properly.

170 + * This incore fallback is only used for small objects or when pack

171 + * streaming is unavailable.

172 + */

173 + oi.sizep = &size_ul;

174 oi.contentp = (void **)&stream.buf;

175 ret = odb_read_object_info_extended(odb, oid, &oi,

176 OBJECT_INFO_DIE_IF_CORRUPT);

177 if (ret)

178 return ret;

179 + stream.base.size = size_ul;

180

181 CALLOC_ARRAY(st, 1);

182 *st = stream;

odb/streaming.h

+1 -1

@@ -21,7 +21,7 @@ struct odb_read_stream {

21 odb_read_stream_close_fn close;

22 odb_read_stream_read_fn read;

23 enum object_type type;

24 - unsigned long size; /* inflated size of full object */

24 + size_t size; /* inflated size of full object */

25 };

27 /*

oss-fuzz/fuzz-pack-headers.c

+1 -1

@@ -6,7 +6,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);

6 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)

7 {

8 enum object_type type;

9 - unsigned long len;

9 + size_t len;

11 unpack_object_header_buffer((const unsigned char *)data,

12 (unsigned long)size, &type, &len);

pack-bitmap.c

+1 -1

@@ -2270,7 +2270,7 @@ static int try_partial_reuse(struct bitmap_index *bitmap_git,

2270 {

2271 off_t delta_obj_offset;

2272 enum object_type type;

2273 - unsigned long size;

2273 + size_t size;

2274

2275 if (pack_pos >= pack->p->num_objects)

2276 return -1; /* not actually in the pack */

pack-check.c

+4 -2

@@ -110,7 +110,7 @@ static int verify_packfile(struct repository *r,

110 void *data;

111 struct object_id oid;

112 enum object_type type;

113 - unsigned long size;

113 + size_t size;

114 off_t curpos;

115 int data_valid;

116

@@ -143,7 +143,9 @@ static int verify_packfile(struct repository *r,

143 data = NULL;

144 data_valid = 0;

145 } else {

146 - data = unpack_entry(r, p, entries[i].offset, &type, &size);

146 + unsigned long sz;

147 + data = unpack_entry(r, p, entries[i].offset, &type, &sz);

148 + size = sz;

149 data_valid = 1;

150 }

151

packfile.c

+15 -9

@@ -1133,7 +1133,7 @@ out:

1133 }

1134

1135 unsigned long unpack_object_header_buffer(const unsigned char *buf,

1136 - unsigned long len, enum object_type *type, unsigned long *sizep)

1136 + unsigned long len, enum object_type *type, size_t *sizep)

1137 {

1138 unsigned shift;

1139 size_t size, c;

@@ -1144,7 +1144,11 @@ unsigned long unpack_object_header_buffer(const unsigned char *buf,

1144 size = c & 15;

1145 shift = 4;

1146 while (c & 0x80) {

1147 - if (len <= used || (bitsizeof(long) - 7) < shift) {

1147 + /*

1148 + * Each continuation byte adds 7 bits. Ensure shift won't

1149 + * overflow size_t (use size_t not long for 64-bit on Windows).

1150 + */

1151 + if (len <= used || (bitsizeof(size_t) - 7) < shift) {

1152 error("bad object header");

1153 size = used = 0;

1154 break;

@@ -1153,7 +1157,7 @@ unsigned long unpack_object_header_buffer(const unsigned char *buf,

1157 size = st_add(size, st_left_shift(c & 0x7f, shift));

1158 shift += 7;

1159 }

1156 - *sizep = cast_size_t_to_ulong(size);

1160 + *sizep = size;

1161 return used;

1162 }

1163

@@ -1215,7 +1219,7 @@ unsigned long get_size_from_delta(struct packed_git *p,

1219 int unpack_object_header(struct packed_git *p,

1220 struct pack_window **w_curs,

1221 off_t *curpos,

1218 - unsigned long *sizep)

1222 + size_t *sizep)

1223 {

1224 unsigned char *base;

1225 unsigned long left;

@@ -1367,7 +1371,7 @@ static enum object_type packed_to_object_type(struct repository *r,

1371

1372 while (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {

1373 off_t base_offset;

1370 - unsigned long size;

1374 + size_t size;

1375 /* Push the object we're going to leave behind */

1376 if (poi_stack_nr >= poi_stack_alloc && poi_stack == small_poi_stack) {

1377 poi_stack_alloc = alloc_nr(poi_stack_nr);

@@ -1586,7 +1590,7 @@ static int packed_object_info_with_index_pos(struct packed_git *p, off_t obj_off

1590 uint32_t *maybe_index_pos, struct object_info *oi)

1591 {

1592 struct pack_window *w_curs = NULL;

1589 - unsigned long size;

1593 + size_t size;

1594 off_t curpos = obj_offset;

1595 enum object_type type = OBJ_NONE;

1596 uint32_t pack_pos;

@@ -1778,7 +1782,7 @@ void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset,

1782 struct pack_window *w_curs = NULL;

1783 off_t curpos = obj_offset;

1784 void *data = NULL;

1781 - unsigned long size;

1785 + size_t size;

1786 enum object_type type;

1787 struct unpack_entry_stack_ent small_delta_stack[UNPACK_ENTRY_STACK_PREALLOC];

1788 struct unpack_entry_stack_ent *delta_stack = small_delta_stack;

@@ -1943,8 +1947,10 @@ void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset,

1947 (uintmax_t)curpos, p->pack_name);

1948 data = NULL;

1949 } else {

1950 + unsigned long sz;

1951 data = patch_delta(base, base_size, delta_data,

1947 - delta_size, &size);

1952 + delta_size, &sz);

1953 + size = sz;

1954

1955 /*

1956 * We could not apply the delta; warn the user, but

@@ -2929,7 +2935,7 @@ int packfile_read_object_stream(struct odb_read_stream **out,

2935 struct odb_packed_read_stream *stream;

2936 struct pack_window *window = NULL;

2937 enum object_type in_pack_type;

2932 - unsigned long size;

2938 + size_t size;

2939

2940 in_pack_type = unpack_object_header(pack, &window, &offset, &size);

2941 unuse_pack(&window);

packfile.h

+2 -2

@@ -456,9 +456,9 @@ off_t find_pack_entry_one(const struct object_id *oid, struct packed_git *);

456

457 int is_pack_valid(struct packed_git *);

458 void *unpack_entry(struct repository *r, struct packed_git *, off_t, enum object_type *, unsigned long *);

459 -unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep);

459 +unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, size_t *sizep);

460 unsigned long get_size_from_delta(struct packed_git *, struct pack_window **, off_t);

461 -int unpack_object_header(struct packed_git *, struct pack_window **, off_t *, unsigned long *);

461 +int unpack_object_header(struct packed_git *, struct pack_window **, off_t *, size_t *);

462 off_t get_delta_base(struct packed_git *p, struct pack_window **w_curs,

463 off_t *curpos, enum object_type type,

464 off_t delta_obj_offset);