@matheusbsbs / matheus-bagliano / commits / 8c1b84bc97

streaming: move logic to read packed objects streams into backend

Move the logic to read packed object streams into the respective subsystem. Signed-off-by: Patrick Steinhardt <ps@pks.im> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Patrick Steinhardt committed Nov 23, 2025 at 19:59 UTC 8c1b84bc977bf1e4515efe0386de87257ec28689
3 files changed +134 -135
packfile.c
+128
@@ -20,6 +20,7 @@
20 #include "tree.h"
21 #include "object-file.h"
22 #include "odb.h"
23 +#include "streaming.h"
24 #include "midx.h"
25 #include "commit-graph.h"
26 #include "pack-revindex.h"
@@ -2406,3 +2407,130 @@ void packfile_store_close(struct packfile_store *store)
2407 close_pack(p);
2408 }
2409 }
2410 +
2411 +struct odb_packed_read_stream {
2412 + struct odb_read_stream base;
2413 + struct packed_git *pack;
2414 + git_zstream z;
2415 + enum {
2416 + ODB_PACKED_READ_STREAM_UNINITIALIZED,
2417 + ODB_PACKED_READ_STREAM_INUSE,
2418 + ODB_PACKED_READ_STREAM_DONE,
2419 + ODB_PACKED_READ_STREAM_ERROR,
2420 + } z_state;
2421 + off_t pos;
2422 +};
2423 +
2424 +static ssize_t read_istream_pack_non_delta(struct odb_read_stream *_st, char *buf,
2425 + size_t sz)
2426 +{
2427 + struct odb_packed_read_stream *st = (struct odb_packed_read_stream *)_st;
2428 + size_t total_read = 0;
2429 +
2430 + switch (st->z_state) {
2431 + case ODB_PACKED_READ_STREAM_UNINITIALIZED:
2432 + memset(&st->z, 0, sizeof(st->z));
2433 + git_inflate_init(&st->z);
2434 + st->z_state = ODB_PACKED_READ_STREAM_INUSE;
2435 + break;
2436 + case ODB_PACKED_READ_STREAM_DONE:
2437 + return 0;
2438 + case ODB_PACKED_READ_STREAM_ERROR:
2439 + return -1;
2440 + case ODB_PACKED_READ_STREAM_INUSE:
2441 + break;
2442 + }
2443 +
2444 + while (total_read < sz) {
2445 + int status;
2446 + struct pack_window *window = NULL;
2447 + unsigned char *mapped;
2448 +
2449 + mapped = use_pack(st->pack, &window,
2450 + st->pos, &st->z.avail_in);
2451 +
2452 + st->z.next_out = (unsigned char *)buf + total_read;
2453 + st->z.avail_out = sz - total_read;
2454 + st->z.next_in = mapped;
2455 + status = git_inflate(&st->z, Z_FINISH);
2456 +
2457 + st->pos += st->z.next_in - mapped;
2458 + total_read = st->z.next_out - (unsigned char *)buf;
2459 + unuse_pack(&window);
2460 +
2461 + if (status == Z_STREAM_END) {
2462 + git_inflate_end(&st->z);
2463 + st->z_state = ODB_PACKED_READ_STREAM_DONE;
2464 + break;
2465 + }
2466 +
2467 + /*
2468 + * Unlike the loose object case, we do not have to worry here
2469 + * about running out of input bytes and spinning infinitely. If
2470 + * we get Z_BUF_ERROR due to too few input bytes, then we'll
2471 + * replenish them in the next use_pack() call when we loop. If
2472 + * we truly hit the end of the pack (i.e., because it's corrupt
2473 + * or truncated), then use_pack() catches that and will die().
2474 + */
2475 + if (status != Z_OK && status != Z_BUF_ERROR) {
2476 + git_inflate_end(&st->z);
2477 + st->z_state = ODB_PACKED_READ_STREAM_ERROR;
2478 + return -1;
2479 + }
2480 + }
2481 + return total_read;
2482 +}
2483 +
2484 +static int close_istream_pack_non_delta(struct odb_read_stream *_st)
2485 +{
2486 + struct odb_packed_read_stream *st = (struct odb_packed_read_stream *)_st;
2487 + if (st->z_state == ODB_PACKED_READ_STREAM_INUSE)
2488 + git_inflate_end(&st->z);
2489 + return 0;
2490 +}
2491 +
2492 +int packfile_store_read_object_stream(struct odb_read_stream **out,
2493 + struct packfile_store *store,
2494 + const struct object_id *oid)
2495 +{
2496 + struct odb_packed_read_stream *stream;
2497 + struct pack_window *window = NULL;
2498 + struct object_info oi = OBJECT_INFO_INIT;
2499 + enum object_type in_pack_type;
2500 + unsigned long size;
2501 +
2502 + oi.sizep = &size;
2503 +
2504 + if (packfile_store_read_object_info(store, oid, &oi, 0) ||
2505 + oi.u.packed.is_delta ||
2506 + repo_settings_get_big_file_threshold(store->odb->repo) >= size)
2507 + return -1;
2508 +
2509 + in_pack_type = unpack_object_header(oi.u.packed.pack,
2510 + &window,
2511 + &oi.u.packed.offset,
2512 + &size);
2513 + unuse_pack(&window);
2514 + switch (in_pack_type) {
2515 + default:
2516 + return -1; /* we do not do deltas for now */
2517 + case OBJ_COMMIT:
2518 + case OBJ_TREE:
2519 + case OBJ_BLOB:
2520 + case OBJ_TAG:
2521 + break;
2522 + }
2523 +
2524 + CALLOC_ARRAY(stream, 1);
2525 + stream->base.close = close_istream_pack_non_delta;
2526 + stream->base.read = read_istream_pack_non_delta;
2527 + stream->base.type = in_pack_type;
2528 + stream->base.size = size;
2529 + stream->z_state = ODB_PACKED_READ_STREAM_UNINITIALIZED;
2530 + stream->pack = oi.u.packed.pack;
2531 + stream->pos = oi.u.packed.offset;
2532 +
2533 + *out = &stream->base;
2534 +
2535 + return 0;
2536 +}
packfile.h
+5
@@ -8,6 +8,7 @@
8
9 /* in odb.h */
10 struct object_info;
11 +struct odb_read_stream;
12
13 struct packed_git {
14 struct hashmap_entry packmap_ent;
@@ -144,6 +145,10 @@ void packfile_store_add_pack(struct packfile_store *store,
145 #define repo_for_each_pack(repo, p) \
146 for (p = packfile_store_get_packs(repo->objects->packfiles); p; p = p->next)
147
148 +int packfile_store_read_object_stream(struct odb_read_stream **out,
149 + struct packfile_store *store,
150 + const struct object_id *oid);
151 +
152 /*
153 * Try to read the object identified by its ID from the object store and
154 * populate the object info with its data. Returns 1 in case the object was
streaming.c
+1 -135
@@ -114,140 +114,6 @@ static struct odb_read_stream *attach_stream_filter(struct odb_read_stream *st,
114 return &fs->base;
115 }
116
117 -/*****************************************************************
118 - *
119 - * Non-delta packed object stream
120 - *
121 - *****************************************************************/
122 -
123 -struct odb_packed_read_stream {
124 - struct odb_read_stream base;
125 - struct packed_git *pack;
126 - git_zstream z;
127 - enum {
128 - ODB_PACKED_READ_STREAM_UNINITIALIZED,
129 - ODB_PACKED_READ_STREAM_INUSE,
130 - ODB_PACKED_READ_STREAM_DONE,
131 - ODB_PACKED_READ_STREAM_ERROR,
132 - } z_state;
133 - off_t pos;
134 -};
135 -
136 -static ssize_t read_istream_pack_non_delta(struct odb_read_stream *_st, char *buf,
137 - size_t sz)
138 -{
139 - struct odb_packed_read_stream *st = (struct odb_packed_read_stream *)_st;
140 - size_t total_read = 0;
141 -
142 - switch (st->z_state) {
143 - case ODB_PACKED_READ_STREAM_UNINITIALIZED:
144 - memset(&st->z, 0, sizeof(st->z));
145 - git_inflate_init(&st->z);
146 - st->z_state = ODB_PACKED_READ_STREAM_INUSE;
147 - break;
148 - case ODB_PACKED_READ_STREAM_DONE:
149 - return 0;
150 - case ODB_PACKED_READ_STREAM_ERROR:
151 - return -1;
152 - case ODB_PACKED_READ_STREAM_INUSE:
153 - break;
154 - }
155 -
156 - while (total_read < sz) {
157 - int status;
158 - struct pack_window *window = NULL;
159 - unsigned char *mapped;
160 -
161 - mapped = use_pack(st->pack, &window,
162 - st->pos, &st->z.avail_in);
163 -
164 - st->z.next_out = (unsigned char *)buf + total_read;
165 - st->z.avail_out = sz - total_read;
166 - st->z.next_in = mapped;
167 - status = git_inflate(&st->z, Z_FINISH);
168 -
169 - st->pos += st->z.next_in - mapped;
170 - total_read = st->z.next_out - (unsigned char *)buf;
171 - unuse_pack(&window);
172 -
173 - if (status == Z_STREAM_END) {
174 - git_inflate_end(&st->z);
175 - st->z_state = ODB_PACKED_READ_STREAM_DONE;
176 - break;
177 - }
178 -
179 - /*
180 - * Unlike the loose object case, we do not have to worry here
181 - * about running out of input bytes and spinning infinitely. If
182 - * we get Z_BUF_ERROR due to too few input bytes, then we'll
183 - * replenish them in the next use_pack() call when we loop. If
184 - * we truly hit the end of the pack (i.e., because it's corrupt
185 - * or truncated), then use_pack() catches that and will die().
186 - */
187 - if (status != Z_OK && status != Z_BUF_ERROR) {
188 - git_inflate_end(&st->z);
189 - st->z_state = ODB_PACKED_READ_STREAM_ERROR;
190 - return -1;
191 - }
192 - }
193 - return total_read;
194 -}
195 -
196 -static int close_istream_pack_non_delta(struct odb_read_stream *_st)
197 -{
198 - struct odb_packed_read_stream *st = (struct odb_packed_read_stream *)_st;
199 - if (st->z_state == ODB_PACKED_READ_STREAM_INUSE)
200 - git_inflate_end(&st->z);
201 - return 0;
202 -}
203 -
204 -static int open_istream_pack_non_delta(struct odb_read_stream **out,
205 - struct object_database *odb,
206 - const struct object_id *oid)
207 -{
208 - struct odb_packed_read_stream *stream;
209 - struct pack_window *window = NULL;
210 - struct object_info oi = OBJECT_INFO_INIT;
211 - enum object_type in_pack_type;
212 - unsigned long size;
213 -
214 - oi.sizep = &size;
215 -
216 - if (packfile_store_read_object_info(odb->packfiles, oid, &oi, 0) ||
217 - oi.u.packed.is_delta ||
218 - repo_settings_get_big_file_threshold(odb->repo) >= size)
219 - return -1;
220 -
221 - in_pack_type = unpack_object_header(oi.u.packed.pack,
222 - &window,
223 - &oi.u.packed.offset,
224 - &size);
225 - unuse_pack(&window);
226 - switch (in_pack_type) {
227 - default:
228 - return -1; /* we do not do deltas for now */
229 - case OBJ_COMMIT:
230 - case OBJ_TREE:
231 - case OBJ_BLOB:
232 - case OBJ_TAG:
233 - break;
234 - }
235 -
236 - CALLOC_ARRAY(stream, 1);
237 - stream->base.close = close_istream_pack_non_delta;
238 - stream->base.read = read_istream_pack_non_delta;
239 - stream->base.type = in_pack_type;
240 - stream->base.size = size;
241 - stream->z_state = ODB_PACKED_READ_STREAM_UNINITIALIZED;
242 - stream->pack = oi.u.packed.pack;
243 - stream->pos = oi.u.packed.offset;
244 -
245 - *out = &stream->base;
246 -
247 - return 0;
248 -}
249 -
250 -
117 /*****************************************************************
118 *
119 * In-core stream
@@ -319,7 +185,7 @@ static int istream_source(struct odb_read_stream **out,
185 {
186 struct odb_source *source;
187
322 - if (!open_istream_pack_non_delta(out, r->objects, oid))
188 + if (!packfile_store_read_object_stream(out, r->objects->packfiles, oid))
189 return 0;
190
191 odb_prepare_alternates(r->objects);