streaming: allocate stream inside the backend-specific logic
When creating a new stream we first allocate it and then call into backend-specific logic to populate the stream. This design requires that the stream itself contains a `union` with backend-specific members that then ultimately get populated by the backend-specific logic. This works, but it's awkward in the context of pluggable object databases. Each backend will need its own member in that union, and as the structure itself is completely opaque (it's only defined in "streaming.c") it also has the consequence that we must have the logic that is specific to backends in "streaming.c". Ideally though, the infrastructure would be reversed: we have a generic `struct odb_read_stream` and some helper functions in "streaming.c", whereas the backend-specific logic sits in the backend's subsystem itself. This can be realized by using a design that is similar to how we handle reference databases: instead of having a union of members, we instead have backend-specific structures with a `struct odb_read_stream base` as its first member. The backends would thus hand out the pointer to the base, but internally they know to cast back to the backend-specific type. This means though that we need to allocate different structures depending on the backend. To prepare for this, move allocation of the structure into the backend-specific functions that open a new stream. Subsequent commits will then create those new backend-specific structs. Signed-off-by: Patrick Steinhardt <ps@pks.im> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Patrick Steinhardt committed
Nov 23, 2025 at 19:59 UTC
595296e124f5e8a67c4669fcaeb1b28e71c2d751
1 file changed
+65
-38
streaming.c
+65
-38
@@ -222,27 +222,34 @@ static int close_istream_loose(struct odb_read_stream *st)
222
return 0;
223
}
224
225
-static int open_istream_loose(struct odb_read_stream *st, struct repository *r,
225
+static int open_istream_loose(struct odb_read_stream **out,
226
+ struct repository *r,
227
const struct object_id *oid)
228
{
229
struct object_info oi = OBJECT_INFO_INIT;
230
+ struct odb_read_stream *st;
231
struct odb_source *source;
230
-
231
- oi.sizep = &st->size;
232
- oi.typep = &st->type;
232
+ unsigned long mapsize;
233
+ void *mapped;
234
235
odb_prepare_alternates(r->objects);
236
for (source = r->objects->sources; source; source = source->next) {
236
- st->u.loose.mapped = odb_source_loose_map_object(source, oid,
237
- &st->u.loose.mapsize);
238
- if (st->u.loose.mapped)
237
+ mapped = odb_source_loose_map_object(source, oid, &mapsize);
238
+ if (mapped)
239
break;
240
}
241
- if (!st->u.loose.mapped)
241
+ if (!mapped)
242
return -1;
243
244
- switch (unpack_loose_header(&st->z, st->u.loose.mapped,
245
- st->u.loose.mapsize, st->u.loose.hdr,
244
+ /*
245
+ * Note: we must allocate this structure early even though we may still
246
+ * fail. This is because we need to initialize the zlib stream, and it
247
+ * is not possible to copy the stream around after the fact because it
248
+ * has self-referencing pointers.
249
+ */
250
+ CALLOC_ARRAY(st, 1);
251
+
252
+ switch (unpack_loose_header(&st->z, mapped, mapsize, st->u.loose.hdr,
253
sizeof(st->u.loose.hdr))) {
254
case ULHR_OK:
255
break;
@@ -250,19 +257,28 @@ static int open_istream_loose(struct odb_read_stream *st, struct repository *r,
257
case ULHR_TOO_LONG:
258
goto error;
259
}
260
+
261
+ oi.sizep = &st->size;
262
+ oi.typep = &st->type;
263
+
264
if (parse_loose_header(st->u.loose.hdr, &oi) < 0 || st->type < 0)
265
goto error;
266
267
+ st->u.loose.mapped = mapped;
268
+ st->u.loose.mapsize = mapsize;
269
st->u.loose.hdr_used = strlen(st->u.loose.hdr) + 1;
270
st->u.loose.hdr_avail = st->z.total_out;
271
st->z_state = z_used;
272
st->close = close_istream_loose;
273
st->read = read_istream_loose;
274
275
+ *out = st;
276
+
277
return 0;
278
error:
279
git_inflate_end(&st->z);
280
munmap(st->u.loose.mapped, st->u.loose.mapsize);
281
+ free(st);
282
return -1;
283
}
284
@@ -338,12 +354,16 @@ static int close_istream_pack_non_delta(struct odb_read_stream *st)
354
return 0;
355
}
356
341
-static int open_istream_pack_non_delta(struct odb_read_stream *st,
357
+static int open_istream_pack_non_delta(struct odb_read_stream **out,
358
struct repository *r UNUSED,
359
const struct object_id *oid UNUSED,
360
struct packed_git *pack,
361
off_t offset)
362
{
363
+ struct odb_read_stream stream = {
364
+ .close = close_istream_pack_non_delta,
365
+ .read = read_istream_pack_non_delta,
366
+ };
367
struct pack_window *window;
368
enum object_type in_pack_type;
369
@@ -352,7 +372,7 @@ static int open_istream_pack_non_delta(struct odb_read_stream *st,
372
in_pack_type = unpack_object_header(pack,
373
&window,
374
&offset,
355
- &st->size);
375
+ &stream.size);
376
unuse_pack(&window);
377
switch (in_pack_type) {
378
default:
@@ -363,12 +383,13 @@ static int open_istream_pack_non_delta(struct odb_read_stream *st,
383
case OBJ_TAG:
384
break;
385
}
366
- st->type = in_pack_type;
367
- st->z_state = z_unused;
368
- st->close = close_istream_pack_non_delta;
369
- st->read = read_istream_pack_non_delta;
370
- st->u.in_pack.pack = pack;
371
- st->u.in_pack.pos = offset;
386
+ stream.type = in_pack_type;
387
+ stream.z_state = z_unused;
388
+ stream.u.in_pack.pack = pack;
389
+ stream.u.in_pack.pos = offset;
390
+
391
+ CALLOC_ARRAY(*out, 1);
392
+ **out = stream;
393
394
return 0;
395
}
@@ -400,27 +421,35 @@ static ssize_t read_istream_incore(struct odb_read_stream *st, char *buf, size_t
421
return read_size;
422
}
423
403
-static int open_istream_incore(struct odb_read_stream *st, struct repository *r,
424
+static int open_istream_incore(struct odb_read_stream **out,
425
+ struct repository *r,
426
const struct object_id *oid)
427
{
428
struct object_info oi = OBJECT_INFO_INIT;
407
-
408
- st->u.incore.read_ptr = 0;
409
- st->close = close_istream_incore;
410
- st->read = read_istream_incore;
411
-
412
- oi.typep = &st->type;
413
- oi.sizep = &st->size;
414
- oi.contentp = (void **)&st->u.incore.buf;
415
- return odb_read_object_info_extended(r->objects, oid, &oi,
416
- OBJECT_INFO_DIE_IF_CORRUPT);
429
+ struct odb_read_stream stream = {
430
+ .close = close_istream_incore,
431
+ .read = read_istream_incore,
432
+ };
433
+ int ret;
434
+
435
+ oi.typep = &stream.type;
436
+ oi.sizep = &stream.size;
437
+ oi.contentp = (void **)&stream.u.incore.buf;
438
+ ret = odb_read_object_info_extended(r->objects, oid, &oi,
439
+ OBJECT_INFO_DIE_IF_CORRUPT);
440
+ if (ret)
441
+ return ret;
442
+
443
+ CALLOC_ARRAY(*out, 1);
444
+ **out = stream;
445
+ return 0;
446
}
447
448
/*****************************************************************************
449
* static helpers variables and functions for users of streaming interface
450
*****************************************************************************/
451
423
-static int istream_source(struct odb_read_stream *st,
452
+static int istream_source(struct odb_read_stream **out,
453
struct repository *r,
454
const struct object_id *oid)
455
{
@@ -435,13 +464,13 @@ static int istream_source(struct odb_read_stream *st,
464
465
switch (oi.whence) {
466
case OI_LOOSE:
438
- if (open_istream_loose(st, r, oid) < 0)
467
+ if (open_istream_loose(out, r, oid) < 0)
468
break;
469
return 0;
470
case OI_PACKED:
471
if (oi.u.packed.is_delta ||
472
repo_settings_get_big_file_threshold(the_repository) >= size ||
444
- open_istream_pack_non_delta(st, r, oid, oi.u.packed.pack,
473
+ open_istream_pack_non_delta(out, r, oid, oi.u.packed.pack,
474
oi.u.packed.offset) < 0)
475
break;
476
return 0;
@@ -449,7 +478,7 @@ static int istream_source(struct odb_read_stream *st,
478
break;
479
}
480
452
- return open_istream_incore(st, r, oid);
481
+ return open_istream_incore(out, r, oid);
482
}
483
484
/****************************************************************
@@ -474,14 +503,12 @@ struct odb_read_stream *open_istream(struct repository *r,
503
unsigned long *size,
504
struct stream_filter *filter)
505
{
477
- struct odb_read_stream *st = xmalloc(sizeof(*st));
506
+ struct odb_read_stream *st;
507
const struct object_id *real = lookup_replace_object(r, oid);
479
- int ret = istream_source(st, r, real);
508
+ int ret = istream_source(&st, r, real);
509
481
- if (ret) {
482
- free(st);
510
+ if (ret)
511
return NULL;
484
- }
512
513
if (filter) {
514
/* Add "&& !is_null_stream_filter(filter)" for performance */