cat-file: use packed_object_info() for --batch-all-objects

When "cat-file --batch-all-objects" iterates over each object, it knows where to find each one. But when we look up details of the object, we don't use that information at all. This patch teaches it to use the pack/offset pair when we're iterating over objects in a pack. This yields a measurable speed improvement (timings on a fully packed clone of linux.git): Benchmark #1: ./git.old cat-file --batch-all-objects --unordered --batch-check="%(objecttype) %(objectname)" Time (mean ± σ): 8.128 s ± 0.118 s [User: 7.968 s, System: 0.156 s] Range (min … max): 8.007 s … 8.301 s 10 runs Benchmark #2: ./git.new cat-file --batch-all-objects --unordered --batch-check="%(objecttype) %(objectname)" Time (mean ± σ): 4.294 s ± 0.064 s [User: 4.167 s, System: 0.125 s] Range (min … max): 4.227 s … 4.457 s 10 runs Summary './git.new cat-file --batch-all-objects --unordered --batch-check="%(objecttype) %(objectname)"' ran 1.89 ± 0.04 times faster than './git.old cat-file --batch-all-objects --unordered --batch-check="%(objecttype) %(objectname)" The implementation is pretty simple: we just call packed_object_info() instead of oid_object_info_extended() when we can. Most of the changes are just plumbing the pack/offset pair through the callstack. There is one subtlety: replace lookups are not handled by packed_object_info(). But since those are disabled for --batch-all-objects, and since we'll only have pack info when that option is in effect, we don't have to worry about that. There are a few limitations to this optimization which we could address with further work: - I didn't bother recording when we found an object loose. Technically this could save us doing a fruitless lookup in the pack index. But opening and mmap-ing a loose object is so expensive in the first place that this doesn't matter much. And if your repository is large enough to care about per-object performance, most objects are going to be packed anyway. - This works only in --unordered mode. For the sorted mode, we'd have to record the pack/offset pair as part of our oid-collection. That's more code, plus at least 16 extra bytes of heap per object. It would probably still be a net win in runtime, but we'd need to measure. - For --batch, this still helps us with getting the object metadata, but we still do a from-scratch lookup for the object contents. This probably doesn't matter that much, because the lookup cost will be much smaller relative to the cost of actually unpacking and printing the objects. For small objects, we could probably swap out read_object_file() for using packed_object_info() with a "object_info.contentp" to get the contents. But we'd still need to deal with streaming for larger objects. A better path forward here is to teach the initial oid_object_info_extended() / packed_object_info() calls to retrieve the contents of smaller objects while they are already being accessed. That would save the extra lookup entirely. But it's a non-trivial feature to add to the object_info code, so I left it for now. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Jeff King committed Oct 5, 2021 at 16:38 UTC bf972896d7186f0d29f7807b600f6fdb2837de18
1 file changed +36 -14
builtin/cat-file.c
+36 -14
@@ -355,18 +355,34 @@ static void print_object_or_die(struct batch_options *opt, struct expand_data *d
355 }
356 }
357
358 +/*
359 + * If "pack" is non-NULL, then "offset" is the byte offset within the pack from
360 + * which the object may be accessed (though note that we may also rely on
361 + * data->oid, too). If "pack" is NULL, then offset is ignored.
362 + */
363 static void batch_object_write(const char *obj_name,
364 struct strbuf *scratch,
365 struct batch_options *opt,
361 - struct expand_data *data)
366 + struct expand_data *data,
367 + struct packed_git *pack,
368 + off_t offset)
369 {
363 - if (!data->skip_object_info &&
364 - oid_object_info_extended(the_repository, &data->oid, &data->info,
365 - OBJECT_INFO_LOOKUP_REPLACE) < 0) {
366 - printf("%s missing\n",
367 - obj_name ? obj_name : oid_to_hex(&data->oid));
368 - fflush(stdout);
369 - return;
370 + if (!data->skip_object_info) {
371 + int ret;
372 +
373 + if (pack)
374 + ret = packed_object_info(the_repository, pack, offset,
375 + &data->info);
376 + else
377 + ret = oid_object_info_extended(the_repository,
378 + &data->oid, &data->info,
379 + OBJECT_INFO_LOOKUP_REPLACE);
380 + if (ret < 0) {
381 + printf("%s missing\n",
382 + obj_name ? obj_name : oid_to_hex(&data->oid));
383 + fflush(stdout);
384 + return;
385 + }
386 }
387
388 strbuf_reset(scratch);
@@ -428,7 +444,7 @@ static void batch_one_object(const char *obj_name,
444 return;
445 }
446
431 - batch_object_write(obj_name, scratch, opt, data);
447 + batch_object_write(obj_name, scratch, opt, data, NULL, 0);
448 }
449
450 struct object_cb_data {
@@ -442,7 +458,8 @@ static int batch_object_cb(const struct object_id *oid, void *vdata)
458 {
459 struct object_cb_data *data = vdata;
460 oidcpy(&data->expand->oid, oid);
445 - batch_object_write(NULL, data->scratch, data->opt, data->expand);
461 + batch_object_write(NULL, data->scratch, data->opt, data->expand,
462 + NULL, 0);
463 return 0;
464 }
465
@@ -463,7 +480,9 @@ static int collect_packed_object(const struct object_id *oid,
480 return 0;
481 }
482
466 -static int batch_unordered_object(const struct object_id *oid, void *vdata)
483 +static int batch_unordered_object(const struct object_id *oid,
484 + struct packed_git *pack, off_t offset,
485 + void *vdata)
486 {
487 struct object_cb_data *data = vdata;
488
@@ -471,7 +490,8 @@ static int batch_unordered_object(const struct object_id *oid, void *vdata)
490 return 0;
491
492 oidcpy(&data->expand->oid, oid);
474 - batch_object_write(NULL, data->scratch, data->opt, data->expand);
493 + batch_object_write(NULL, data->scratch, data->opt, data->expand,
494 + pack, offset);
495 return 0;
496 }
497
@@ -479,7 +499,7 @@ static int batch_unordered_loose(const struct object_id *oid,
499 const char *path,
500 void *data)
501 {
482 - return batch_unordered_object(oid, data);
502 + return batch_unordered_object(oid, NULL, 0, data);
503 }
504
505 static int batch_unordered_packed(const struct object_id *oid,
@@ -487,7 +507,9 @@ static int batch_unordered_packed(const struct object_id *oid,
507 uint32_t pos,
508 void *data)
509 {
490 - return batch_unordered_object(oid, data);
510 + return batch_unordered_object(oid, pack,
511 + nth_packed_object_offset(pack, pos),
512 + data);
513 }
514
515 static int batch_objects(struct batch_options *opt)