pack-bitmap: remove bitmap_git global variable
Remove the bitmap_git global variable. Instead, generate on demand an instance of struct bitmap_index for code that needs to access it. This allows us significant control over the lifetime of instances of struct bitmap_index. In particular, packs can now be closed without worrying if an unnecessarily long-lived "pack" field in struct bitmap_index still points to it. The bitmap API is also clearer in that we need to first obtain a struct bitmap_index, then we use it. This patch raises two potential issues: (1) memory for the struct bitmap_index is allocated without being freed, and (2) prepare_bitmap_git() and prepare_bitmap_walk() can reuse a previously loaded bitmap. For (1), this will be dealt with in a subsequent patch in this patch set that also deals with freeing the contents of the struct bitmap_index (which were not freed previously, because they have global scope). For (2), current bitmap users only load the bitmap once at most (note that pack-objects can use bitmaps or write bitmaps, but not both at the same time), so support for reuse has no effect - and future users can pass around the struct bitmap_index * obtained if they need to do 2 or more things with the same bitmap. Helped-by: Stefan Beller <sbeller@google.com> Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Helped-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jonathan Tan committed
Jun 7, 2018 at 12:04 UTC
3ae5fa0768f7f9781b40b1d40cb2f9f4c753bad4
5 files changed
+201
-158
builtin/pack-objects.c
+4
-2
@@ -2929,11 +2929,13 @@ static int pack_options_allow_reuse(void)
2929
2930
static int get_object_list_from_bitmap(struct rev_info *revs)
2931
{
2932
- if (prepare_bitmap_walk(revs) < 0)
2932
+ struct bitmap_index *bitmap_git;
2933
+ if (!(bitmap_git = prepare_bitmap_walk(revs)))
2934
return -1;
2935
2936
if (pack_options_allow_reuse() &&
2937
!reuse_partial_packfile_from_bitmap(
2938
+ bitmap_git,
2939
&reuse_packfile,
2940
&reuse_packfile_objects,
2941
&reuse_packfile_offset)) {
@@ -2942,7 +2944,7 @@ static int get_object_list_from_bitmap(struct rev_info *revs)
2944
display_progress(progress_state, nr_result);
2945
}
2946
2945
- traverse_bitmap_commit_list(&add_object_entry_from_bitmap);
2947
+ traverse_bitmap_commit_list(bitmap_git, &add_object_entry_from_bitmap);
2948
return 0;
2949
}
2950
builtin/rev-list.c
+7
-4
@@ -16,6 +16,7 @@
16
#include "reflog-walk.h"
17
#include "oidset.h"
18
#include "packfile.h"
19
+#include "object-store.h"
20
21
static const char rev_list_usage[] =
22
"git rev-list [OPTION] <commit-id>... [ -- paths... ]\n"
@@ -514,8 +515,9 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix)
515
if (revs.count && !revs.left_right && !revs.cherry_mark) {
516
uint32_t commit_count;
517
int max_count = revs.max_count;
517
- if (!prepare_bitmap_walk(&revs)) {
518
- count_bitmap_commit_list(&commit_count, NULL, NULL, NULL);
518
+ struct bitmap_index *bitmap_git;
519
+ if ((bitmap_git = prepare_bitmap_walk(&revs))) {
520
+ count_bitmap_commit_list(bitmap_git, &commit_count, NULL, NULL, NULL);
521
if (max_count >= 0 && max_count < commit_count)
522
commit_count = max_count;
523
printf("%d\n", commit_count);
@@ -523,8 +525,9 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix)
525
}
526
} else if (revs.max_count < 0 &&
527
revs.tag_objects && revs.tree_objects && revs.blob_objects) {
526
- if (!prepare_bitmap_walk(&revs)) {
527
- traverse_bitmap_commit_list(&show_object_fast);
528
+ struct bitmap_index *bitmap_git;
529
+ if ((bitmap_git = prepare_bitmap_walk(&revs))) {
530
+ traverse_bitmap_commit_list(bitmap_git, &show_object_fast);
531
return 0;
532
}
533
}
pack-bitmap-write.c
+4
-2
@@ -360,11 +360,13 @@ static int date_compare(const void *_a, const void *_b)
360
361
void bitmap_writer_reuse_bitmaps(struct packing_data *to_pack)
362
{
363
- if (prepare_bitmap_git() < 0)
363
+ struct bitmap_index *bitmap_git;
364
+ if (!(bitmap_git = prepare_bitmap_git()))
365
return;
366
367
writer.reused = kh_init_sha1();
367
- rebuild_existing_bitmaps(to_pack, writer.reused, writer.show_progress);
368
+ rebuild_existing_bitmaps(bitmap_git, to_pack, writer.reused,
369
+ writer.show_progress);
370
}
371
372
static struct ewah_bitmap *find_reused_bitmap(const unsigned char *sha1)
pack-bitmap.c
+173
-144
@@ -25,14 +25,14 @@ struct stored_bitmap {
25
};
26
27
/*
28
- * The currently active bitmap index. By design, repositories only have
28
+ * The active bitmap index for a repository. By design, repositories only have
29
* a single bitmap index available (the index for the biggest packfile in
30
* the repository), since bitmap indexes need full closure.
31
*
32
* If there is more than one bitmap index available (e.g. because of alternates),
33
* the active bitmap index is the largest one.
34
*/
35
-static struct bitmap_index {
35
+struct bitmap_index {
36
/* Packfile to which this bitmap index belongs to */
37
struct packed_git *pack;
38
@@ -90,8 +90,7 @@ static struct bitmap_index {
90
unsigned int version;
91
92
unsigned loaded : 1;
93
-
94
-} bitmap_git;
93
+};
94
95
static struct ewah_bitmap *lookup_stored_bitmap(struct stored_bitmap *st)
96
{
@@ -259,7 +258,7 @@ static char *pack_bitmap_filename(struct packed_git *p)
258
return xstrfmt("%.*s.bitmap", (int)len, p->pack_name);
259
}
260
262
-static int open_pack_bitmap_1(struct packed_git *packfile)
261
+static int open_pack_bitmap_1(struct bitmap_index *bitmap_git, struct packed_git *packfile)
262
{
263
int fd;
264
struct stat st;
@@ -280,117 +279,121 @@ static int open_pack_bitmap_1(struct packed_git *packfile)
279
return -1;
280
}
281
283
- if (bitmap_git.pack) {
282
+ if (bitmap_git->pack) {
283
warning("ignoring extra bitmap file: %s", packfile->pack_name);
284
close(fd);
285
return -1;
286
}
287
289
- bitmap_git.pack = packfile;
290
- bitmap_git.map_size = xsize_t(st.st_size);
291
- bitmap_git.map = xmmap(NULL, bitmap_git.map_size, PROT_READ, MAP_PRIVATE, fd, 0);
292
- bitmap_git.map_pos = 0;
288
+ bitmap_git->pack = packfile;
289
+ bitmap_git->map_size = xsize_t(st.st_size);
290
+ bitmap_git->map = xmmap(NULL, bitmap_git->map_size, PROT_READ, MAP_PRIVATE, fd, 0);
291
+ bitmap_git->map_pos = 0;
292
close(fd);
293
295
- if (load_bitmap_header(&bitmap_git) < 0) {
296
- munmap(bitmap_git.map, bitmap_git.map_size);
297
- bitmap_git.map = NULL;
298
- bitmap_git.map_size = 0;
294
+ if (load_bitmap_header(bitmap_git) < 0) {
295
+ munmap(bitmap_git->map, bitmap_git->map_size);
296
+ bitmap_git->map = NULL;
297
+ bitmap_git->map_size = 0;
298
return -1;
299
}
300
301
return 0;
302
}
303
305
-static int load_pack_bitmap(void)
304
+static int load_pack_bitmap(struct bitmap_index *bitmap_git)
305
{
307
- assert(bitmap_git.map && !bitmap_git.loaded);
306
+ assert(bitmap_git->map && !bitmap_git->loaded);
307
309
- bitmap_git.bitmaps = kh_init_sha1();
310
- bitmap_git.ext_index.positions = kh_init_sha1_pos();
311
- load_pack_revindex(bitmap_git.pack);
308
+ bitmap_git->bitmaps = kh_init_sha1();
309
+ bitmap_git->ext_index.positions = kh_init_sha1_pos();
310
+ load_pack_revindex(bitmap_git->pack);
311
313
- if (!(bitmap_git.commits = read_bitmap_1(&bitmap_git)) ||
314
- !(bitmap_git.trees = read_bitmap_1(&bitmap_git)) ||
315
- !(bitmap_git.blobs = read_bitmap_1(&bitmap_git)) ||
316
- !(bitmap_git.tags = read_bitmap_1(&bitmap_git)))
312
+ if (!(bitmap_git->commits = read_bitmap_1(bitmap_git)) ||
313
+ !(bitmap_git->trees = read_bitmap_1(bitmap_git)) ||
314
+ !(bitmap_git->blobs = read_bitmap_1(bitmap_git)) ||
315
+ !(bitmap_git->tags = read_bitmap_1(bitmap_git)))
316
goto failed;
317
319
- if (load_bitmap_entries_v1(&bitmap_git) < 0)
318
+ if (load_bitmap_entries_v1(bitmap_git) < 0)
319
goto failed;
320
322
- bitmap_git.loaded = 1;
321
+ bitmap_git->loaded = 1;
322
return 0;
323
324
failed:
326
- munmap(bitmap_git.map, bitmap_git.map_size);
327
- bitmap_git.map = NULL;
328
- bitmap_git.map_size = 0;
325
+ munmap(bitmap_git->map, bitmap_git->map_size);
326
+ bitmap_git->map = NULL;
327
+ bitmap_git->map_size = 0;
328
return -1;
329
}
330
332
-static int open_pack_bitmap(void)
331
+static int open_pack_bitmap(struct bitmap_index *bitmap_git)
332
{
333
struct packed_git *p;
334
int ret = -1;
335
337
- assert(!bitmap_git.map && !bitmap_git.loaded);
336
+ assert(!bitmap_git->map && !bitmap_git->loaded);
337
338
for (p = get_packed_git(the_repository); p; p = p->next) {
340
- if (open_pack_bitmap_1(p) == 0)
339
+ if (open_pack_bitmap_1(bitmap_git, p) == 0)
340
ret = 0;
341
}
342
343
return ret;
344
}
345
347
-int prepare_bitmap_git(void)
346
+struct bitmap_index *prepare_bitmap_git(void)
347
{
349
- if (bitmap_git.loaded)
350
- return 0;
348
+ struct bitmap_index *bitmap_git = xcalloc(1, sizeof(*bitmap_git));
349
352
- if (!open_pack_bitmap())
353
- return load_pack_bitmap();
350
+ if (!open_pack_bitmap(bitmap_git) && !load_pack_bitmap(bitmap_git))
351
+ return bitmap_git;
352
355
- return -1;
353
+ return NULL;
354
}
355
356
struct include_data {
357
+ struct bitmap_index *bitmap_git;
358
struct bitmap *base;
359
struct bitmap *seen;
360
};
361
363
-static inline int bitmap_position_extended(const unsigned char *sha1)
362
+static inline int bitmap_position_extended(struct bitmap_index *bitmap_git,
363
+ const unsigned char *sha1)
364
{
365
- khash_sha1_pos *positions = bitmap_git.ext_index.positions;
365
+ khash_sha1_pos *positions = bitmap_git->ext_index.positions;
366
khiter_t pos = kh_get_sha1_pos(positions, sha1);
367
368
if (pos < kh_end(positions)) {
369
int bitmap_pos = kh_value(positions, pos);
370
- return bitmap_pos + bitmap_git.pack->num_objects;
370
+ return bitmap_pos + bitmap_git->pack->num_objects;
371
}
372
373
return -1;
374
}
375
376
-static inline int bitmap_position_packfile(const unsigned char *sha1)
376
+static inline int bitmap_position_packfile(struct bitmap_index *bitmap_git,
377
+ const unsigned char *sha1)
378
{
378
- off_t offset = find_pack_entry_one(sha1, bitmap_git.pack);
379
+ off_t offset = find_pack_entry_one(sha1, bitmap_git->pack);
380
if (!offset)
381
return -1;
382
382
- return find_revindex_position(bitmap_git.pack, offset);
383
+ return find_revindex_position(bitmap_git->pack, offset);
384
}
385
385
-static int bitmap_position(const unsigned char *sha1)
386
+static int bitmap_position(struct bitmap_index *bitmap_git,
387
+ const unsigned char *sha1)
388
{
387
- int pos = bitmap_position_packfile(sha1);
388
- return (pos >= 0) ? pos : bitmap_position_extended(sha1);
389
+ int pos = bitmap_position_packfile(bitmap_git, sha1);
390
+ return (pos >= 0) ? pos : bitmap_position_extended(bitmap_git, sha1);
391
}
392
391
-static int ext_index_add_object(struct object *object, const char *name)
393
+static int ext_index_add_object(struct bitmap_index *bitmap_git,
394
+ struct object *object, const char *name)
395
{
393
- struct eindex *eindex = &bitmap_git.ext_index;
396
+ struct eindex *eindex = &bitmap_git->ext_index;
397
398
khiter_t hash_pos;
399
int hash_ret;
@@ -413,27 +416,34 @@ static int ext_index_add_object(struct object *object, const char *name)
416
bitmap_pos = kh_value(eindex->positions, hash_pos);
417
}
418
416
- return bitmap_pos + bitmap_git.pack->num_objects;
419
+ return bitmap_pos + bitmap_git->pack->num_objects;
420
}
421
419
-static void show_object(struct object *object, const char *name, void *data)
422
+struct bitmap_show_data {
423
+ struct bitmap_index *bitmap_git;
424
+ struct bitmap *base;
425
+};
426
+
427
+static void show_object(struct object *object, const char *name, void *data_)
428
{
421
- struct bitmap *base = data;
429
+ struct bitmap_show_data *data = data_;
430
int bitmap_pos;
431
424
- bitmap_pos = bitmap_position(object->oid.hash);
432
+ bitmap_pos = bitmap_position(data->bitmap_git, object->oid.hash);
433
434
if (bitmap_pos < 0)
427
- bitmap_pos = ext_index_add_object(object, name);
435
+ bitmap_pos = ext_index_add_object(data->bitmap_git, object,
436
+ name);
437
429
- bitmap_set(base, bitmap_pos);
438
+ bitmap_set(data->base, bitmap_pos);
439
}
440
441
static void show_commit(struct commit *commit, void *data)
442
{
443
}
444
436
-static int add_to_include_set(struct include_data *data,
445
+static int add_to_include_set(struct bitmap_index *bitmap_git,
446
+ struct include_data *data,
447
const unsigned char *sha1,
448
int bitmap_pos)
449
{
@@ -445,9 +455,9 @@ static int add_to_include_set(struct include_data *data,
455
if (bitmap_get(data->base, bitmap_pos))
456
return 0;
457
448
- hash_pos = kh_get_sha1(bitmap_git.bitmaps, sha1);
449
- if (hash_pos < kh_end(bitmap_git.bitmaps)) {
450
- struct stored_bitmap *st = kh_value(bitmap_git.bitmaps, hash_pos);
458
+ hash_pos = kh_get_sha1(bitmap_git->bitmaps, sha1);
459
+ if (hash_pos < kh_end(bitmap_git->bitmaps)) {
460
+ struct stored_bitmap *st = kh_value(bitmap_git->bitmaps, hash_pos);
461
bitmap_or_ewah(data->base, lookup_stored_bitmap(st));
462
return 0;
463
}
@@ -461,11 +471,14 @@ static int should_include(struct commit *commit, void *_data)
471
struct include_data *data = _data;
472
int bitmap_pos;
473
464
- bitmap_pos = bitmap_position(commit->object.oid.hash);
474
+ bitmap_pos = bitmap_position(data->bitmap_git, commit->object.oid.hash);
475
if (bitmap_pos < 0)
466
- bitmap_pos = ext_index_add_object((struct object *)commit, NULL);
476
+ bitmap_pos = ext_index_add_object(data->bitmap_git,
477
+ (struct object *)commit,
478
+ NULL);
479
468
- if (!add_to_include_set(data, commit->object.oid.hash, bitmap_pos)) {
480
+ if (!add_to_include_set(data->bitmap_git, data, commit->object.oid.hash,
481
+ bitmap_pos)) {
482
struct commit_list *parent = commit->parents;
483
484
while (parent) {
@@ -479,7 +492,8 @@ static int should_include(struct commit *commit, void *_data)
492
return 1;
493
}
494
482
-static struct bitmap *find_objects(struct rev_info *revs,
495
+static struct bitmap *find_objects(struct bitmap_index *bitmap_git,
496
+ struct rev_info *revs,
497
struct object_list *roots,
498
struct bitmap *seen)
499
{
@@ -501,10 +515,10 @@ static struct bitmap *find_objects(struct rev_info *revs,
515
roots = roots->next;
516
517
if (object->type == OBJ_COMMIT) {
504
- khiter_t pos = kh_get_sha1(bitmap_git.bitmaps, object->oid.hash);
518
+ khiter_t pos = kh_get_sha1(bitmap_git->bitmaps, object->oid.hash);
519
506
- if (pos < kh_end(bitmap_git.bitmaps)) {
507
- struct stored_bitmap *st = kh_value(bitmap_git.bitmaps, pos);
520
+ if (pos < kh_end(bitmap_git->bitmaps)) {
521
+ struct stored_bitmap *st = kh_value(bitmap_git->bitmaps, pos);
522
struct ewah_bitmap *or_with = lookup_stored_bitmap(st);
523
524
if (base == NULL)
@@ -543,7 +557,7 @@ static struct bitmap *find_objects(struct rev_info *revs,
557
int pos;
558
559
roots = roots->next;
546
- pos = bitmap_position(object->oid.hash);
560
+ pos = bitmap_position(bitmap_git, object->oid.hash);
561
562
if (pos < 0 || base == NULL || !bitmap_get(base, pos)) {
563
object->flags &= ~UNINTERESTING;
@@ -556,10 +570,12 @@ static struct bitmap *find_objects(struct rev_info *revs,
570
571
if (needs_walk) {
572
struct include_data incdata;
573
+ struct bitmap_show_data show_data;
574
575
if (base == NULL)
576
base = bitmap_new();
577
578
+ incdata.bitmap_git = bitmap_git;
579
incdata.base = base;
580
incdata.seen = seen;
581
@@ -569,22 +585,27 @@ static struct bitmap *find_objects(struct rev_info *revs,
585
if (prepare_revision_walk(revs))
586
die("revision walk setup failed");
587
572
- traverse_commit_list(revs, show_commit, show_object, base);
588
+ show_data.bitmap_git = bitmap_git;
589
+ show_data.base = base;
590
+
591
+ traverse_commit_list(revs, show_commit, show_object,
592
+ &show_data);
593
}
594
595
return base;
596
}
597
578
-static void show_extended_objects(struct bitmap *objects,
598
+static void show_extended_objects(struct bitmap_index *bitmap_git,
599
show_reachable_fn show_reach)
600
{
581
- struct eindex *eindex = &bitmap_git.ext_index;
601
+ struct bitmap *objects = bitmap_git->result;
602
+ struct eindex *eindex = &bitmap_git->ext_index;
603
uint32_t i;
604
605
for (i = 0; i < eindex->count; ++i) {
606
struct object *obj;
607
587
- if (!bitmap_get(objects, bitmap_git.pack->num_objects + i))
608
+ if (!bitmap_get(objects, bitmap_git->pack->num_objects + i))
609
continue;
610
611
obj = eindex->objects[i];
@@ -593,7 +614,7 @@ static void show_extended_objects(struct bitmap *objects,
614
}
615
616
static void show_objects_for_type(
596
- struct bitmap *objects,
617
+ struct bitmap_index *bitmap_git,
618
struct ewah_bitmap *type_filter,
619
enum object_type object_type,
620
show_reachable_fn show_reach)
@@ -604,7 +625,9 @@ static void show_objects_for_type(
625
struct ewah_iterator it;
626
eword_t filter;
627
607
- if (bitmap_git.reuse_objects == bitmap_git.pack->num_objects)
628
+ struct bitmap *objects = bitmap_git->result;
629
+
630
+ if (bitmap_git->reuse_objects == bitmap_git->pack->num_objects)
631
return;
632
633
ewah_iterator_init(&it, type_filter);
@@ -622,16 +645,16 @@ static void show_objects_for_type(
645
646
offset += ewah_bit_ctz64(word >> offset);
647
625
- if (pos + offset < bitmap_git.reuse_objects)
648
+ if (pos + offset < bitmap_git->reuse_objects)
649
continue;
650
628
- entry = &bitmap_git.pack->revindex[pos + offset];
629
- nth_packed_object_oid(&oid, bitmap_git.pack, entry->nr);
651
+ entry = &bitmap_git->pack->revindex[pos + offset];
652
+ nth_packed_object_oid(&oid, bitmap_git->pack, entry->nr);
653
631
- if (bitmap_git.hashes)
632
- hash = get_be32(bitmap_git.hashes + entry->nr);
654
+ if (bitmap_git->hashes)
655
+ hash = get_be32(bitmap_git->hashes + entry->nr);
656
634
- show_reach(&oid, object_type, 0, hash, bitmap_git.pack, entry->offset);
657
+ show_reach(&oid, object_type, 0, hash, bitmap_git->pack, entry->offset);
658
}
659
660
pos += BITS_IN_EWORD;
@@ -639,20 +662,21 @@ static void show_objects_for_type(
662
}
663
}
664
642
-static int in_bitmapped_pack(struct object_list *roots)
665
+static int in_bitmapped_pack(struct bitmap_index *bitmap_git,
666
+ struct object_list *roots)
667
{
668
while (roots) {
669
struct object *object = roots->item;
670
roots = roots->next;
671
648
- if (find_pack_entry_one(object->oid.hash, bitmap_git.pack) > 0)
672
+ if (find_pack_entry_one(object->oid.hash, bitmap_git->pack) > 0)
673
return 1;
674
}
675
676
return 0;
677
}
678
655
-int prepare_bitmap_walk(struct rev_info *revs)
679
+struct bitmap_index *prepare_bitmap_walk(struct rev_info *revs)
680
{
681
unsigned int i;
682
@@ -662,12 +686,11 @@ int prepare_bitmap_walk(struct rev_info *revs)
686
struct bitmap *wants_bitmap = NULL;
687
struct bitmap *haves_bitmap = NULL;
688
665
- if (!bitmap_git.loaded) {
666
- /* try to open a bitmapped pack, but don't parse it yet
667
- * because we may not need to use it */
668
- if (open_pack_bitmap() < 0)
669
- return -1;
670
- }
689
+ struct bitmap_index *bitmap_git = xcalloc(1, sizeof(*bitmap_git));
690
+ /* try to open a bitmapped pack, but don't parse it yet
691
+ * because we may not need to use it */
692
+ if (open_pack_bitmap(bitmap_git) < 0)
693
+ return NULL;
694
695
for (i = 0; i < revs->pending.nr; ++i) {
696
struct object *object = revs->pending.objects[i].item;
@@ -699,26 +722,26 @@ int prepare_bitmap_walk(struct rev_info *revs)
722
* in the packfile that has a bitmap, we don't have anything to
723
* optimize here
724
*/
702
- if (haves && !in_bitmapped_pack(haves))
703
- return -1;
725
+ if (haves && !in_bitmapped_pack(bitmap_git, haves))
726
+ return NULL;
727
728
/* if we don't want anything, we're done here */
729
if (!wants)
707
- return -1;
730
+ return NULL;
731
732
/*
733
* now we're going to use bitmaps, so load the actual bitmap entries
734
* from disk. this is the point of no return; after this the rev_list
735
* becomes invalidated and we must perform the revwalk through bitmaps
736
*/
714
- if (!bitmap_git.loaded && load_pack_bitmap() < 0)
715
- return -1;
737
+ if (!bitmap_git->loaded && load_pack_bitmap(bitmap_git) < 0)
738
+ return NULL;
739
740
object_array_clear(&revs->pending);
741
742
if (haves) {
743
revs->ignore_missing_links = 1;
721
- haves_bitmap = find_objects(revs, haves, NULL);
744
+ haves_bitmap = find_objects(bitmap_git, revs, haves, NULL);
745
reset_revision_walk();
746
revs->ignore_missing_links = 0;
747
@@ -726,7 +749,7 @@ int prepare_bitmap_walk(struct rev_info *revs)
749
BUG("failed to perform bitmap walk");
750
}
751
729
- wants_bitmap = find_objects(revs, wants, haves_bitmap);
752
+ wants_bitmap = find_objects(bitmap_git, revs, wants, haves_bitmap);
753
754
if (!wants_bitmap)
755
BUG("failed to perform bitmap walk");
@@ -734,13 +757,14 @@ int prepare_bitmap_walk(struct rev_info *revs)
757
if (haves_bitmap)
758
bitmap_and_not(wants_bitmap, haves_bitmap);
759
737
- bitmap_git.result = wants_bitmap;
760
+ bitmap_git->result = wants_bitmap;
761
762
bitmap_free(haves_bitmap);
740
- return 0;
763
+ return bitmap_git;
764
}
765
743
-int reuse_partial_packfile_from_bitmap(struct packed_git **packfile,
766
+int reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git,
767
+ struct packed_git **packfile,
768
uint32_t *entries,
769
off_t *up_to)
770
{
@@ -750,7 +774,7 @@ int reuse_partial_packfile_from_bitmap(struct packed_git **packfile,
774
*/
775
static const double REUSE_PERCENT = 0.9;
776
753
- struct bitmap *result = bitmap_git.result;
777
+ struct bitmap *result = bitmap_git->result;
778
uint32_t reuse_threshold;
779
uint32_t i, reuse_objects = 0;
780
@@ -770,8 +794,8 @@ int reuse_partial_packfile_from_bitmap(struct packed_git **packfile,
794
const unsigned char *sha1;
795
struct revindex_entry *entry;
796
773
- entry = &bitmap_git.reverse_index->revindex[reuse_objects];
774
- sha1 = nth_packed_object_sha1(bitmap_git.pack, entry->nr);
797
+ entry = &bitmap_git->reverse_index->revindex[reuse_objects];
798
+ sha1 = nth_packed_object_sha1(bitmap_git->pack, entry->nr);
799
800
fprintf(stderr, "Failed to reuse at %d (%016llx)\n",
801
reuse_objects, result->words[i]);
@@ -782,48 +806,50 @@ int reuse_partial_packfile_from_bitmap(struct packed_git **packfile,
806
if (!reuse_objects)
807
return -1;
808
785
- if (reuse_objects >= bitmap_git.pack->num_objects) {
786
- bitmap_git.reuse_objects = *entries = bitmap_git.pack->num_objects;
809
+ if (reuse_objects >= bitmap_git->pack->num_objects) {
810
+ bitmap_git->reuse_objects = *entries = bitmap_git->pack->num_objects;
811
*up_to = -1; /* reuse the full pack */
788
- *packfile = bitmap_git.pack;
812
+ *packfile = bitmap_git->pack;
813
return 0;
814
}
815
792
- reuse_threshold = bitmap_popcount(bitmap_git.result) * REUSE_PERCENT;
816
+ reuse_threshold = bitmap_popcount(bitmap_git->result) * REUSE_PERCENT;
817
818
if (reuse_objects < reuse_threshold)
819
return -1;
820
797
- bitmap_git.reuse_objects = *entries = reuse_objects;
798
- *up_to = bitmap_git.pack->revindex[reuse_objects].offset;
799
- *packfile = bitmap_git.pack;
821
+ bitmap_git->reuse_objects = *entries = reuse_objects;
822
+ *up_to = bitmap_git->pack->revindex[reuse_objects].offset;
823
+ *packfile = bitmap_git->pack;
824
825
return 0;
826
}
827
804
-void traverse_bitmap_commit_list(show_reachable_fn show_reachable)
828
+void traverse_bitmap_commit_list(struct bitmap_index *bitmap_git,
829
+ show_reachable_fn show_reachable)
830
{
806
- assert(bitmap_git.result);
831
+ assert(bitmap_git->result);
832
808
- show_objects_for_type(bitmap_git.result, bitmap_git.commits,
833
+ show_objects_for_type(bitmap_git, bitmap_git->commits,
834
OBJ_COMMIT, show_reachable);
810
- show_objects_for_type(bitmap_git.result, bitmap_git.trees,
835
+ show_objects_for_type(bitmap_git, bitmap_git->trees,
836
OBJ_TREE, show_reachable);
812
- show_objects_for_type(bitmap_git.result, bitmap_git.blobs,
837
+ show_objects_for_type(bitmap_git, bitmap_git->blobs,
838
OBJ_BLOB, show_reachable);
814
- show_objects_for_type(bitmap_git.result, bitmap_git.tags,
839
+ show_objects_for_type(bitmap_git, bitmap_git->tags,
840
OBJ_TAG, show_reachable);
841
817
- show_extended_objects(bitmap_git.result, show_reachable);
842
+ show_extended_objects(bitmap_git, show_reachable);
843
819
- bitmap_free(bitmap_git.result);
820
- bitmap_git.result = NULL;
844
+ bitmap_free(bitmap_git->result);
845
+ bitmap_git->result = NULL;
846
}
847
823
-static uint32_t count_object_type(struct bitmap *objects,
848
+static uint32_t count_object_type(struct bitmap_index *bitmap_git,
849
enum object_type type)
850
{
826
- struct eindex *eindex = &bitmap_git.ext_index;
851
+ struct bitmap *objects = bitmap_git->result;
852
+ struct eindex *eindex = &bitmap_git->ext_index;
853
854
uint32_t i = 0, count = 0;
855
struct ewah_iterator it;
@@ -831,19 +857,19 @@ static uint32_t count_object_type(struct bitmap *objects,
857
858
switch (type) {
859
case OBJ_COMMIT:
834
- ewah_iterator_init(&it, bitmap_git.commits);
860
+ ewah_iterator_init(&it, bitmap_git->commits);
861
break;
862
863
case OBJ_TREE:
838
- ewah_iterator_init(&it, bitmap_git.trees);
864
+ ewah_iterator_init(&it, bitmap_git->trees);
865
break;
866
867
case OBJ_BLOB:
842
- ewah_iterator_init(&it, bitmap_git.blobs);
868
+ ewah_iterator_init(&it, bitmap_git->blobs);
869
break;
870
871
case OBJ_TAG:
846
- ewah_iterator_init(&it, bitmap_git.tags);
872
+ ewah_iterator_init(&it, bitmap_git->tags);
873
break;
874
875
default:
@@ -857,32 +883,34 @@ static uint32_t count_object_type(struct bitmap *objects,
883
884
for (i = 0; i < eindex->count; ++i) {
885
if (eindex->objects[i]->type == type &&
860
- bitmap_get(objects, bitmap_git.pack->num_objects + i))
886
+ bitmap_get(objects, bitmap_git->pack->num_objects + i))
887
count++;
888
}
889
890
return count;
891
}
892
867
-void count_bitmap_commit_list(uint32_t *commits, uint32_t *trees,
893
+void count_bitmap_commit_list(struct bitmap_index *bitmap_git,
894
+ uint32_t *commits, uint32_t *trees,
895
uint32_t *blobs, uint32_t *tags)
896
{
870
- assert(bitmap_git.result);
897
+ assert(bitmap_git->result);
898
899
if (commits)
873
- *commits = count_object_type(bitmap_git.result, OBJ_COMMIT);
900
+ *commits = count_object_type(bitmap_git, OBJ_COMMIT);
901
902
if (trees)
876
- *trees = count_object_type(bitmap_git.result, OBJ_TREE);
903
+ *trees = count_object_type(bitmap_git, OBJ_TREE);
904
905
if (blobs)
879
- *blobs = count_object_type(bitmap_git.result, OBJ_BLOB);
906
+ *blobs = count_object_type(bitmap_git, OBJ_BLOB);
907
908
if (tags)
882
- *tags = count_object_type(bitmap_git.result, OBJ_TAG);
909
+ *tags = count_object_type(bitmap_git, OBJ_TAG);
910
}
911
912
struct bitmap_test_data {
913
+ struct bitmap_index *bitmap_git;
914
struct bitmap *base;
915
struct progress *prg;
916
size_t seen;
@@ -894,7 +922,7 @@ static void test_show_object(struct object *object, const char *name,
922
struct bitmap_test_data *tdata = data;
923
int bitmap_pos;
924
897
- bitmap_pos = bitmap_position(object->oid.hash);
925
+ bitmap_pos = bitmap_position(tdata->bitmap_git, object->oid.hash);
926
if (bitmap_pos < 0)
927
die("Object not in bitmap: %s\n", oid_to_hex(&object->oid));
928
@@ -907,7 +935,8 @@ static void test_show_commit(struct commit *commit, void *data)
935
struct bitmap_test_data *tdata = data;
936
int bitmap_pos;
937
910
- bitmap_pos = bitmap_position(commit->object.oid.hash);
938
+ bitmap_pos = bitmap_position(tdata->bitmap_git,
939
+ commit->object.oid.hash);
940
if (bitmap_pos < 0)
941
die("Object not in bitmap: %s\n", oid_to_hex(&commit->object.oid));
942
@@ -922,21 +951,22 @@ void test_bitmap_walk(struct rev_info *revs)
951
khiter_t pos;
952
size_t result_popcnt;
953
struct bitmap_test_data tdata;
954
+ struct bitmap_index *bitmap_git;
955
926
- if (prepare_bitmap_git())
956
+ if (!(bitmap_git = prepare_bitmap_git()))
957
die("failed to load bitmap indexes");
958
959
if (revs->pending.nr != 1)
960
die("you must specify exactly one commit to test");
961
962
fprintf(stderr, "Bitmap v%d test (%d entries loaded)\n",
933
- bitmap_git.version, bitmap_git.entry_count);
963
+ bitmap_git->version, bitmap_git->entry_count);
964
965
root = revs->pending.objects[0].item;
936
- pos = kh_get_sha1(bitmap_git.bitmaps, root->oid.hash);
966
+ pos = kh_get_sha1(bitmap_git->bitmaps, root->oid.hash);
967
938
- if (pos < kh_end(bitmap_git.bitmaps)) {
939
- struct stored_bitmap *st = kh_value(bitmap_git.bitmaps, pos);
968
+ if (pos < kh_end(bitmap_git->bitmaps)) {
969
+ struct stored_bitmap *st = kh_value(bitmap_git->bitmaps, pos);
970
struct ewah_bitmap *bm = lookup_stored_bitmap(st);
971
972
fprintf(stderr, "Found bitmap for %s. %d bits / %08x checksum\n",
@@ -957,6 +987,7 @@ void test_bitmap_walk(struct rev_info *revs)
987
if (prepare_revision_walk(revs))
988
die("revision walk setup failed");
989
990
+ tdata.bitmap_git = bitmap_git;
991
tdata.base = bitmap_new();
992
tdata.prg = start_progress("Verifying bitmap entries", result_popcnt);
993
tdata.seen = 0;
@@ -1004,7 +1035,8 @@ static int rebuild_bitmap(uint32_t *reposition,
1035
return 0;
1036
}
1037
1007
-int rebuild_existing_bitmaps(struct packing_data *mapping,
1038
+int rebuild_existing_bitmaps(struct bitmap_index *bitmap_git,
1039
+ struct packing_data *mapping,
1040
khash_sha1 *reused_bitmaps,
1041
int show_progress)
1042
{
@@ -1017,10 +1049,7 @@ int rebuild_existing_bitmaps(struct packing_data *mapping,
1049
khiter_t hash_pos;
1050
int hash_ret;
1051
1020
- if (prepare_bitmap_git() < 0)
1021
- return -1;
1022
-
1023
- num_objects = bitmap_git.pack->num_objects;
1052
+ num_objects = bitmap_git->pack->num_objects;
1053
reposition = xcalloc(num_objects, sizeof(uint32_t));
1054
1055
for (i = 0; i < num_objects; ++i) {
@@ -1028,8 +1057,8 @@ int rebuild_existing_bitmaps(struct packing_data *mapping,
1057
struct revindex_entry *entry;
1058
struct object_entry *oe;
1059
1031
- entry = &bitmap_git.pack->revindex[i];
1032
- sha1 = nth_packed_object_sha1(bitmap_git.pack, entry->nr);
1060
+ entry = &bitmap_git->pack->revindex[i];
1061
+ sha1 = nth_packed_object_sha1(bitmap_git->pack, entry->nr);
1062
oe = packlist_find(mapping, sha1, NULL);
1063
1064
if (oe)
@@ -1042,7 +1071,7 @@ int rebuild_existing_bitmaps(struct packing_data *mapping,
1071
if (show_progress)
1072
progress = start_progress("Reusing bitmaps", 0);
1073
1045
- kh_foreach_value(bitmap_git.bitmaps, stored, {
1074
+ kh_foreach_value(bitmap_git->bitmaps, stored, {
1075
if (stored->flags & BITMAP_FLAG_REUSE) {
1076
if (!rebuild_bitmap(reposition,
1077
lookup_stored_bitmap(stored),
pack-bitmap.h
+13
-6
@@ -34,13 +34,20 @@ typedef int (*show_reachable_fn)(
34
struct packed_git *found_pack,
35
off_t found_offset);
36
37
-int prepare_bitmap_git(void);
38
-void count_bitmap_commit_list(uint32_t *commits, uint32_t *trees, uint32_t *blobs, uint32_t *tags);
39
-void traverse_bitmap_commit_list(show_reachable_fn show_reachable);
37
+struct bitmap_index;
38
+
39
+struct bitmap_index *prepare_bitmap_git(void);
40
+void count_bitmap_commit_list(struct bitmap_index *, uint32_t *commits,
41
+ uint32_t *trees, uint32_t *blobs, uint32_t *tags);
42
+void traverse_bitmap_commit_list(struct bitmap_index *,
43
+ show_reachable_fn show_reachable);
44
void test_bitmap_walk(struct rev_info *revs);
41
-int prepare_bitmap_walk(struct rev_info *revs);
42
-int reuse_partial_packfile_from_bitmap(struct packed_git **packfile, uint32_t *entries, off_t *up_to);
43
-int rebuild_existing_bitmaps(struct packing_data *mapping, khash_sha1 *reused_bitmaps, int show_progress);
45
+struct bitmap_index *prepare_bitmap_walk(struct rev_info *revs);
46
+int reuse_partial_packfile_from_bitmap(struct bitmap_index *,
47
+ struct packed_git **packfile,
48
+ uint32_t *entries, off_t *up_to);
49
+int rebuild_existing_bitmaps(struct bitmap_index *, struct packing_data *mapping,
50
+ khash_sha1 *reused_bitmaps, int show_progress);
51
52
void bitmap_writer_show_progress(int show);
53
void bitmap_writer_set_checksum(unsigned char *sha1);