pack-objects: refactor the packing list
The hash table that stores the packing list for a given `pack-objects` run was tightly coupled to the pack-objects code. In this commit, we refactor the hash table and the underlying storage array into a `packing_data` struct. The functionality for accessing and adding entries to the packing list is hence accessible from other parts of Git besides the `pack-objects` builtin. This refactoring is a requirement for further patches in this series that will require accessing the commit packing list from outside of `pack-objects`. The hash table implementation has been minimally altered: we now use table sizes which are always a power of two, to ensure a uniform index distribution in the array. Signed-off-by: Vicent Marti <tanoku@gmail.com> Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Vicent Marti committed
Oct 24, 2013 at 14:01 UTC
2834bc27c16e4ae103768dd6dda530f0e46116af
4 files changed
+200
-135
Makefile
+2
@@ -694,6 +694,7 @@ LIB_H += notes-merge.h
694
LIB_H += notes-utils.h
695
LIB_H += notes.h
696
LIB_H += object.h
697
+LIB_H += pack-objects.h
698
LIB_H += pack-revindex.h
699
LIB_H += pack.h
700
LIB_H += parse-options.h
@@ -831,6 +832,7 @@ LIB_OBJS += notes-merge.o
832
LIB_OBJS += notes-utils.o
833
LIB_OBJS += object.o
834
LIB_OBJS += pack-check.o
835
+LIB_OBJS += pack-objects.o
836
LIB_OBJS += pack-revindex.o
837
LIB_OBJS += pack-write.o
838
LIB_OBJS += pager.o
builtin/pack-objects.c
+40
-135
@@ -14,6 +14,7 @@
14
#include "diff.h"
15
#include "revision.h"
16
#include "list-objects.h"
17
+#include "pack-objects.h"
18
#include "progress.h"
19
#include "refs.h"
20
#include "streaming.h"
@@ -25,42 +26,15 @@ static const char *pack_usage[] = {
26
NULL
27
};
28
28
-struct object_entry {
29
- struct pack_idx_entry idx;
30
- unsigned long size; /* uncompressed size */
31
- struct packed_git *in_pack; /* already in pack */
32
- off_t in_pack_offset;
33
- struct object_entry *delta; /* delta base object */
34
- struct object_entry *delta_child; /* deltified objects who bases me */
35
- struct object_entry *delta_sibling; /* other deltified objects who
36
- * uses the same base as me
37
- */
38
- void *delta_data; /* cached delta (uncompressed) */
39
- unsigned long delta_size; /* delta data size (uncompressed) */
40
- unsigned long z_delta_size; /* delta data size (compressed) */
41
- enum object_type type;
42
- enum object_type in_pack_type; /* could be delta */
43
- uint32_t hash; /* name hint hash */
44
- unsigned char in_pack_header_size;
45
- unsigned preferred_base:1; /*
46
- * we do not pack this, but is available
47
- * to be used as the base object to delta
48
- * objects against.
49
- */
50
- unsigned no_try_delta:1;
51
- unsigned tagged:1; /* near the very tip of refs */
52
- unsigned filled:1; /* assigned write-order */
53
-};
54
-
29
/*
56
- * Objects we are going to pack are collected in objects array (dynamically
57
- * expanded). nr_objects & nr_alloc controls this array. They are stored
58
- * in the order we see -- typically rev-list --objects order that gives us
59
- * nice "minimum seek" order.
30
+ * Objects we are going to pack are collected in the `to_pack` structure.
31
+ * It contains an array (dynamically expanded) of the object data, and a map
32
+ * that can resolve SHA1s to their position in the array.
33
*/
61
-static struct object_entry *objects;
34
+static struct packing_data to_pack;
35
+
36
static struct pack_idx_entry **written_list;
63
-static uint32_t nr_objects, nr_alloc, nr_result, nr_written;
37
+static uint32_t nr_result, nr_written;
38
39
static int non_empty;
40
static int reuse_delta = 1, reuse_object = 1;
@@ -89,22 +63,12 @@ static unsigned long cache_max_small_delta_size = 1000;
63
64
static unsigned long window_memory_limit = 0;
65
92
-/*
93
- * The object names in objects array are hashed with this hashtable,
94
- * to help looking up the entry by object name.
95
- * This hashtable is built after all the objects are seen.
96
- */
97
-static int *object_ix;
98
-static int object_ix_hashsz;
99
-static struct object_entry *locate_object_entry(const unsigned char *sha1);
100
-
66
/*
67
* stats
68
*/
69
static uint32_t written, written_delta;
70
static uint32_t reused, reused_delta;
71
107
-
72
static void *get_delta(struct object_entry *entry)
73
{
74
unsigned long size, base_size, delta_size;
@@ -553,12 +517,12 @@ static int mark_tagged(const char *path, const unsigned char *sha1, int flag,
517
void *cb_data)
518
{
519
unsigned char peeled[20];
556
- struct object_entry *entry = locate_object_entry(sha1);
520
+ struct object_entry *entry = packlist_find(&to_pack, sha1, NULL);
521
522
if (entry)
523
entry->tagged = 1;
524
if (!peel_ref(path, peeled)) {
561
- entry = locate_object_entry(peeled);
525
+ entry = packlist_find(&to_pack, peeled, NULL);
526
if (entry)
527
entry->tagged = 1;
528
}
@@ -633,9 +597,10 @@ static struct object_entry **compute_write_order(void)
597
{
598
unsigned int i, wo_end, last_untagged;
599
636
- struct object_entry **wo = xmalloc(nr_objects * sizeof(*wo));
600
+ struct object_entry **wo = xmalloc(to_pack.nr_objects * sizeof(*wo));
601
+ struct object_entry *objects = to_pack.objects;
602
638
- for (i = 0; i < nr_objects; i++) {
603
+ for (i = 0; i < to_pack.nr_objects; i++) {
604
objects[i].tagged = 0;
605
objects[i].filled = 0;
606
objects[i].delta_child = NULL;
@@ -647,7 +612,7 @@ static struct object_entry **compute_write_order(void)
612
* Make sure delta_sibling is sorted in the original
613
* recency order.
614
*/
650
- for (i = nr_objects; i > 0;) {
615
+ for (i = to_pack.nr_objects; i > 0;) {
616
struct object_entry *e = &objects[--i];
617
if (!e->delta)
618
continue;
@@ -665,7 +630,7 @@ static struct object_entry **compute_write_order(void)
630
* Give the objects in the original recency order until
631
* we see a tagged tip.
632
*/
668
- for (i = wo_end = 0; i < nr_objects; i++) {
633
+ for (i = wo_end = 0; i < to_pack.nr_objects; i++) {
634
if (objects[i].tagged)
635
break;
636
add_to_write_order(wo, &wo_end, &objects[i]);
@@ -675,7 +640,7 @@ static struct object_entry **compute_write_order(void)
640
/*
641
* Then fill all the tagged tips.
642
*/
678
- for (; i < nr_objects; i++) {
643
+ for (; i < to_pack.nr_objects; i++) {
644
if (objects[i].tagged)
645
add_to_write_order(wo, &wo_end, &objects[i]);
646
}
@@ -683,7 +648,7 @@ static struct object_entry **compute_write_order(void)
648
/*
649
* And then all remaining commits and tags.
650
*/
686
- for (i = last_untagged; i < nr_objects; i++) {
651
+ for (i = last_untagged; i < to_pack.nr_objects; i++) {
652
if (objects[i].type != OBJ_COMMIT &&
653
objects[i].type != OBJ_TAG)
654
continue;
@@ -693,7 +658,7 @@ static struct object_entry **compute_write_order(void)
658
/*
659
* And then all the trees.
660
*/
696
- for (i = last_untagged; i < nr_objects; i++) {
661
+ for (i = last_untagged; i < to_pack.nr_objects; i++) {
662
if (objects[i].type != OBJ_TREE)
663
continue;
664
add_to_write_order(wo, &wo_end, &objects[i]);
@@ -702,13 +667,13 @@ static struct object_entry **compute_write_order(void)
667
/*
668
* Finally all the rest in really tight order
669
*/
705
- for (i = last_untagged; i < nr_objects; i++) {
670
+ for (i = last_untagged; i < to_pack.nr_objects; i++) {
671
if (!objects[i].filled)
672
add_family_to_write_order(wo, &wo_end, &objects[i]);
673
}
674
710
- if (wo_end != nr_objects)
711
- die("ordered %u objects, expected %"PRIu32, wo_end, nr_objects);
675
+ if (wo_end != to_pack.nr_objects)
676
+ die("ordered %u objects, expected %"PRIu32, wo_end, to_pack.nr_objects);
677
678
return wo;
679
}
@@ -724,7 +689,7 @@ static void write_pack_file(void)
689
690
if (progress > pack_to_stdout)
691
progress_state = start_progress("Writing objects", nr_result);
727
- written_list = xmalloc(nr_objects * sizeof(*written_list));
692
+ written_list = xmalloc(to_pack.nr_objects * sizeof(*written_list));
693
write_order = compute_write_order();
694
695
do {
@@ -740,7 +705,7 @@ static void write_pack_file(void)
705
if (!offset)
706
die_errno("unable to write pack header");
707
nr_written = 0;
743
- for (; i < nr_objects; i++) {
708
+ for (; i < to_pack.nr_objects; i++) {
709
struct object_entry *e = write_order[i];
710
if (write_one(f, e, &offset) == WRITE_ONE_BREAK)
711
break;
@@ -803,7 +768,7 @@ static void write_pack_file(void)
768
written_list[j]->offset = (off_t)-1;
769
}
770
nr_remaining -= nr_written;
806
- } while (nr_remaining && i < nr_objects);
771
+ } while (nr_remaining && i < to_pack.nr_objects);
772
773
free(written_list);
774
free(write_order);
@@ -813,53 +778,6 @@ static void write_pack_file(void)
778
written, nr_result);
779
}
780
816
-static int locate_object_entry_hash(const unsigned char *sha1)
817
-{
818
- int i;
819
- unsigned int ui;
820
- memcpy(&ui, sha1, sizeof(unsigned int));
821
- i = ui % object_ix_hashsz;
822
- while (0 < object_ix[i]) {
823
- if (!hashcmp(sha1, objects[object_ix[i] - 1].idx.sha1))
824
- return i;
825
- if (++i == object_ix_hashsz)
826
- i = 0;
827
- }
828
- return -1 - i;
829
-}
830
-
831
-static struct object_entry *locate_object_entry(const unsigned char *sha1)
832
-{
833
- int i;
834
-
835
- if (!object_ix_hashsz)
836
- return NULL;
837
-
838
- i = locate_object_entry_hash(sha1);
839
- if (0 <= i)
840
- return &objects[object_ix[i]-1];
841
- return NULL;
842
-}
843
-
844
-static void rehash_objects(void)
845
-{
846
- uint32_t i;
847
- struct object_entry *oe;
848
-
849
- object_ix_hashsz = nr_objects * 3;
850
- if (object_ix_hashsz < 1024)
851
- object_ix_hashsz = 1024;
852
- object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz);
853
- memset(object_ix, 0, sizeof(int) * object_ix_hashsz);
854
- for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
855
- int ix = locate_object_entry_hash(oe->idx.sha1);
856
- if (0 <= ix)
857
- continue;
858
- ix = -1 - ix;
859
- object_ix[ix] = i + 1;
860
- }
861
-}
862
-
781
static uint32_t name_hash(const char *name)
782
{
783
uint32_t c, hash = 0;
@@ -908,13 +826,12 @@ static int add_object_entry(const unsigned char *sha1, enum object_type type,
826
struct object_entry *entry;
827
struct packed_git *p, *found_pack = NULL;
828
off_t found_offset = 0;
911
- int ix;
829
uint32_t hash = name_hash(name);
830
+ uint32_t index_pos;
831
914
- ix = nr_objects ? locate_object_entry_hash(sha1) : -1;
915
- if (ix >= 0) {
832
+ entry = packlist_find(&to_pack, sha1, &index_pos);
833
+ if (entry) {
834
if (exclude) {
917
- entry = objects + object_ix[ix] - 1;
835
if (!entry->preferred_base)
836
nr_result--;
837
entry->preferred_base = 1;
@@ -947,14 +864,7 @@ static int add_object_entry(const unsigned char *sha1, enum object_type type,
864
}
865
}
866
950
- if (nr_objects >= nr_alloc) {
951
- nr_alloc = (nr_alloc + 1024) * 3 / 2;
952
- objects = xrealloc(objects, nr_alloc * sizeof(*entry));
953
- }
954
-
955
- entry = objects + nr_objects++;
956
- memset(entry, 0, sizeof(*entry));
957
- hashcpy(entry->idx.sha1, sha1);
867
+ entry = packlist_alloc(&to_pack, sha1, index_pos);
868
entry->hash = hash;
869
if (type)
870
entry->type = type;
@@ -967,12 +877,7 @@ static int add_object_entry(const unsigned char *sha1, enum object_type type,
877
entry->in_pack_offset = found_offset;
878
}
879
970
- if (object_ix_hashsz * 3 <= nr_objects * 4)
971
- rehash_objects();
972
- else
973
- object_ix[-1 - ix] = nr_objects;
974
-
975
- display_progress(progress_state, nr_objects);
880
+ display_progress(progress_state, to_pack.nr_objects);
881
882
if (name && no_try_delta(name))
883
entry->no_try_delta = 1;
@@ -1329,7 +1234,7 @@ static void check_object(struct object_entry *entry)
1234
break;
1235
}
1236
1332
- if (base_ref && (base_entry = locate_object_entry(base_ref))) {
1237
+ if (base_ref && (base_entry = packlist_find(&to_pack, base_ref, NULL))) {
1238
/*
1239
* If base_ref was set above that means we wish to
1240
* reuse delta data, and we even found that base
@@ -1403,12 +1308,12 @@ static void get_object_details(void)
1308
uint32_t i;
1309
struct object_entry **sorted_by_offset;
1310
1406
- sorted_by_offset = xcalloc(nr_objects, sizeof(struct object_entry *));
1407
- for (i = 0; i < nr_objects; i++)
1408
- sorted_by_offset[i] = objects + i;
1409
- qsort(sorted_by_offset, nr_objects, sizeof(*sorted_by_offset), pack_offset_sort);
1311
+ sorted_by_offset = xcalloc(to_pack.nr_objects, sizeof(struct object_entry *));
1312
+ for (i = 0; i < to_pack.nr_objects; i++)
1313
+ sorted_by_offset[i] = to_pack.objects + i;
1314
+ qsort(sorted_by_offset, to_pack.nr_objects, sizeof(*sorted_by_offset), pack_offset_sort);
1315
1411
- for (i = 0; i < nr_objects; i++) {
1316
+ for (i = 0; i < to_pack.nr_objects; i++) {
1317
struct object_entry *entry = sorted_by_offset[i];
1318
check_object(entry);
1319
if (big_file_threshold < entry->size)
@@ -2034,7 +1939,7 @@ static int add_ref_tag(const char *path, const unsigned char *sha1, int flag, vo
1939
1940
if (!prefixcmp(path, "refs/tags/") && /* is a tag? */
1941
!peel_ref(path, peeled) && /* peelable? */
2037
- locate_object_entry(peeled)) /* object packed? */
1942
+ packlist_find(&to_pack, peeled, NULL)) /* object packed? */
1943
add_object_entry(sha1, OBJ_TAG, NULL, 0);
1944
return 0;
1945
}
@@ -2057,14 +1962,14 @@ static void prepare_pack(int window, int depth)
1962
if (!pack_to_stdout)
1963
do_check_packed_object_crc = 1;
1964
2060
- if (!nr_objects || !window || !depth)
1965
+ if (!to_pack.nr_objects || !window || !depth)
1966
return;
1967
2063
- delta_list = xmalloc(nr_objects * sizeof(*delta_list));
1968
+ delta_list = xmalloc(to_pack.nr_objects * sizeof(*delta_list));
1969
nr_deltas = n = 0;
1970
2066
- for (i = 0; i < nr_objects; i++) {
2067
- struct object_entry *entry = objects + i;
1971
+ for (i = 0; i < to_pack.nr_objects; i++) {
1972
+ struct object_entry *entry = to_pack.objects + i;
1973
1974
if (entry->delta)
1975
/* This happens if we decided to reuse existing
@@ -2342,7 +2247,7 @@ static void loosen_unused_packed_objects(struct rev_info *revs)
2247
2248
for (i = 0; i < p->num_objects; i++) {
2249
sha1 = nth_packed_object_sha1(p, i);
2345
- if (!locate_object_entry(sha1) &&
2250
+ if (!packlist_find(&to_pack, sha1, NULL) &&
2251
!has_sha1_pack_kept_or_nonlocal(sha1))
2252
if (force_object_loose(sha1, p->mtime))
2253
die("unable to force loose object");
pack-objects.c
new
+111
@@ -0,0 +1,111 @@
1
+#include "cache.h"
2
+#include "object.h"
3
+#include "pack.h"
4
+#include "pack-objects.h"
5
+
6
+static uint32_t locate_object_entry_hash(struct packing_data *pdata,
7
+ const unsigned char *sha1,
8
+ int *found)
9
+{
10
+ uint32_t i, hash, mask = (pdata->index_size - 1);
11
+
12
+ memcpy(&hash, sha1, sizeof(uint32_t));
13
+ i = hash & mask;
14
+
15
+ while (pdata->index[i] > 0) {
16
+ uint32_t pos = pdata->index[i] - 1;
17
+
18
+ if (!hashcmp(sha1, pdata->objects[pos].idx.sha1)) {
19
+ *found = 1;
20
+ return i;
21
+ }
22
+
23
+ i = (i + 1) & mask;
24
+ }
25
+
26
+ *found = 0;
27
+ return i;
28
+}
29
+
30
+static inline uint32_t closest_pow2(uint32_t v)
31
+{
32
+ v = v - 1;
33
+ v |= v >> 1;
34
+ v |= v >> 2;
35
+ v |= v >> 4;
36
+ v |= v >> 8;
37
+ v |= v >> 16;
38
+ return v + 1;
39
+}
40
+
41
+static void rehash_objects(struct packing_data *pdata)
42
+{
43
+ uint32_t i;
44
+ struct object_entry *entry;
45
+
46
+ pdata->index_size = closest_pow2(pdata->nr_objects * 3);
47
+ if (pdata->index_size < 1024)
48
+ pdata->index_size = 1024;
49
+
50
+ pdata->index = xrealloc(pdata->index, sizeof(uint32_t) * pdata->index_size);
51
+ memset(pdata->index, 0, sizeof(int) * pdata->index_size);
52
+
53
+ entry = pdata->objects;
54
+
55
+ for (i = 0; i < pdata->nr_objects; i++) {
56
+ int found;
57
+ uint32_t ix = locate_object_entry_hash(pdata, entry->idx.sha1, &found);
58
+
59
+ if (found)
60
+ die("BUG: Duplicate object in hash");
61
+
62
+ pdata->index[ix] = i + 1;
63
+ entry++;
64
+ }
65
+}
66
+
67
+struct object_entry *packlist_find(struct packing_data *pdata,
68
+ const unsigned char *sha1,
69
+ uint32_t *index_pos)
70
+{
71
+ uint32_t i;
72
+ int found;
73
+
74
+ if (!pdata->index_size)
75
+ return NULL;
76
+
77
+ i = locate_object_entry_hash(pdata, sha1, &found);
78
+
79
+ if (index_pos)
80
+ *index_pos = i;
81
+
82
+ if (!found)
83
+ return NULL;
84
+
85
+ return &pdata->objects[pdata->index[i] - 1];
86
+}
87
+
88
+struct object_entry *packlist_alloc(struct packing_data *pdata,
89
+ const unsigned char *sha1,
90
+ uint32_t index_pos)
91
+{
92
+ struct object_entry *new_entry;
93
+
94
+ if (pdata->nr_objects >= pdata->nr_alloc) {
95
+ pdata->nr_alloc = (pdata->nr_alloc + 1024) * 3 / 2;
96
+ pdata->objects = xrealloc(pdata->objects,
97
+ pdata->nr_alloc * sizeof(*new_entry));
98
+ }
99
+
100
+ new_entry = pdata->objects + pdata->nr_objects++;
101
+
102
+ memset(new_entry, 0, sizeof(*new_entry));
103
+ hashcpy(new_entry->idx.sha1, sha1);
104
+
105
+ if (pdata->index_size * 3 <= pdata->nr_objects * 4)
106
+ rehash_objects(pdata);
107
+ else
108
+ pdata->index[index_pos] = pdata->nr_objects;
109
+
110
+ return new_entry;
111
+}
pack-objects.h
new
+47
@@ -0,0 +1,47 @@
1
+#ifndef PACK_OBJECTS_H
2
+#define PACK_OBJECTS_H
3
+
4
+struct object_entry {
5
+ struct pack_idx_entry idx;
6
+ unsigned long size; /* uncompressed size */
7
+ struct packed_git *in_pack; /* already in pack */
8
+ off_t in_pack_offset;
9
+ struct object_entry *delta; /* delta base object */
10
+ struct object_entry *delta_child; /* deltified objects who bases me */
11
+ struct object_entry *delta_sibling; /* other deltified objects who
12
+ * uses the same base as me
13
+ */
14
+ void *delta_data; /* cached delta (uncompressed) */
15
+ unsigned long delta_size; /* delta data size (uncompressed) */
16
+ unsigned long z_delta_size; /* delta data size (compressed) */
17
+ enum object_type type;
18
+ enum object_type in_pack_type; /* could be delta */
19
+ uint32_t hash; /* name hint hash */
20
+ unsigned char in_pack_header_size;
21
+ unsigned preferred_base:1; /*
22
+ * we do not pack this, but is available
23
+ * to be used as the base object to delta
24
+ * objects against.
25
+ */
26
+ unsigned no_try_delta:1;
27
+ unsigned tagged:1; /* near the very tip of refs */
28
+ unsigned filled:1; /* assigned write-order */
29
+};
30
+
31
+struct packing_data {
32
+ struct object_entry *objects;
33
+ uint32_t nr_objects, nr_alloc;
34
+
35
+ int32_t *index;
36
+ uint32_t index_size;
37
+};
38
+
39
+struct object_entry *packlist_alloc(struct packing_data *pdata,
40
+ const unsigned char *sha1,
41
+ uint32_t index_pos);
42
+
43
+struct object_entry *packlist_find(struct packing_data *pdata,
44
+ const unsigned char *sha1,
45
+ uint32_t *index_pos);
46
+
47
+#endif