@matheusbsbs / matheus-bagliano / commits / a49dd05fd0
Fork 0

pack-objects: reuse data from existing packs.

When generating a new pack, notice if we have already needed objects in existing packs. If an object is stored deltified, and its base object is also what we are going to pack, then reuse the existing deltified representation unconditionally, bypassing all the expensive find_deltas() and try_deltas() calls. Also, notice if what we are going to write out exactly match what is already in an existing pack (either deltified or just compressed). In such a case, we can just copy it instead of going through the usual uncompressing & recompressing cycle. Without this patch, in linux-2.6 repository with about 1500 loose objects and a single mega pack: $ git-rev-list --objects v2.6.16-rc3 >RL $ wc -l RL 184141 RL $ time git-pack-objects p <RL Generating pack... Done counting 184141 objects. Packing 184141 objects.................... a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2 real 12m4.323s user 11m2.560s sys 0m55.950s With this patch, the same input: $ time ../git.junio/git-pack-objects q <RL Generating pack... Done counting 184141 objects. Packing 184141 objects..................... a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2 Total 184141, written 184141, reused 182441 real 1m2.608s user 0m55.090s sys 0m1.830s Signed-off-by: Junio C Hamano <junkio@cox.net>

Junio C Hamano committed Feb 15, 2006 at 17:34 UTC a49dd05fd047f504a74fb053bb8ddbb9e4bd152b
3 files changed +332 -58
pack-objects.c
+310 -57
@@ -9,15 +9,31 @@ static const char pack_usage[] = "git-pack-objects [-q] [--non-empty] [--local]
9
10 struct object_entry {
11 unsigned char sha1[20];
12 - unsigned long size;
13 - unsigned long offset;
14 - unsigned int depth;
15 - unsigned int hash;
12 + unsigned long size; /* uncompressed size */
13 + unsigned long offset; /* offset into the final pack file (nonzero if already written) */
14 + unsigned int depth; /* delta depth */
15 + unsigned int hash; /* name hint hash */
16 enum object_type type;
17 - unsigned long delta_size;
18 - struct object_entry *delta;
17 + unsigned long delta_size; /* delta data size (uncompressed) */
18 + struct object_entry *delta; /* delta base object */
19 + struct packed_git *in_pack; /* already in pack */
20 + enum object_type in_pack_type; /* could be delta */
21 + unsigned int in_pack_offset;
22 };
23
24 +/*
25 + * Objects we are going to pack are colected in objects array (dynamically
26 + * expanded). nr_objects & nr_alloc controls this array. They are stored
27 + * in the order we see -- typically rev-list --objects order that gives us
28 + * nice "minimum seek" order.
29 + *
30 + * sorted-by-sha ans sorted-by-type are arrays of pointers that point at
31 + * elements in the objects array. The former is used to build the pack
32 + * index (lists object names in the ascending order to help offset lookup),
33 + * and the latter is used to group similar things together by try_delta()
34 + * heuristics.
35 + */
36 +
37 static unsigned char object_list_sha1[20];
38 static int non_empty = 0;
39 static int local = 0;
@@ -29,6 +45,135 @@ static const char *base_name;
45 static unsigned char pack_file_sha1[20];
46 static int progress = 1;
47
48 +/*
49 + * The object names in objects array are hashed with this hashtable,
50 + * to help looking up the entry by object name. Binary search from
51 + * sorted_by_sha is also possible but this was easier to code and faster.
52 + * This hashtable is built after all the objects are seen.
53 + */
54 +static int *object_ix = NULL;
55 +static int object_ix_hashsz = 0;
56 +
57 +/*
58 + * Pack index for existing packs give us easy access to the offsets into
59 + * corresponding pack file where each object's data starts, but the entries
60 + * do not store the size of the compressed representation (uncompressed
61 + * size is easily available by examining the pack entry header). We build
62 + * a hashtable of existing packs (pack_revindex), and keep reverse index
63 + * here -- pack index file is sorted by object name mapping to offset; this
64 + * pack_revindex[].revindex array is an ordered list of offsets, so if you
65 + * know the offset of an object, next offset is where its packed
66 + * representation ends.
67 + */
68 +struct pack_revindex {
69 + struct packed_git *p;
70 + unsigned long *revindex;
71 +} *pack_revindex = NULL;
72 +static int pack_revindex_hashsz = 0;
73 +
74 +/*
75 + * stats
76 + */
77 +static int written = 0;
78 +static int reused = 0;
79 +
80 +static int pack_revindex_ix(struct packed_git *p)
81 +{
82 + unsigned int ui = (unsigned int) p;
83 + int i;
84 +
85 + ui = ui ^ (ui >> 16); /* defeat structure alignment */
86 + i = (int)(ui % pack_revindex_hashsz);
87 + while (pack_revindex[i].p) {
88 + if (pack_revindex[i].p == p)
89 + return i;
90 + if (++i == pack_revindex_hashsz)
91 + i = 0;
92 + }
93 + return -1 - i;
94 +}
95 +
96 +static void prepare_pack_ix(void)
97 +{
98 + int num;
99 + struct packed_git *p;
100 + for (num = 0, p = packed_git; p; p = p->next)
101 + num++;
102 + if (!num)
103 + return;
104 + pack_revindex_hashsz = num * 11;
105 + pack_revindex = xcalloc(sizeof(*pack_revindex), pack_revindex_hashsz);
106 + for (p = packed_git; p; p = p->next) {
107 + num = pack_revindex_ix(p);
108 + num = - 1 - num;
109 + pack_revindex[num].p = p;
110 + }
111 + /* revindex elements are lazily initialized */
112 +}
113 +
114 +static int cmp_offset(const void *a_, const void *b_)
115 +{
116 + unsigned long a = *(unsigned long *) a_;
117 + unsigned long b = *(unsigned long *) b_;
118 + if (a < b)
119 + return -1;
120 + else if (a == b)
121 + return 0;
122 + else
123 + return 1;
124 +}
125 +
126 +/*
127 + * Ordered list of offsets of objects in the pack.
128 + */
129 +static void prepare_pack_revindex(struct pack_revindex *rix)
130 +{
131 + struct packed_git *p = rix->p;
132 + int num_ent = num_packed_objects(p);
133 + int i;
134 + void *index = p->index_base + 256;
135 +
136 + rix->revindex = xmalloc(sizeof(unsigned long) * (num_ent + 1));
137 + for (i = 0; i < num_ent; i++) {
138 + long hl = *((long *)(index + 24 * i));
139 + rix->revindex[i] = ntohl(hl);
140 + }
141 + /* This knows the pack format -- the 20-byte trailer
142 + * follows immediately after the last object data.
143 + */
144 + rix->revindex[num_ent] = p->pack_size - 20;
145 + qsort(rix->revindex, num_ent, sizeof(unsigned long), cmp_offset);
146 +}
147 +
148 +static unsigned long find_packed_object_size(struct packed_git *p,
149 + unsigned long ofs)
150 +{
151 + int num;
152 + int lo, hi;
153 + struct pack_revindex *rix;
154 + unsigned long *revindex;
155 + num = pack_revindex_ix(p);
156 + if (num < 0)
157 + die("internal error: pack revindex uninitialized");
158 + rix = &pack_revindex[num];
159 + if (!rix->revindex)
160 + prepare_pack_revindex(rix);
161 + revindex = rix->revindex;
162 + lo = 0;
163 + hi = num_packed_objects(p) + 1;
164 + do {
165 + int mi = (lo + hi) / 2;
166 + if (revindex[mi] == ofs) {
167 + return revindex[mi+1] - ofs;
168 + }
169 + else if (ofs < revindex[mi])
170 + hi = mi;
171 + else
172 + lo = mi + 1;
173 + } while (lo < hi);
174 + die("internal error: pack revindex corrupt");
175 +}
176 +
177 static void *delta_against(void *buf, unsigned long size, struct object_entry *entry)
178 {
179 unsigned long othersize, delta_size;
@@ -78,35 +223,52 @@ static unsigned long write_object(struct sha1file *f, struct object_entry *entry
223 {
224 unsigned long size;
225 char type[10];
81 - void *buf = read_sha1_file(entry->sha1, type, &size);
226 + void *buf;
227 unsigned char header[10];
228 unsigned hdrlen, datalen;
229 enum object_type obj_type;
230
86 - if (!buf)
87 - die("unable to read %s", sha1_to_hex(entry->sha1));
88 - if (size != entry->size)
89 - die("object %s size inconsistency (%lu vs %lu)", sha1_to_hex(entry->sha1), size, entry->size);
90 -
91 - /*
92 - * The object header is a byte of 'type' followed by zero or
93 - * more bytes of length. For deltas, the 20 bytes of delta sha1
94 - * follows that.
95 - */
231 obj_type = entry->type;
97 - if (entry->delta) {
98 - buf = delta_against(buf, size, entry);
99 - size = entry->delta_size;
100 - obj_type = OBJ_DELTA;
232 + if (!entry->in_pack ||
233 + (obj_type != entry->in_pack_type)) {
234 + buf = read_sha1_file(entry->sha1, type, &size);
235 + if (!buf)
236 + die("unable to read %s", sha1_to_hex(entry->sha1));
237 + if (size != entry->size)
238 + die("object %s size inconsistency (%lu vs %lu)",
239 + sha1_to_hex(entry->sha1), size, entry->size);
240 + if (entry->delta) {
241 + buf = delta_against(buf, size, entry);
242 + size = entry->delta_size;
243 + obj_type = OBJ_DELTA;
244 + }
245 + /*
246 + * The object header is a byte of 'type' followed by zero or
247 + * more bytes of length. For deltas, the 20 bytes of delta
248 + * sha1 follows that.
249 + */
250 + hdrlen = encode_header(obj_type, size, header);
251 + sha1write(f, header, hdrlen);
252 +
253 + if (entry->delta) {
254 + sha1write(f, entry->delta, 20);
255 + hdrlen += 20;
256 + }
257 + datalen = sha1write_compressed(f, buf, size);
258 + free(buf);
259 }
102 - hdrlen = encode_header(obj_type, size, header);
103 - sha1write(f, header, hdrlen);
104 - if (entry->delta) {
105 - sha1write(f, entry->delta, 20);
106 - hdrlen += 20;
260 + else {
261 + struct packed_git *p = entry->in_pack;
262 + use_packed_git(p);
263 +
264 + datalen = find_packed_object_size(p, entry->in_pack_offset);
265 + buf = p->pack_base + entry->in_pack_offset;
266 + sha1write(f, buf, datalen);
267 + unuse_packed_git(p);
268 + hdrlen = 0; /* not really */
269 + reused++;
270 }
108 - datalen = sha1write_compressed(f, buf, size);
109 - free(buf);
271 + written++;
272 return hdrlen + datalen;
273 }
274
@@ -148,8 +310,6 @@ static void write_pack_file(void)
310 offset = write_one(f, objects + i, offset);
311
312 sha1close(f, pack_file_sha1, 1);
151 - mb = offset >> 20;
152 - offset &= 0xfffff;
313 }
314
315 static void write_index_file(void)
@@ -196,18 +356,21 @@ static int add_object_entry(unsigned char *sha1, unsigned int hash)
356 {
357 unsigned int idx = nr_objects;
358 struct object_entry *entry;
199 -
200 - if (incremental || local) {
201 - struct packed_git *p;
202 -
203 - for (p = packed_git; p; p = p->next) {
204 - struct pack_entry e;
205 -
206 - if (find_pack_entry_one(sha1, &e, p)) {
207 - if (incremental)
208 - return 0;
209 - if (local && !p->pack_local)
210 - return 0;
359 + struct packed_git *p;
360 + unsigned int found_offset;
361 + struct packed_git *found_pack;
362 +
363 + found_pack = NULL;
364 + for (p = packed_git; p; p = p->next) {
365 + struct pack_entry e;
366 + if (find_pack_entry_one(sha1, &e, p)) {
367 + if (incremental)
368 + return 0;
369 + if (local && !p->pack_local)
370 + return 0;
371 + if (!found_pack) {
372 + found_offset = e.offset;
373 + found_pack = e.p;
374 }
375 }
376 }
@@ -221,30 +384,107 @@ static int add_object_entry(unsigned char *sha1, unsigned int hash)
384 memset(entry, 0, sizeof(*entry));
385 memcpy(entry->sha1, sha1, 20);
386 entry->hash = hash;
387 + if (found_pack) {
388 + entry->in_pack = found_pack;
389 + entry->in_pack_offset = found_offset;
390 + }
391 nr_objects = idx+1;
392 return 1;
393 }
394
395 +static int locate_object_entry_hash(unsigned char *sha1)
396 +{
397 + int i;
398 + unsigned int ui;
399 + memcpy(&ui, sha1, sizeof(unsigned int));
400 + i = ui % object_ix_hashsz;
401 + while (0 < object_ix[i]) {
402 + if (!memcmp(sha1, objects[object_ix[i]-1].sha1, 20))
403 + return i;
404 + if (++i == object_ix_hashsz)
405 + i = 0;
406 + }
407 + return -1 - i;
408 +}
409 +
410 +static struct object_entry *locate_object_entry(unsigned char *sha1)
411 +{
412 + int i = locate_object_entry_hash(sha1);
413 + if (0 <= i)
414 + return &objects[object_ix[i]-1];
415 + return NULL;
416 +}
417 +
418 static void check_object(struct object_entry *entry)
419 {
420 char type[20];
421
232 - if (!sha1_object_info(entry->sha1, type, &entry->size)) {
233 - if (!strcmp(type, "commit")) {
234 - entry->type = OBJ_COMMIT;
235 - } else if (!strcmp(type, "tree")) {
236 - entry->type = OBJ_TREE;
237 - } else if (!strcmp(type, "blob")) {
238 - entry->type = OBJ_BLOB;
239 - } else if (!strcmp(type, "tag")) {
240 - entry->type = OBJ_TAG;
241 - } else
242 - die("unable to pack object %s of type %s",
243 - sha1_to_hex(entry->sha1), type);
422 + if (entry->in_pack) {
423 + /* Check if it is delta, and the base is also an object
424 + * we are going to pack. If so we will reuse the existing
425 + * delta.
426 + */
427 + unsigned char base[20];
428 + unsigned long size;
429 + struct object_entry *base_entry;
430 + if (!check_reuse_pack_delta(entry->in_pack,
431 + entry->in_pack_offset,
432 + base, &size,
433 + &entry->in_pack_type) &&
434 + (base_entry = locate_object_entry(base))) {
435 + /* We do not know depth at this point, but it
436 + * does not matter. Getting delta_chain_length
437 + * with packed_object_info_detail() is not so
438 + * expensive, so we could do that later if we
439 + * wanted to. Calling sha1_object_info to get
440 + * the true size (and later an uncompressed
441 + * representation) of deeply deltified object
442 + * is quite expensive.
443 + */
444 + entry->depth = 1;
445 + /* uncompressed size */
446 + entry->size = entry->delta_size = size;
447 + entry->delta = base_entry;
448 + entry->type = OBJ_DELTA;
449 + return;
450 + }
451 + /* Otherwise we would do the usual */
452 }
245 - else
453 +
454 + if (sha1_object_info(entry->sha1, type, &entry->size))
455 die("unable to get type of object %s",
456 sha1_to_hex(entry->sha1));
457 +
458 + if (!strcmp(type, "commit")) {
459 + entry->type = OBJ_COMMIT;
460 + } else if (!strcmp(type, "tree")) {
461 + entry->type = OBJ_TREE;
462 + } else if (!strcmp(type, "blob")) {
463 + entry->type = OBJ_BLOB;
464 + } else if (!strcmp(type, "tag")) {
465 + entry->type = OBJ_TAG;
466 + } else
467 + die("unable to pack object %s of type %s",
468 + sha1_to_hex(entry->sha1), type);
469 +}
470 +
471 +static void hash_objects(void)
472 +{
473 + int i;
474 + struct object_entry *oe;
475 +
476 + object_ix_hashsz = nr_objects * 2;
477 + object_ix = xcalloc(sizeof(int), object_ix_hashsz);
478 + for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
479 + int ix = locate_object_entry_hash(oe->sha1);
480 + if (0 <= ix) {
481 + error("the same object '%s' added twice",
482 + sha1_to_hex(oe->sha1));
483 + continue;
484 + }
485 + ix = -1 - ix;
486 + object_ix[ix] = i + 1;
487 + }
488 }
489
490 static void get_object_details(void)
@@ -252,6 +492,8 @@ static void get_object_details(void)
492 int i;
493 struct object_entry *entry = objects;
494
495 + hash_objects();
496 + prepare_pack_ix();
497 for (i = 0; i < nr_objects; i++)
498 check_object(entry++);
499 }
@@ -382,6 +624,13 @@ static void find_deltas(struct object_entry **list, int window, int depth)
624 eye_candy -= nr_objects / 20;
625 fputc('.', stderr);
626 }
627 +
628 + if (entry->delta)
629 + /* This happens if we decided to reuse existing
630 + * delta from a pack.
631 + */
632 + continue;
633 +
634 free(n->data);
635 n->entry = entry;
636 n->data = read_sha1_file(entry->sha1, type, &size);
@@ -411,10 +660,12 @@ static void find_deltas(struct object_entry **list, int window, int depth)
660
661 static void prepare_pack(int window, int depth)
662 {
414 - get_object_details();
415 -
663 if (progress)
664 fprintf(stderr, "Packing %d objects", nr_objects);
665 + get_object_details();
666 + if (progress)
667 + fprintf(stderr, ".");
668 +
669 sorted_by_type = create_sorted_list(type_size_sort);
670 if (window && depth)
671 find_deltas(sorted_by_type, window+1, depth);
@@ -599,5 +850,7 @@ int main(int argc, char **argv)
850 puts(sha1_to_hex(object_list_sha1));
851 }
852 }
853 + fprintf(stderr, "Total %d, written %d, reused %d\n",
854 + nr_objects, written, reused);
855 return 0;
856 }
pack.h
+3 -1
@@ -29,5 +29,7 @@ struct pack_header {
29 };
30
31 extern int verify_pack(struct packed_git *, int);
32 -
32 +extern int check_reuse_pack_delta(struct packed_git *, unsigned long,
33 + unsigned char *, unsigned long *,
34 + enum object_type *);
35 #endif
sha1_file.c
+19
@@ -826,6 +826,25 @@ static unsigned long unpack_object_header(struct packed_git *p, unsigned long of
826 return offset;
827 }
828
829 +int check_reuse_pack_delta(struct packed_git *p, unsigned long offset,
830 + unsigned char *base, unsigned long *sizep,
831 + enum object_type *kindp)
832 +{
833 + unsigned long ptr;
834 + int status = -1;
835 +
836 + use_packed_git(p);
837 + ptr = offset;
838 + ptr = unpack_object_header(p, ptr, kindp, sizep);
839 + if (*kindp != OBJ_DELTA)
840 + goto done;
841 + memcpy(base, p->pack_base + ptr, 20);
842 + status = 0;
843 + done:
844 + unuse_packed_git(p);
845 + return status;
846 +}
847 +
848 void packed_object_info_detail(struct pack_entry *e,
849 char *type,
850 unsigned long *size,