9
10
struct object_entry {
11
unsigned char sha1[20];
12
- unsigned long size;
13
- unsigned long offset;
14
- unsigned int depth;
15
- unsigned int hash;
12
+ unsigned long size; /* uncompressed size */
13
+ unsigned long offset; /* offset into the final pack file (nonzero if already written) */
14
+ unsigned int depth; /* delta depth */
15
+ unsigned int hash; /* name hint hash */
16
enum object_type type;
17
- unsigned long delta_size;
18
- struct object_entry *delta;
17
+ unsigned long delta_size; /* delta data size (uncompressed) */
18
+ struct object_entry *delta; /* delta base object */
19
+ struct packed_git *in_pack; /* already in pack */
20
+ enum object_type in_pack_type; /* could be delta */
21
+ unsigned int in_pack_offset;
22
};
23
24
+/*
25
+ * Objects we are going to pack are colected in objects array (dynamically
26
+ * expanded). nr_objects & nr_alloc controls this array. They are stored
27
+ * in the order we see -- typically rev-list --objects order that gives us
28
+ * nice "minimum seek" order.
29
+ *
30
+ * sorted-by-sha ans sorted-by-type are arrays of pointers that point at
31
+ * elements in the objects array. The former is used to build the pack
32
+ * index (lists object names in the ascending order to help offset lookup),
33
+ * and the latter is used to group similar things together by try_delta()
34
+ * heuristics.
35
+ */
36
+
37
static unsigned char object_list_sha1[20];
38
static int non_empty = 0;
39
static int local = 0;
45
static unsigned char pack_file_sha1[20];
46
static int progress = 1;
47
48
+/*
49
+ * The object names in objects array are hashed with this hashtable,
50
+ * to help looking up the entry by object name. Binary search from
51
+ * sorted_by_sha is also possible but this was easier to code and faster.
52
+ * This hashtable is built after all the objects are seen.
53
+ */
54
+static int *object_ix = NULL;
55
+static int object_ix_hashsz = 0;
56
+
57
+/*
58
+ * Pack index for existing packs give us easy access to the offsets into
59
+ * corresponding pack file where each object's data starts, but the entries
60
+ * do not store the size of the compressed representation (uncompressed
61
+ * size is easily available by examining the pack entry header). We build
62
+ * a hashtable of existing packs (pack_revindex), and keep reverse index
63
+ * here -- pack index file is sorted by object name mapping to offset; this
64
+ * pack_revindex[].revindex array is an ordered list of offsets, so if you
65
+ * know the offset of an object, next offset is where its packed
66
+ * representation ends.
67
+ */
68
+struct pack_revindex {
69
+ struct packed_git *p;
70
+ unsigned long *revindex;
71
+} *pack_revindex = NULL;
72
+static int pack_revindex_hashsz = 0;
73
+
74
+/*
75
+ * stats
76
+ */
77
+static int written = 0;
78
+static int reused = 0;
79
+
80
+static int pack_revindex_ix(struct packed_git *p)
81
+{
82
+ unsigned int ui = (unsigned int) p;
83
+ int i;
84
+
85
+ ui = ui ^ (ui >> 16); /* defeat structure alignment */
86
+ i = (int)(ui % pack_revindex_hashsz);
87
+ while (pack_revindex[i].p) {
88
+ if (pack_revindex[i].p == p)
89
+ return i;
90
+ if (++i == pack_revindex_hashsz)
91
+ i = 0;
92
+ }
93
+ return -1 - i;
94
+}
95
+
96
+static void prepare_pack_ix(void)
97
+{
98
+ int num;
99
+ struct packed_git *p;
100
+ for (num = 0, p = packed_git; p; p = p->next)
101
+ num++;
102
+ if (!num)
103
+ return;
104
+ pack_revindex_hashsz = num * 11;
105
+ pack_revindex = xcalloc(sizeof(*pack_revindex), pack_revindex_hashsz);
106
+ for (p = packed_git; p; p = p->next) {
107
+ num = pack_revindex_ix(p);
108
+ num = - 1 - num;
109
+ pack_revindex[num].p = p;
110
+ }
111
+ /* revindex elements are lazily initialized */
112
+}
113
+
114
+static int cmp_offset(const void *a_, const void *b_)
115
+{
116
+ unsigned long a = *(unsigned long *) a_;
117
+ unsigned long b = *(unsigned long *) b_;
118
+ if (a < b)
119
+ return -1;
120
+ else if (a == b)
121
+ return 0;
122
+ else
123
+ return 1;
124
+}
125
+
126
+/*
127
+ * Ordered list of offsets of objects in the pack.
128
+ */
129
+static void prepare_pack_revindex(struct pack_revindex *rix)
130
+{
131
+ struct packed_git *p = rix->p;
132
+ int num_ent = num_packed_objects(p);
133
+ int i;
134
+ void *index = p->index_base + 256;
135
+
136
+ rix->revindex = xmalloc(sizeof(unsigned long) * (num_ent + 1));
137
+ for (i = 0; i < num_ent; i++) {
138
+ long hl = *((long *)(index + 24 * i));
139
+ rix->revindex[i] = ntohl(hl);
140
+ }
141
+ /* This knows the pack format -- the 20-byte trailer
142
+ * follows immediately after the last object data.
143
+ */
144
+ rix->revindex[num_ent] = p->pack_size - 20;
145
+ qsort(rix->revindex, num_ent, sizeof(unsigned long), cmp_offset);
146
+}
147
+
148
+static unsigned long find_packed_object_size(struct packed_git *p,
149
+ unsigned long ofs)
150
+{
151
+ int num;
152
+ int lo, hi;
153
+ struct pack_revindex *rix;
154
+ unsigned long *revindex;
155
+ num = pack_revindex_ix(p);
156
+ if (num < 0)
157
+ die("internal error: pack revindex uninitialized");
158
+ rix = &pack_revindex[num];
159
+ if (!rix->revindex)
160
+ prepare_pack_revindex(rix);
161
+ revindex = rix->revindex;
162
+ lo = 0;
163
+ hi = num_packed_objects(p) + 1;
164
+ do {
165
+ int mi = (lo + hi) / 2;
166
+ if (revindex[mi] == ofs) {
167
+ return revindex[mi+1] - ofs;
168
+ }
169
+ else if (ofs < revindex[mi])
170
+ hi = mi;
171
+ else
172
+ lo = mi + 1;
173
+ } while (lo < hi);
174
+ die("internal error: pack revindex corrupt");
175
+}
176
+
177
static void *delta_against(void *buf, unsigned long size, struct object_entry *entry)
178
{
179
unsigned long othersize, delta_size;
223
{
224
unsigned long size;
225
char type[10];
81
- void *buf = read_sha1_file(entry->sha1, type, &size);
226
+ void *buf;
227
unsigned char header[10];
228
unsigned hdrlen, datalen;
229
enum object_type obj_type;
230
86
- if (!buf)
87
- die("unable to read %s", sha1_to_hex(entry->sha1));
88
- if (size != entry->size)
89
- die("object %s size inconsistency (%lu vs %lu)", sha1_to_hex(entry->sha1), size, entry->size);
90
-
91
- /*
92
- * The object header is a byte of 'type' followed by zero or
93
- * more bytes of length. For deltas, the 20 bytes of delta sha1
94
- * follows that.
95
- */
231
obj_type = entry->type;
97
- if (entry->delta) {
98
- buf = delta_against(buf, size, entry);
99
- size = entry->delta_size;
100
- obj_type = OBJ_DELTA;
232
+ if (!entry->in_pack ||
233
+ (obj_type != entry->in_pack_type)) {
234
+ buf = read_sha1_file(entry->sha1, type, &size);
235
+ if (!buf)
236
+ die("unable to read %s", sha1_to_hex(entry->sha1));
237
+ if (size != entry->size)
238
+ die("object %s size inconsistency (%lu vs %lu)",
239
+ sha1_to_hex(entry->sha1), size, entry->size);
240
+ if (entry->delta) {
241
+ buf = delta_against(buf, size, entry);
242
+ size = entry->delta_size;
243
+ obj_type = OBJ_DELTA;
244
+ }
245
+ /*
246
+ * The object header is a byte of 'type' followed by zero or
247
+ * more bytes of length. For deltas, the 20 bytes of delta
248
+ * sha1 follows that.
249
+ */
250
+ hdrlen = encode_header(obj_type, size, header);
251
+ sha1write(f, header, hdrlen);
252
+
253
+ if (entry->delta) {
254
+ sha1write(f, entry->delta, 20);
255
+ hdrlen += 20;
256
+ }
257
+ datalen = sha1write_compressed(f, buf, size);
258
+ free(buf);
259
}
102
- hdrlen = encode_header(obj_type, size, header);
103
- sha1write(f, header, hdrlen);
104
- if (entry->delta) {
105
- sha1write(f, entry->delta, 20);
106
- hdrlen += 20;
260
+ else {
261
+ struct packed_git *p = entry->in_pack;
262
+ use_packed_git(p);
263
+
264
+ datalen = find_packed_object_size(p, entry->in_pack_offset);
265
+ buf = p->pack_base + entry->in_pack_offset;
266
+ sha1write(f, buf, datalen);
267
+ unuse_packed_git(p);
268
+ hdrlen = 0; /* not really */
269
+ reused++;
270
}
108
- datalen = sha1write_compressed(f, buf, size);
109
- free(buf);
271
+ written++;
272
return hdrlen + datalen;
273
}
274
310
offset = write_one(f, objects + i, offset);
311
312
sha1close(f, pack_file_sha1, 1);
151
- mb = offset >> 20;
152
- offset &= 0xfffff;
313
}
314
315
static void write_index_file(void)
356
{
357
unsigned int idx = nr_objects;
358
struct object_entry *entry;
199
-
200
- if (incremental || local) {
201
- struct packed_git *p;
202
-
203
- for (p = packed_git; p; p = p->next) {
204
- struct pack_entry e;
205
-
206
- if (find_pack_entry_one(sha1, &e, p)) {
207
- if (incremental)
208
- return 0;
209
- if (local && !p->pack_local)
210
- return 0;
359
+ struct packed_git *p;
360
+ unsigned int found_offset;
361
+ struct packed_git *found_pack;
362
+
363
+ found_pack = NULL;
364
+ for (p = packed_git; p; p = p->next) {
365
+ struct pack_entry e;
366
+ if (find_pack_entry_one(sha1, &e, p)) {
367
+ if (incremental)
368
+ return 0;
369
+ if (local && !p->pack_local)
370
+ return 0;
371
+ if (!found_pack) {
372
+ found_offset = e.offset;
373
+ found_pack = e.p;
374
}
375
}
376
}
384
memset(entry, 0, sizeof(*entry));
385
memcpy(entry->sha1, sha1, 20);
386
entry->hash = hash;
387
+ if (found_pack) {
388
+ entry->in_pack = found_pack;
389
+ entry->in_pack_offset = found_offset;
390
+ }
391
nr_objects = idx+1;
392
return 1;
393
}
394
395
+static int locate_object_entry_hash(unsigned char *sha1)
396
+{
397
+ int i;
398
+ unsigned int ui;
399
+ memcpy(&ui, sha1, sizeof(unsigned int));
400
+ i = ui % object_ix_hashsz;
401
+ while (0 < object_ix[i]) {
402
+ if (!memcmp(sha1, objects[object_ix[i]-1].sha1, 20))
403
+ return i;
404
+ if (++i == object_ix_hashsz)
405
+ i = 0;
406
+ }
407
+ return -1 - i;
408
+}
409
+
410
+static struct object_entry *locate_object_entry(unsigned char *sha1)
411
+{
412
+ int i = locate_object_entry_hash(sha1);
413
+ if (0 <= i)
414
+ return &objects[object_ix[i]-1];
415
+ return NULL;
416
+}
417
+
418
static void check_object(struct object_entry *entry)
419
{
420
char type[20];
421
232
- if (!sha1_object_info(entry->sha1, type, &entry->size)) {
233
- if (!strcmp(type, "commit")) {
234
- entry->type = OBJ_COMMIT;
235
- } else if (!strcmp(type, "tree")) {
236
- entry->type = OBJ_TREE;
237
- } else if (!strcmp(type, "blob")) {
238
- entry->type = OBJ_BLOB;
239
- } else if (!strcmp(type, "tag")) {
240
- entry->type = OBJ_TAG;
241
- } else
242
- die("unable to pack object %s of type %s",
243
- sha1_to_hex(entry->sha1), type);
422
+ if (entry->in_pack) {
423
+ /* Check if it is delta, and the base is also an object
424
+ * we are going to pack. If so we will reuse the existing
425
+ * delta.
426
+ */
427
+ unsigned char base[20];
428
+ unsigned long size;
429
+ struct object_entry *base_entry;
430
+ if (!check_reuse_pack_delta(entry->in_pack,
431
+ entry->in_pack_offset,
432
+ base, &size,
433
+ &entry->in_pack_type) &&
434
+ (base_entry = locate_object_entry(base))) {
435
+ /* We do not know depth at this point, but it
436
+ * does not matter. Getting delta_chain_length
437
+ * with packed_object_info_detail() is not so
438
+ * expensive, so we could do that later if we
439
+ * wanted to. Calling sha1_object_info to get
440
+ * the true size (and later an uncompressed
441
+ * representation) of deeply deltified object
442
+ * is quite expensive.
443
+ */
444
+ entry->depth = 1;
445
+ /* uncompressed size */
446
+ entry->size = entry->delta_size = size;
447
+ entry->delta = base_entry;
448
+ entry->type = OBJ_DELTA;
449
+ return;
450
+ }
451
+ /* Otherwise we would do the usual */
452
}
245
- else
453
+
454
+ if (sha1_object_info(entry->sha1, type, &entry->size))
455
die("unable to get type of object %s",
456
sha1_to_hex(entry->sha1));
457
+
458
+ if (!strcmp(type, "commit")) {
459
+ entry->type = OBJ_COMMIT;
460
+ } else if (!strcmp(type, "tree")) {
461
+ entry->type = OBJ_TREE;
462
+ } else if (!strcmp(type, "blob")) {
463
+ entry->type = OBJ_BLOB;
464
+ } else if (!strcmp(type, "tag")) {
465
+ entry->type = OBJ_TAG;
466
+ } else
467
+ die("unable to pack object %s of type %s",
468
+ sha1_to_hex(entry->sha1), type);
469
+}
470
+
471
+static void hash_objects(void)
472
+{
473
+ int i;
474
+ struct object_entry *oe;
475
+
476
+ object_ix_hashsz = nr_objects * 2;
477
+ object_ix = xcalloc(sizeof(int), object_ix_hashsz);
478
+ for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
479
+ int ix = locate_object_entry_hash(oe->sha1);
480
+ if (0 <= ix) {
481
+ error("the same object '%s' added twice",
482
+ sha1_to_hex(oe->sha1));
483
+ continue;
484
+ }
485
+ ix = -1 - ix;
486
+ object_ix[ix] = i + 1;
487
+ }
488
}
489
490
static void get_object_details(void)
492
int i;
493
struct object_entry *entry = objects;
494
495
+ hash_objects();
496
+ prepare_pack_ix();
497
for (i = 0; i < nr_objects; i++)
498
check_object(entry++);
499
}
624
eye_candy -= nr_objects / 20;
625
fputc('.', stderr);
626
}
627
+
628
+ if (entry->delta)
629
+ /* This happens if we decided to reuse existing
630
+ * delta from a pack.
631
+ */
632
+ continue;
633
+
634
free(n->data);
635
n->entry = entry;
636
n->data = read_sha1_file(entry->sha1, type, &size);
660
661
static void prepare_pack(int window, int depth)
662
{
414
- get_object_details();
415
-
663
if (progress)
664
fprintf(stderr, "Packing %d objects", nr_objects);
665
+ get_object_details();
666
+ if (progress)
667
+ fprintf(stderr, ".");
668
+
669
sorted_by_type = create_sorted_list(type_size_sort);
670
if (window && depth)
671
find_deltas(sorted_by_type, window+1, depth);
850
puts(sha1_to_hex(object_list_sha1));
851
}
852
}
853
+ fprintf(stderr, "Total %d, written %d, reused %d\n",
854
+ nr_objects, written, reused);
855
return 0;
856
}