Added automatic index generation to fast-import.
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Shawn O. Pearce committed
Aug 6, 2006 at 13:51 UTC
8bcce30126b90af83c1291e072f74950e73a2584
1 file changed
+163
-19
fast-import.c
+163
-19
@@ -8,11 +8,75 @@
8
9
static int max_depth = 10;
10
static unsigned long object_count;
11
+static unsigned long duplicate_count;
12
+static unsigned long packoff;
13
+static unsigned long overflow_count;
14
static int packfd;
15
static int current_depth;
16
static void *lastdat;
17
static unsigned long lastdatlen;
18
static unsigned char lastsha1[20];
19
+static unsigned char packsha1[20];
20
+
21
+struct object_entry
22
+{
23
+ struct object_entry *next;
24
+ unsigned long offset;
25
+ unsigned char sha1[20];
26
+};
27
+
28
+struct overflow_object_entry
29
+{
30
+ struct overflow_object_entry *next;
31
+ struct object_entry oe;
32
+};
33
+
34
+struct object_entry *pool_start;
35
+struct object_entry *pool_next;
36
+struct object_entry *pool_end;
37
+struct overflow_object_entry *overflow;
38
+struct object_entry *table[1 << 16];
39
+
40
+static struct object_entry* new_object(unsigned char *sha1)
41
+{
42
+ if (pool_next != pool_end) {
43
+ struct object_entry *e = pool_next++;
44
+ memcpy(e->sha1, sha1, sizeof(e->sha1));
45
+ return e;
46
+ } else {
47
+ struct overflow_object_entry *e;
48
+
49
+ e = xmalloc(sizeof(struct overflow_object_entry));
50
+ e->next = overflow;
51
+ memcpy(e->oe.sha1, sha1, sizeof(e->oe.sha1));
52
+ overflow = e;
53
+ overflow_count++;
54
+ return &e->oe;
55
+ }
56
+}
57
+
58
+static struct object_entry* insert_object(unsigned char *sha1)
59
+{
60
+ unsigned int h = sha1[0] << 8 | sha1[1];
61
+ struct object_entry *e = table[h];
62
+ struct object_entry *p = 0;
63
+
64
+ while (e) {
65
+ if (!memcmp(sha1, e->sha1, sizeof(e->sha1)))
66
+ return e;
67
+ p = e;
68
+ e = e->next;
69
+ }
70
+
71
+ e = new_object(sha1);
72
+ e->next = 0;
73
+ e->offset = 0;
74
+ if (p)
75
+ p->next = e;
76
+ else
77
+ table[h] = e;
78
+ return e;
79
+}
80
81
static ssize_t yread(int fd, void *buffer, size_t length)
82
{
@@ -66,7 +130,7 @@ static unsigned long encode_header(enum object_type type, unsigned long size, un
130
return n;
131
}
132
69
-static void write_blob (void *dat, unsigned long datlen)
133
+static void write_blob(void *dat, unsigned long datlen)
134
{
135
z_stream s;
136
void *out, *delta;
@@ -92,6 +156,7 @@ static void write_blob (void *dat, unsigned long datlen)
156
die("Can't write object header: %s", strerror(errno));
157
if (ywrite(packfd, lastsha1, sizeof(lastsha1)) != sizeof(lastsha1))
158
die("Can't write object base: %s", strerror(errno));
159
+ packoff += hdrlen + sizeof(lastsha1);
160
} else {
161
current_depth = 0;
162
s.next_in = dat;
@@ -99,6 +164,7 @@ static void write_blob (void *dat, unsigned long datlen)
164
hdrlen = encode_header(OBJ_BLOB, datlen, hdr);
165
if (ywrite(packfd, hdr, hdrlen) != hdrlen)
166
die("Can't write object header: %s", strerror(errno));
167
+ packoff += hdrlen;
168
}
169
170
s.avail_out = deflateBound(&s, s.avail_in);
@@ -109,13 +175,14 @@ static void write_blob (void *dat, unsigned long datlen)
175
176
if (ywrite(packfd, out, s.total_out) != s.total_out)
177
die("Failed writing compressed data %s", strerror(errno));
178
+ packoff += s.total_out;
179
180
free(out);
181
if (delta)
182
free(delta);
183
}
184
118
-static void init_pack_header ()
185
+static void init_pack_header()
186
{
187
const char* magic = "PACK";
188
unsigned long version = 2;
@@ -129,13 +196,13 @@ static void init_pack_header ()
196
die("Can't write pack version: %s", strerror(errno));
197
if (ywrite(packfd, &zero, 4) != 4)
198
die("Can't write 0 object count: %s", strerror(errno));
199
+ packoff = 4 * 3;
200
}
201
134
-static void fixup_header_footer ()
202
+static void fixup_header_footer()
203
{
204
SHA_CTX c;
205
char hdr[8];
138
- unsigned char sha1[20];
206
unsigned long cnt;
207
char *buf;
208
size_t n;
@@ -148,7 +215,6 @@ static void fixup_header_footer ()
215
die("Failed reading header: %s", strerror(errno));
216
SHA1_Update(&c, hdr, 8);
217
151
-fprintf(stderr, "%lu objects\n", object_count);
218
cnt = htonl(object_count);
219
SHA1_Update(&c, &cnt, 4);
220
if (ywrite(packfd, &cnt, 4) != 4)
@@ -163,16 +229,81 @@ fprintf(stderr, "%lu objects\n", object_count);
229
}
230
free(buf);
231
166
- SHA1_Final(sha1, &c);
167
- if (ywrite(packfd, sha1, sizeof(sha1)) != sizeof(sha1))
232
+ SHA1_Final(packsha1, &c);
233
+ if (ywrite(packfd, packsha1, sizeof(packsha1)) != sizeof(packsha1))
234
die("Failed writing pack checksum: %s", strerror(errno));
235
}
236
171
-int main (int argc, const char **argv)
237
+static int oecmp (const void *_a, const void *_b)
238
{
173
- packfd = open(argv[1], O_RDWR|O_CREAT|O_TRUNC, 0666);
239
+ struct object_entry *a = *((struct object_entry**)_a);
240
+ struct object_entry *b = *((struct object_entry**)_b);
241
+ return memcmp(a->sha1, b->sha1, sizeof(a->sha1));
242
+}
243
+
244
+static void write_index(const char *idx_name)
245
+{
246
+ struct sha1file *f;
247
+ struct object_entry **idx, **c, **last;
248
+ struct object_entry *e;
249
+ struct overflow_object_entry *o;
250
+ unsigned int array[256];
251
+ int i;
252
+
253
+ /* Build the sorted table of object IDs. */
254
+ idx = xmalloc(object_count * sizeof(struct object_entry*));
255
+ c = idx;
256
+ for (e = pool_start; e != pool_next; e++)
257
+ *c++ = e;
258
+ for (o = overflow; o; o = o->next)
259
+ *c++ = &o->oe;
260
+ last = idx + object_count;
261
+ qsort(idx, object_count, sizeof(struct object_entry*), oecmp);
262
+
263
+ /* Generate the fan-out array. */
264
+ c = idx;
265
+ for (i = 0; i < 256; i++) {
266
+ struct object_entry **next = c;;
267
+ while (next < last) {
268
+ if ((*next)->sha1[0] != i)
269
+ break;
270
+ next++;
271
+ }
272
+ array[i] = htonl(next - idx);
273
+ c = next;
274
+ }
275
+
276
+ f = sha1create("%s", idx_name);
277
+ sha1write(f, array, 256 * sizeof(int));
278
+ for (c = idx; c != last; c++) {
279
+ unsigned int offset = htonl((*c)->offset);
280
+ sha1write(f, &offset, 4);
281
+ sha1write(f, (*c)->sha1, sizeof((*c)->sha1));
282
+ }
283
+ sha1write(f, packsha1, sizeof(packsha1));
284
+ sha1close(f, NULL, 1);
285
+ free(idx);
286
+}
287
+
288
+int main(int argc, const char **argv)
289
+{
290
+ const char *base_name = argv[1];
291
+ int est_obj_cnt = atoi(argv[2]);
292
+ char *pack_name;
293
+ char *idx_name;
294
+
295
+ pack_name = xmalloc(strlen(base_name) + 6);
296
+ sprintf(pack_name, "%s.pack", base_name);
297
+ idx_name = xmalloc(strlen(base_name) + 5);
298
+ sprintf(idx_name, "%s.idx", base_name);
299
+
300
+ packfd = open(pack_name, O_RDWR|O_CREAT|O_TRUNC, 0666);
301
if (packfd < 0)
175
- die("Can't create pack file %s: %s", argv[1], strerror(errno));
302
+ die("Can't create pack file %s: %s", pack_name, strerror(errno));
303
+
304
+ pool_start = xmalloc(est_obj_cnt * sizeof(struct object_entry));
305
+ pool_next = pool_start;
306
+ pool_end = pool_start + est_obj_cnt;
307
308
init_pack_header();
309
for (;;) {
@@ -182,8 +313,10 @@ int main (int argc, const char **argv)
313
char hdr[128];
314
unsigned char sha1[20];
315
SHA_CTX c;
316
+ struct object_entry *e;
317
318
if (yread(0, &datlen, 4) != 4)
319
+
320
break;
321
322
dat = xmalloc(datlen);
@@ -196,19 +329,30 @@ int main (int argc, const char **argv)
329
SHA1_Update(&c, dat, datlen);
330
SHA1_Final(sha1, &c);
331
199
- write_blob(dat, datlen);
200
- object_count++;
201
- printf("%s\n", sha1_to_hex(sha1));
202
- fflush(stdout);
332
+ e = insert_object(sha1);
333
+ if (!e->offset) {
334
+ e->offset = packoff;
335
+ write_blob(dat, datlen);
336
+ object_count++;
337
+ printf("%s\n", sha1_to_hex(sha1));
338
+ fflush(stdout);
339
204
- if (lastdat)
205
- free(lastdat);
206
- lastdat = dat;
207
- lastdatlen = datlen;
208
- memcpy(lastsha1, sha1, sizeof(sha1));
340
+ if (lastdat)
341
+ free(lastdat);
342
+ lastdat = dat;
343
+ lastdatlen = datlen;
344
+ memcpy(lastsha1, sha1, sizeof(sha1));
345
+ } else {
346
+ duplicate_count++;
347
+ free(dat);
348
+ }
349
}
350
fixup_header_footer();
351
close(packfd);
352
+ write_index(idx_name);
353
+
354
+ fprintf(stderr, "%lu objects, %lu duplicates, %lu pool overflow\n",
355
+ object_count, duplicate_count, overflow_count);
356
357
return 0;
358
}