@matheusbsbs / matheus-bagliano / commits / 8bcce30126
Fork 0

Added automatic index generation to fast-import.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

Shawn O. Pearce committed Aug 6, 2006 at 13:51 UTC 8bcce30126b90af83c1291e072f74950e73a2584
1 file changed +163 -19
fast-import.c
+163 -19
@@ -8,11 +8,75 @@
8
9 static int max_depth = 10;
10 static unsigned long object_count;
11 +static unsigned long duplicate_count;
12 +static unsigned long packoff;
13 +static unsigned long overflow_count;
14 static int packfd;
15 static int current_depth;
16 static void *lastdat;
17 static unsigned long lastdatlen;
18 static unsigned char lastsha1[20];
19 +static unsigned char packsha1[20];
20 +
21 +struct object_entry
22 +{
23 + struct object_entry *next;
24 + unsigned long offset;
25 + unsigned char sha1[20];
26 +};
27 +
28 +struct overflow_object_entry
29 +{
30 + struct overflow_object_entry *next;
31 + struct object_entry oe;
32 +};
33 +
34 +struct object_entry *pool_start;
35 +struct object_entry *pool_next;
36 +struct object_entry *pool_end;
37 +struct overflow_object_entry *overflow;
38 +struct object_entry *table[1 << 16];
39 +
40 +static struct object_entry* new_object(unsigned char *sha1)
41 +{
42 + if (pool_next != pool_end) {
43 + struct object_entry *e = pool_next++;
44 + memcpy(e->sha1, sha1, sizeof(e->sha1));
45 + return e;
46 + } else {
47 + struct overflow_object_entry *e;
48 +
49 + e = xmalloc(sizeof(struct overflow_object_entry));
50 + e->next = overflow;
51 + memcpy(e->oe.sha1, sha1, sizeof(e->oe.sha1));
52 + overflow = e;
53 + overflow_count++;
54 + return &e->oe;
55 + }
56 +}
57 +
58 +static struct object_entry* insert_object(unsigned char *sha1)
59 +{
60 + unsigned int h = sha1[0] << 8 | sha1[1];
61 + struct object_entry *e = table[h];
62 + struct object_entry *p = 0;
63 +
64 + while (e) {
65 + if (!memcmp(sha1, e->sha1, sizeof(e->sha1)))
66 + return e;
67 + p = e;
68 + e = e->next;
69 + }
70 +
71 + e = new_object(sha1);
72 + e->next = 0;
73 + e->offset = 0;
74 + if (p)
75 + p->next = e;
76 + else
77 + table[h] = e;
78 + return e;
79 +}
80
81 static ssize_t yread(int fd, void *buffer, size_t length)
82 {
@@ -66,7 +130,7 @@ static unsigned long encode_header(enum object_type type, unsigned long size, un
130 return n;
131 }
132
69 -static void write_blob (void *dat, unsigned long datlen)
133 +static void write_blob(void *dat, unsigned long datlen)
134 {
135 z_stream s;
136 void *out, *delta;
@@ -92,6 +156,7 @@ static void write_blob (void *dat, unsigned long datlen)
156 die("Can't write object header: %s", strerror(errno));
157 if (ywrite(packfd, lastsha1, sizeof(lastsha1)) != sizeof(lastsha1))
158 die("Can't write object base: %s", strerror(errno));
159 + packoff += hdrlen + sizeof(lastsha1);
160 } else {
161 current_depth = 0;
162 s.next_in = dat;
@@ -99,6 +164,7 @@ static void write_blob (void *dat, unsigned long datlen)
164 hdrlen = encode_header(OBJ_BLOB, datlen, hdr);
165 if (ywrite(packfd, hdr, hdrlen) != hdrlen)
166 die("Can't write object header: %s", strerror(errno));
167 + packoff += hdrlen;
168 }
169
170 s.avail_out = deflateBound(&s, s.avail_in);
@@ -109,13 +175,14 @@ static void write_blob (void *dat, unsigned long datlen)
175
176 if (ywrite(packfd, out, s.total_out) != s.total_out)
177 die("Failed writing compressed data %s", strerror(errno));
178 + packoff += s.total_out;
179
180 free(out);
181 if (delta)
182 free(delta);
183 }
184
118 -static void init_pack_header ()
185 +static void init_pack_header()
186 {
187 const char* magic = "PACK";
188 unsigned long version = 2;
@@ -129,13 +196,13 @@ static void init_pack_header ()
196 die("Can't write pack version: %s", strerror(errno));
197 if (ywrite(packfd, &zero, 4) != 4)
198 die("Can't write 0 object count: %s", strerror(errno));
199 + packoff = 4 * 3;
200 }
201
134 -static void fixup_header_footer ()
202 +static void fixup_header_footer()
203 {
204 SHA_CTX c;
205 char hdr[8];
138 - unsigned char sha1[20];
206 unsigned long cnt;
207 char *buf;
208 size_t n;
@@ -148,7 +215,6 @@ static void fixup_header_footer ()
215 die("Failed reading header: %s", strerror(errno));
216 SHA1_Update(&c, hdr, 8);
217
151 -fprintf(stderr, "%lu objects\n", object_count);
218 cnt = htonl(object_count);
219 SHA1_Update(&c, &cnt, 4);
220 if (ywrite(packfd, &cnt, 4) != 4)
@@ -163,16 +229,81 @@ fprintf(stderr, "%lu objects\n", object_count);
229 }
230 free(buf);
231
166 - SHA1_Final(sha1, &c);
167 - if (ywrite(packfd, sha1, sizeof(sha1)) != sizeof(sha1))
232 + SHA1_Final(packsha1, &c);
233 + if (ywrite(packfd, packsha1, sizeof(packsha1)) != sizeof(packsha1))
234 die("Failed writing pack checksum: %s", strerror(errno));
235 }
236
171 -int main (int argc, const char **argv)
237 +static int oecmp (const void *_a, const void *_b)
238 {
173 - packfd = open(argv[1], O_RDWR|O_CREAT|O_TRUNC, 0666);
239 + struct object_entry *a = *((struct object_entry**)_a);
240 + struct object_entry *b = *((struct object_entry**)_b);
241 + return memcmp(a->sha1, b->sha1, sizeof(a->sha1));
242 +}
243 +
244 +static void write_index(const char *idx_name)
245 +{
246 + struct sha1file *f;
247 + struct object_entry **idx, **c, **last;
248 + struct object_entry *e;
249 + struct overflow_object_entry *o;
250 + unsigned int array[256];
251 + int i;
252 +
253 + /* Build the sorted table of object IDs. */
254 + idx = xmalloc(object_count * sizeof(struct object_entry*));
255 + c = idx;
256 + for (e = pool_start; e != pool_next; e++)
257 + *c++ = e;
258 + for (o = overflow; o; o = o->next)
259 + *c++ = &o->oe;
260 + last = idx + object_count;
261 + qsort(idx, object_count, sizeof(struct object_entry*), oecmp);
262 +
263 + /* Generate the fan-out array. */
264 + c = idx;
265 + for (i = 0; i < 256; i++) {
266 + struct object_entry **next = c;;
267 + while (next < last) {
268 + if ((*next)->sha1[0] != i)
269 + break;
270 + next++;
271 + }
272 + array[i] = htonl(next - idx);
273 + c = next;
274 + }
275 +
276 + f = sha1create("%s", idx_name);
277 + sha1write(f, array, 256 * sizeof(int));
278 + for (c = idx; c != last; c++) {
279 + unsigned int offset = htonl((*c)->offset);
280 + sha1write(f, &offset, 4);
281 + sha1write(f, (*c)->sha1, sizeof((*c)->sha1));
282 + }
283 + sha1write(f, packsha1, sizeof(packsha1));
284 + sha1close(f, NULL, 1);
285 + free(idx);
286 +}
287 +
288 +int main(int argc, const char **argv)
289 +{
290 + const char *base_name = argv[1];
291 + int est_obj_cnt = atoi(argv[2]);
292 + char *pack_name;
293 + char *idx_name;
294 +
295 + pack_name = xmalloc(strlen(base_name) + 6);
296 + sprintf(pack_name, "%s.pack", base_name);
297 + idx_name = xmalloc(strlen(base_name) + 5);
298 + sprintf(idx_name, "%s.idx", base_name);
299 +
300 + packfd = open(pack_name, O_RDWR|O_CREAT|O_TRUNC, 0666);
301 if (packfd < 0)
175 - die("Can't create pack file %s: %s", argv[1], strerror(errno));
302 + die("Can't create pack file %s: %s", pack_name, strerror(errno));
303 +
304 + pool_start = xmalloc(est_obj_cnt * sizeof(struct object_entry));
305 + pool_next = pool_start;
306 + pool_end = pool_start + est_obj_cnt;
307
308 init_pack_header();
309 for (;;) {
@@ -182,8 +313,10 @@ int main (int argc, const char **argv)
313 char hdr[128];
314 unsigned char sha1[20];
315 SHA_CTX c;
316 + struct object_entry *e;
317
318 if (yread(0, &datlen, 4) != 4)
319 +
320 break;
321
322 dat = xmalloc(datlen);
@@ -196,19 +329,30 @@ int main (int argc, const char **argv)
329 SHA1_Update(&c, dat, datlen);
330 SHA1_Final(sha1, &c);
331
199 - write_blob(dat, datlen);
200 - object_count++;
201 - printf("%s\n", sha1_to_hex(sha1));
202 - fflush(stdout);
332 + e = insert_object(sha1);
333 + if (!e->offset) {
334 + e->offset = packoff;
335 + write_blob(dat, datlen);
336 + object_count++;
337 + printf("%s\n", sha1_to_hex(sha1));
338 + fflush(stdout);
339
204 - if (lastdat)
205 - free(lastdat);
206 - lastdat = dat;
207 - lastdatlen = datlen;
208 - memcpy(lastsha1, sha1, sizeof(sha1));
340 + if (lastdat)
341 + free(lastdat);
342 + lastdat = dat;
343 + lastdatlen = datlen;
344 + memcpy(lastsha1, sha1, sizeof(sha1));
345 + } else {
346 + duplicate_count++;
347 + free(dat);
348 + }
349 }
350 fixup_header_footer();
351 close(packfd);
352 + write_index(idx_name);
353 +
354 + fprintf(stderr, "%lu objects, %lu duplicates, %lu pool overflow\n",
355 + object_count, duplicate_count, overflow_count);
356
357 return 0;
358 }