@matheusbsbs / matheus-bagliano / commits / ac47a738a7
Fork 0

Refactored fast-import's internals for future additions.

Too many globals variables were being used not not enough code was resuable to process trees and commits so this is a simple refactoring of the existing blob processing code to get into a state that will be easier to handle trees and commits in. Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

Shawn O. Pearce committed Aug 8, 2006 at 00:46 UTC ac47a738a7c866eeffc0c6374c0ef3f7ca6ee79d
1 file changed +83 -66
fast-import.c
+83 -66
@@ -18,22 +18,34 @@ struct object_entry_block
18 struct object_entry_block *next_block;
19 struct object_entry *next_free;
20 struct object_entry *end;
21 - struct object_entry entries[0];
21 + struct object_entry entries[FLEX_ARRAY]; /* more */
22 };
23
24 +struct last_object
25 +{
26 + void *data;
27 + unsigned long len;
28 + int depth;
29 + unsigned char sha1[20];
30 +};
31 +
32 +/* Stats and misc. counters. */
33 static int max_depth = 10;
34 static unsigned long alloc_count;
35 static unsigned long object_count;
36 static unsigned long duplicate_count;
28 -static unsigned long packoff;
29 -static int packfd;
30 -static int current_depth;
31 -static void *lastdat;
32 -static unsigned long lastdatlen;
33 -static unsigned char lastsha1[20];
34 -static unsigned char packsha1[20];
35 -struct object_entry *object_table[1 << 16];
37 +
38 +/* The .pack file */
39 +static int pack_fd;
40 +static unsigned long pack_offset;
41 +static unsigned char pack_sha1[20];
42 +
43 +/* Table of objects we've written. */
44 struct object_entry_block *blocks;
45 +struct object_entry *object_table[1 << 16];
46 +
47 +/* Our last blob */
48 +struct last_object last_blob;
49
50 static void alloc_objects(int cnt)
51 {
@@ -115,7 +127,10 @@ static ssize_t ywrite(int fd, void *buffer, size_t length)
127 return ret;
128 }
129
118 -static unsigned long encode_header(enum object_type type, unsigned long size, unsigned char *hdr)
130 +static unsigned long encode_header(
131 + enum object_type type,
132 + unsigned long size,
133 + unsigned char *hdr)
134 {
135 int n = 1;
136 unsigned char c;
@@ -135,41 +150,62 @@ static unsigned long encode_header(enum object_type type, unsigned long size, un
150 return n;
151 }
152
138 -static void write_blob(void *dat, unsigned long datlen)
153 +static int store_object(
154 + enum object_type type,
155 + void *dat,
156 + unsigned long datlen,
157 + struct last_object *last)
158 {
140 - z_stream s;
159 void *out, *delta;
142 - unsigned char hdr[64];
160 + struct object_entry *e;
161 + unsigned char hdr[96];
162 + unsigned char sha1[20];
163 unsigned long hdrlen, deltalen;
164 + SHA_CTX c;
165 + z_stream s;
166 +
167 + hdrlen = sprintf((char*)hdr,"%s %lu",type_names[type],datlen) + 1;
168 + SHA1_Init(&c);
169 + SHA1_Update(&c, hdr, hdrlen);
170 + SHA1_Update(&c, dat, datlen);
171 + SHA1_Final(sha1, &c);
172 +
173 + e = insert_object(sha1);
174 + if (e->offset) {
175 + duplicate_count++;
176 + return 0;
177 + }
178 + e->offset = pack_offset;
179 + object_count++;
180
145 - if (lastdat && current_depth < max_depth) {
146 - delta = diff_delta(lastdat, lastdatlen,
181 + if (last->data && last->depth < max_depth)
182 + delta = diff_delta(last->data, last->len,
183 dat, datlen,
184 &deltalen, 0);
149 - } else
185 + else
186 delta = 0;
187
188 memset(&s, 0, sizeof(s));
189 deflateInit(&s, zlib_compression_level);
190
191 if (delta) {
156 - current_depth++;
192 + last->depth++;
193 s.next_in = delta;
194 s.avail_in = deltalen;
195 hdrlen = encode_header(OBJ_DELTA, deltalen, hdr);
160 - if (ywrite(packfd, hdr, hdrlen) != hdrlen)
196 + if (ywrite(pack_fd, hdr, hdrlen) != hdrlen)
197 die("Can't write object header: %s", strerror(errno));
162 - if (ywrite(packfd, lastsha1, sizeof(lastsha1)) != sizeof(lastsha1))
198 + if (ywrite(pack_fd, last->sha1, sizeof(sha1)) != sizeof(sha1))
199 die("Can't write object base: %s", strerror(errno));
164 - packoff += hdrlen + sizeof(lastsha1);
200 + pack_offset += hdrlen + sizeof(sha1);
201 } else {
166 - current_depth = 0;
202 + last->depth = 0;
203 s.next_in = dat;
204 s.avail_in = datlen;
169 - hdrlen = encode_header(OBJ_BLOB, datlen, hdr);
170 - if (ywrite(packfd, hdr, hdrlen) != hdrlen)
205 + hdrlen = encode_header(type, datlen, hdr);
206 + if (ywrite(pack_fd, hdr, hdrlen) != hdrlen)
207 die("Can't write object header: %s", strerror(errno));
172 - packoff += hdrlen;
208 + pack_offset += hdrlen;
209 }
210
211 s.avail_out = deflateBound(&s, s.avail_in);
@@ -178,13 +214,19 @@ static void write_blob(void *dat, unsigned long datlen)
214 /* nothing */;
215 deflateEnd(&s);
216
181 - if (ywrite(packfd, out, s.total_out) != s.total_out)
217 + if (ywrite(pack_fd, out, s.total_out) != s.total_out)
218 die("Failed writing compressed data %s", strerror(errno));
183 - packoff += s.total_out;
219 + pack_offset += s.total_out;
220
221 free(out);
222 if (delta)
223 free(delta);
224 + if (last->data)
225 + free(last->data);
226 + last->data = dat;
227 + last->len = datlen;
228 + memcpy(last->sha1, sha1, sizeof(sha1));
229 + return 1;
230 }
231
232 static void init_pack_header()
@@ -195,13 +237,13 @@ static void init_pack_header()
237
238 version = htonl(version);
239
198 - if (ywrite(packfd, (char*)magic, 4) != 4)
240 + if (ywrite(pack_fd, (char*)magic, 4) != 4)
241 die("Can't write pack magic: %s", strerror(errno));
200 - if (ywrite(packfd, &version, 4) != 4)
242 + if (ywrite(pack_fd, &version, 4) != 4)
243 die("Can't write pack version: %s", strerror(errno));
202 - if (ywrite(packfd, &zero, 4) != 4)
244 + if (ywrite(pack_fd, &zero, 4) != 4)
245 die("Can't write 0 object count: %s", strerror(errno));
204 - packoff = 4 * 3;
246 + pack_offset = 4 * 3;
247 }
248
249 static void fixup_header_footer()
@@ -212,30 +254,30 @@ static void fixup_header_footer()
254 char *buf;
255 size_t n;
256
215 - if (lseek(packfd, 0, SEEK_SET) != 0)
257 + if (lseek(pack_fd, 0, SEEK_SET) != 0)
258 die("Failed seeking to start: %s", strerror(errno));
259
260 SHA1_Init(&c);
219 - if (yread(packfd, hdr, 8) != 8)
261 + if (yread(pack_fd, hdr, 8) != 8)
262 die("Failed reading header: %s", strerror(errno));
263 SHA1_Update(&c, hdr, 8);
264
265 cnt = htonl(object_count);
266 SHA1_Update(&c, &cnt, 4);
225 - if (ywrite(packfd, &cnt, 4) != 4)
267 + if (ywrite(pack_fd, &cnt, 4) != 4)
268 die("Failed writing object count: %s", strerror(errno));
269
270 buf = xmalloc(128 * 1024);
271 for (;;) {
230 - n = xread(packfd, buf, 128 * 1024);
272 + n = xread(pack_fd, buf, 128 * 1024);
273 if (n <= 0)
274 break;
275 SHA1_Update(&c, buf, n);
276 }
277 free(buf);
278
237 - SHA1_Final(packsha1, &c);
238 - if (ywrite(packfd, packsha1, sizeof(packsha1)) != sizeof(packsha1))
279 + SHA1_Final(pack_sha1, &c);
280 + if (ywrite(pack_fd, pack_sha1, sizeof(pack_sha1)) != sizeof(pack_sha1))
281 die("Failed writing pack checksum: %s", strerror(errno));
282 }
283
@@ -284,7 +326,7 @@ static void write_index(const char *idx_name)
326 sha1write(f, &offset, 4);
327 sha1write(f, (*c)->sha1, sizeof((*c)->sha1));
328 }
287 - sha1write(f, packsha1, sizeof(packsha1));
329 + sha1write(f, pack_sha1, sizeof(pack_sha1));
330 sha1close(f, NULL, 1);
331 free(idx);
332 }
@@ -301,53 +343,28 @@ int main(int argc, const char **argv)
343 idx_name = xmalloc(strlen(base_name) + 5);
344 sprintf(idx_name, "%s.idx", base_name);
345
304 - packfd = open(pack_name, O_RDWR|O_CREAT|O_EXCL, 0666);
305 - if (packfd < 0)
346 + pack_fd = open(pack_name, O_RDWR|O_CREAT|O_EXCL, 0666);
347 + if (pack_fd < 0)
348 die("Can't create pack file %s: %s", pack_name, strerror(errno));
349
350 alloc_objects(est_obj_cnt);
351 init_pack_header();
352 for (;;) {
353 unsigned long datlen;
312 - int hdrlen;
354 void *dat;
314 - char hdr[128];
315 - unsigned char sha1[20];
316 - SHA_CTX c;
317 - struct object_entry *e;
355
356 if (yread(0, &datlen, 4) != 4)
320 -
357 break;
358
359 dat = xmalloc(datlen);
360 if (yread(0, dat, datlen) != datlen)
361 break;
362
327 - hdrlen = sprintf(hdr, "blob %lu", datlen) + 1;
328 - SHA1_Init(&c);
329 - SHA1_Update(&c, hdr, hdrlen);
330 - SHA1_Update(&c, dat, datlen);
331 - SHA1_Final(sha1, &c);
332 -
333 - e = insert_object(sha1);
334 - if (!e->offset) {
335 - e->offset = packoff;
336 - write_blob(dat, datlen);
337 - object_count++;
338 -
339 - if (lastdat)
340 - free(lastdat);
341 - lastdat = dat;
342 - lastdatlen = datlen;
343 - memcpy(lastsha1, sha1, sizeof(sha1));
344 - } else {
345 - duplicate_count++;
363 + if (!store_object(OBJ_BLOB, dat, datlen, &last_blob))
364 free(dat);
347 - }
365 }
366 fixup_header_footer();
350 - close(packfd);
367 + close(pack_fd);
368 write_index(idx_name);
369
370 fprintf(stderr, "%lu objects, %lu duplicates, %lu allocated (%lu overflow)\n",