Refactored fast-import's internals for future additions.
Too many globals variables were being used not not enough code was resuable to process trees and commits so this is a simple refactoring of the existing blob processing code to get into a state that will be easier to handle trees and commits in. Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Shawn O. Pearce committed
Aug 8, 2006 at 00:46 UTC
ac47a738a7c866eeffc0c6374c0ef3f7ca6ee79d
1 file changed
+83
-66
fast-import.c
+83
-66
@@ -18,22 +18,34 @@ struct object_entry_block
18
struct object_entry_block *next_block;
19
struct object_entry *next_free;
20
struct object_entry *end;
21
- struct object_entry entries[0];
21
+ struct object_entry entries[FLEX_ARRAY]; /* more */
22
};
23
24
+struct last_object
25
+{
26
+ void *data;
27
+ unsigned long len;
28
+ int depth;
29
+ unsigned char sha1[20];
30
+};
31
+
32
+/* Stats and misc. counters. */
33
static int max_depth = 10;
34
static unsigned long alloc_count;
35
static unsigned long object_count;
36
static unsigned long duplicate_count;
28
-static unsigned long packoff;
29
-static int packfd;
30
-static int current_depth;
31
-static void *lastdat;
32
-static unsigned long lastdatlen;
33
-static unsigned char lastsha1[20];
34
-static unsigned char packsha1[20];
35
-struct object_entry *object_table[1 << 16];
37
+
38
+/* The .pack file */
39
+static int pack_fd;
40
+static unsigned long pack_offset;
41
+static unsigned char pack_sha1[20];
42
+
43
+/* Table of objects we've written. */
44
struct object_entry_block *blocks;
45
+struct object_entry *object_table[1 << 16];
46
+
47
+/* Our last blob */
48
+struct last_object last_blob;
49
50
static void alloc_objects(int cnt)
51
{
@@ -115,7 +127,10 @@ static ssize_t ywrite(int fd, void *buffer, size_t length)
127
return ret;
128
}
129
118
-static unsigned long encode_header(enum object_type type, unsigned long size, unsigned char *hdr)
130
+static unsigned long encode_header(
131
+ enum object_type type,
132
+ unsigned long size,
133
+ unsigned char *hdr)
134
{
135
int n = 1;
136
unsigned char c;
@@ -135,41 +150,62 @@ static unsigned long encode_header(enum object_type type, unsigned long size, un
150
return n;
151
}
152
138
-static void write_blob(void *dat, unsigned long datlen)
153
+static int store_object(
154
+ enum object_type type,
155
+ void *dat,
156
+ unsigned long datlen,
157
+ struct last_object *last)
158
{
140
- z_stream s;
159
void *out, *delta;
142
- unsigned char hdr[64];
160
+ struct object_entry *e;
161
+ unsigned char hdr[96];
162
+ unsigned char sha1[20];
163
unsigned long hdrlen, deltalen;
164
+ SHA_CTX c;
165
+ z_stream s;
166
+
167
+ hdrlen = sprintf((char*)hdr,"%s %lu",type_names[type],datlen) + 1;
168
+ SHA1_Init(&c);
169
+ SHA1_Update(&c, hdr, hdrlen);
170
+ SHA1_Update(&c, dat, datlen);
171
+ SHA1_Final(sha1, &c);
172
+
173
+ e = insert_object(sha1);
174
+ if (e->offset) {
175
+ duplicate_count++;
176
+ return 0;
177
+ }
178
+ e->offset = pack_offset;
179
+ object_count++;
180
145
- if (lastdat && current_depth < max_depth) {
146
- delta = diff_delta(lastdat, lastdatlen,
181
+ if (last->data && last->depth < max_depth)
182
+ delta = diff_delta(last->data, last->len,
183
dat, datlen,
184
&deltalen, 0);
149
- } else
185
+ else
186
delta = 0;
187
188
memset(&s, 0, sizeof(s));
189
deflateInit(&s, zlib_compression_level);
190
191
if (delta) {
156
- current_depth++;
192
+ last->depth++;
193
s.next_in = delta;
194
s.avail_in = deltalen;
195
hdrlen = encode_header(OBJ_DELTA, deltalen, hdr);
160
- if (ywrite(packfd, hdr, hdrlen) != hdrlen)
196
+ if (ywrite(pack_fd, hdr, hdrlen) != hdrlen)
197
die("Can't write object header: %s", strerror(errno));
162
- if (ywrite(packfd, lastsha1, sizeof(lastsha1)) != sizeof(lastsha1))
198
+ if (ywrite(pack_fd, last->sha1, sizeof(sha1)) != sizeof(sha1))
199
die("Can't write object base: %s", strerror(errno));
164
- packoff += hdrlen + sizeof(lastsha1);
200
+ pack_offset += hdrlen + sizeof(sha1);
201
} else {
166
- current_depth = 0;
202
+ last->depth = 0;
203
s.next_in = dat;
204
s.avail_in = datlen;
169
- hdrlen = encode_header(OBJ_BLOB, datlen, hdr);
170
- if (ywrite(packfd, hdr, hdrlen) != hdrlen)
205
+ hdrlen = encode_header(type, datlen, hdr);
206
+ if (ywrite(pack_fd, hdr, hdrlen) != hdrlen)
207
die("Can't write object header: %s", strerror(errno));
172
- packoff += hdrlen;
208
+ pack_offset += hdrlen;
209
}
210
211
s.avail_out = deflateBound(&s, s.avail_in);
@@ -178,13 +214,19 @@ static void write_blob(void *dat, unsigned long datlen)
214
/* nothing */;
215
deflateEnd(&s);
216
181
- if (ywrite(packfd, out, s.total_out) != s.total_out)
217
+ if (ywrite(pack_fd, out, s.total_out) != s.total_out)
218
die("Failed writing compressed data %s", strerror(errno));
183
- packoff += s.total_out;
219
+ pack_offset += s.total_out;
220
221
free(out);
222
if (delta)
223
free(delta);
224
+ if (last->data)
225
+ free(last->data);
226
+ last->data = dat;
227
+ last->len = datlen;
228
+ memcpy(last->sha1, sha1, sizeof(sha1));
229
+ return 1;
230
}
231
232
static void init_pack_header()
@@ -195,13 +237,13 @@ static void init_pack_header()
237
238
version = htonl(version);
239
198
- if (ywrite(packfd, (char*)magic, 4) != 4)
240
+ if (ywrite(pack_fd, (char*)magic, 4) != 4)
241
die("Can't write pack magic: %s", strerror(errno));
200
- if (ywrite(packfd, &version, 4) != 4)
242
+ if (ywrite(pack_fd, &version, 4) != 4)
243
die("Can't write pack version: %s", strerror(errno));
202
- if (ywrite(packfd, &zero, 4) != 4)
244
+ if (ywrite(pack_fd, &zero, 4) != 4)
245
die("Can't write 0 object count: %s", strerror(errno));
204
- packoff = 4 * 3;
246
+ pack_offset = 4 * 3;
247
}
248
249
static void fixup_header_footer()
@@ -212,30 +254,30 @@ static void fixup_header_footer()
254
char *buf;
255
size_t n;
256
215
- if (lseek(packfd, 0, SEEK_SET) != 0)
257
+ if (lseek(pack_fd, 0, SEEK_SET) != 0)
258
die("Failed seeking to start: %s", strerror(errno));
259
260
SHA1_Init(&c);
219
- if (yread(packfd, hdr, 8) != 8)
261
+ if (yread(pack_fd, hdr, 8) != 8)
262
die("Failed reading header: %s", strerror(errno));
263
SHA1_Update(&c, hdr, 8);
264
265
cnt = htonl(object_count);
266
SHA1_Update(&c, &cnt, 4);
225
- if (ywrite(packfd, &cnt, 4) != 4)
267
+ if (ywrite(pack_fd, &cnt, 4) != 4)
268
die("Failed writing object count: %s", strerror(errno));
269
270
buf = xmalloc(128 * 1024);
271
for (;;) {
230
- n = xread(packfd, buf, 128 * 1024);
272
+ n = xread(pack_fd, buf, 128 * 1024);
273
if (n <= 0)
274
break;
275
SHA1_Update(&c, buf, n);
276
}
277
free(buf);
278
237
- SHA1_Final(packsha1, &c);
238
- if (ywrite(packfd, packsha1, sizeof(packsha1)) != sizeof(packsha1))
279
+ SHA1_Final(pack_sha1, &c);
280
+ if (ywrite(pack_fd, pack_sha1, sizeof(pack_sha1)) != sizeof(pack_sha1))
281
die("Failed writing pack checksum: %s", strerror(errno));
282
}
283
@@ -284,7 +326,7 @@ static void write_index(const char *idx_name)
326
sha1write(f, &offset, 4);
327
sha1write(f, (*c)->sha1, sizeof((*c)->sha1));
328
}
287
- sha1write(f, packsha1, sizeof(packsha1));
329
+ sha1write(f, pack_sha1, sizeof(pack_sha1));
330
sha1close(f, NULL, 1);
331
free(idx);
332
}
@@ -301,53 +343,28 @@ int main(int argc, const char **argv)
343
idx_name = xmalloc(strlen(base_name) + 5);
344
sprintf(idx_name, "%s.idx", base_name);
345
304
- packfd = open(pack_name, O_RDWR|O_CREAT|O_EXCL, 0666);
305
- if (packfd < 0)
346
+ pack_fd = open(pack_name, O_RDWR|O_CREAT|O_EXCL, 0666);
347
+ if (pack_fd < 0)
348
die("Can't create pack file %s: %s", pack_name, strerror(errno));
349
350
alloc_objects(est_obj_cnt);
351
init_pack_header();
352
for (;;) {
353
unsigned long datlen;
312
- int hdrlen;
354
void *dat;
314
- char hdr[128];
315
- unsigned char sha1[20];
316
- SHA_CTX c;
317
- struct object_entry *e;
355
356
if (yread(0, &datlen, 4) != 4)
320
-
357
break;
358
359
dat = xmalloc(datlen);
360
if (yread(0, dat, datlen) != datlen)
361
break;
362
327
- hdrlen = sprintf(hdr, "blob %lu", datlen) + 1;
328
- SHA1_Init(&c);
329
- SHA1_Update(&c, hdr, hdrlen);
330
- SHA1_Update(&c, dat, datlen);
331
- SHA1_Final(sha1, &c);
332
-
333
- e = insert_object(sha1);
334
- if (!e->offset) {
335
- e->offset = packoff;
336
- write_blob(dat, datlen);
337
- object_count++;
338
-
339
- if (lastdat)
340
- free(lastdat);
341
- lastdat = dat;
342
- lastdatlen = datlen;
343
- memcpy(lastsha1, sha1, sizeof(sha1));
344
- } else {
345
- duplicate_count++;
363
+ if (!store_object(OBJ_BLOB, dat, datlen, &last_blob))
364
free(dat);
347
- }
365
}
366
fixup_header_footer();
350
- close(packfd);
367
+ close(pack_fd);
368
write_index(idx_name);
369
370
fprintf(stderr, "%lu objects, %lu duplicates, %lu allocated (%lu overflow)\n",