git-repack --max-pack-size: write_{object,one}() respect pack limit
With --max-pack-size, generate the appropriate write limit for each object and check against it before each group of writes. Update delta usability rules to handle base being in a previously- written pack. Inline sha1write_compress() so we know the exact size of the written data when it needs to be compressed. Detect and return write "failure". Signed-off-by: Dana L. How <danahow@gmail.com> Signed-off-by: Junio C Hamano <junkio@cox.net>
Dana L. How committed
May 13, 2007 at 12:06 UTC
17b08f2cd0000d42bb9f0a36354ab9300d64d420
1 file changed
+104
-27
builtin-pack-objects.c
+104
-27
@@ -352,16 +352,31 @@ static void copy_pack_data(struct sha1file *f,
352
}
353
354
static unsigned long write_object(struct sha1file *f,
355
- struct object_entry *entry)
355
+ struct object_entry *entry,
356
+ off_t write_offset)
357
{
358
unsigned long size;
359
enum object_type type;
360
void *buf;
361
unsigned char header[10];
362
+ unsigned char dheader[10];
363
unsigned hdrlen;
364
off_t datalen;
365
enum object_type obj_type;
366
int to_reuse = 0;
367
+ /* write limit if limited packsize and not first object */
368
+ unsigned long limit = pack_size_limit && nr_written ?
369
+ pack_size_limit - write_offset : 0;
370
+ /* no if no delta */
371
+ int usable_delta = !entry->delta ? 0 :
372
+ /* yes if unlimited packfile */
373
+ !pack_size_limit ? 1 :
374
+ /* no if base written to previous pack */
375
+ entry->delta->offset == (off_t)-1 ? 0 :
376
+ /* otherwise double-check written to this
377
+ * pack, like we do below
378
+ */
379
+ entry->delta->offset ? 1 : 0;
380
381
if (!pack_to_stdout)
382
crc32_begin(f);
@@ -372,7 +387,9 @@ static unsigned long write_object(struct sha1file *f,
387
else if (!entry->in_pack)
388
to_reuse = 0; /* can't reuse what we don't have */
389
else if (obj_type == OBJ_REF_DELTA || obj_type == OBJ_OFS_DELTA)
375
- to_reuse = 1; /* check_object() decided it for us */
390
+ /* check_object() decided it for us ... */
391
+ to_reuse = usable_delta;
392
+ /* ... but pack split may override that */
393
else if (obj_type != entry->in_pack_type)
394
to_reuse = 0; /* pack has delta which is unusable */
395
else if (entry->delta)
@@ -383,24 +400,48 @@ static unsigned long write_object(struct sha1file *f,
400
*/
401
402
if (!to_reuse) {
403
+ z_stream stream;
404
+ unsigned long maxsize;
405
+ void *out;
406
buf = read_sha1_file(entry->sha1, &type, &size);
407
if (!buf)
408
die("unable to read %s", sha1_to_hex(entry->sha1));
409
if (size != entry->size)
410
die("object %s size inconsistency (%lu vs %lu)",
411
sha1_to_hex(entry->sha1), size, entry->size);
392
- if (entry->delta) {
412
+ if (usable_delta) {
413
buf = delta_against(buf, size, entry);
414
size = entry->delta_size;
415
obj_type = (allow_ofs_delta && entry->delta->offset) ?
416
OBJ_OFS_DELTA : OBJ_REF_DELTA;
417
+ } else {
418
+ /*
419
+ * recover real object type in case
420
+ * check_object() wanted to re-use a delta,
421
+ * but we couldn't since base was in previous split pack
422
+ */
423
+ obj_type = type;
424
}
425
+ /* compress the data to store and put compressed length in datalen */
426
+ memset(&stream, 0, sizeof(stream));
427
+ deflateInit(&stream, pack_compression_level);
428
+ maxsize = deflateBound(&stream, size);
429
+ out = xmalloc(maxsize);
430
+ /* Compress it */
431
+ stream.next_in = buf;
432
+ stream.avail_in = size;
433
+ stream.next_out = out;
434
+ stream.avail_out = maxsize;
435
+ while (deflate(&stream, Z_FINISH) == Z_OK)
436
+ /* nothing */;
437
+ deflateEnd(&stream);
438
+ datalen = stream.total_out;
439
+ deflateEnd(&stream);
440
/*
441
* The object header is a byte of 'type' followed by zero or
442
* more bytes of length.
443
*/
444
hdrlen = encode_header(obj_type, size, header);
403
- sha1write(f, header, hdrlen);
445
446
if (obj_type == OBJ_OFS_DELTA) {
447
/*
@@ -409,21 +450,41 @@ static unsigned long write_object(struct sha1file *f,
450
* base from this object's position in the pack.
451
*/
452
off_t ofs = entry->offset - entry->delta->offset;
412
- unsigned pos = sizeof(header) - 1;
413
- header[pos] = ofs & 127;
453
+ unsigned pos = sizeof(dheader) - 1;
454
+ dheader[pos] = ofs & 127;
455
while (ofs >>= 7)
415
- header[--pos] = 128 | (--ofs & 127);
416
- sha1write(f, header + pos, sizeof(header) - pos);
417
- hdrlen += sizeof(header) - pos;
456
+ dheader[--pos] = 128 | (--ofs & 127);
457
+ if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) {
458
+ free(out);
459
+ free(buf);
460
+ return 0;
461
+ }
462
+ sha1write(f, header, hdrlen);
463
+ sha1write(f, dheader + pos, sizeof(dheader) - pos);
464
+ hdrlen += sizeof(dheader) - pos;
465
} else if (obj_type == OBJ_REF_DELTA) {
466
/*
467
* Deltas with a base reference contain
468
* an additional 20 bytes for the base sha1.
469
*/
470
+ if (limit && hdrlen + 20 + datalen + 20 >= limit) {
471
+ free(out);
472
+ free(buf);
473
+ return 0;
474
+ }
475
+ sha1write(f, header, hdrlen);
476
sha1write(f, entry->delta->sha1, 20);
477
hdrlen += 20;
478
+ } else {
479
+ if (limit && hdrlen + datalen + 20 >= limit) {
480
+ free(out);
481
+ free(buf);
482
+ return 0;
483
+ }
484
+ sha1write(f, header, hdrlen);
485
}
426
- datalen = sha1write_compressed(f, buf, size, pack_compression_level);
486
+ sha1write(f, out, datalen);
487
+ free(out);
488
free(buf);
489
}
490
else {
@@ -438,20 +499,6 @@ static unsigned long write_object(struct sha1file *f,
499
reused_delta++;
500
}
501
hdrlen = encode_header(obj_type, entry->size, header);
441
- sha1write(f, header, hdrlen);
442
- if (obj_type == OBJ_OFS_DELTA) {
443
- off_t ofs = entry->offset - entry->delta->offset;
444
- unsigned pos = sizeof(header) - 1;
445
- header[pos] = ofs & 127;
446
- while (ofs >>= 7)
447
- header[--pos] = 128 | (--ofs & 127);
448
- sha1write(f, header + pos, sizeof(header) - pos);
449
- hdrlen += sizeof(header) - pos;
450
- } else if (obj_type == OBJ_REF_DELTA) {
451
- sha1write(f, entry->delta->sha1, 20);
452
- hdrlen += 20;
453
- }
454
-
502
offset = entry->in_pack_offset;
503
revidx = find_packed_object(p, offset);
504
datalen = revidx[1].offset - offset;
@@ -460,6 +507,29 @@ static unsigned long write_object(struct sha1file *f,
507
die("bad packed object CRC for %s", sha1_to_hex(entry->sha1));
508
offset += entry->in_pack_header_size;
509
datalen -= entry->in_pack_header_size;
510
+ if (obj_type == OBJ_OFS_DELTA) {
511
+ off_t ofs = entry->offset - entry->delta->offset;
512
+ unsigned pos = sizeof(dheader) - 1;
513
+ dheader[pos] = ofs & 127;
514
+ while (ofs >>= 7)
515
+ dheader[--pos] = 128 | (--ofs & 127);
516
+ if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit)
517
+ return 0;
518
+ sha1write(f, header, hdrlen);
519
+ sha1write(f, dheader + pos, sizeof(dheader) - pos);
520
+ hdrlen += sizeof(dheader) - pos;
521
+ } else if (obj_type == OBJ_REF_DELTA) {
522
+ if (limit && hdrlen + 20 + datalen + 20 >= limit)
523
+ return 0;
524
+ sha1write(f, header, hdrlen);
525
+ sha1write(f, entry->delta->sha1, 20);
526
+ hdrlen += 20;
527
+ } else {
528
+ if (limit && hdrlen + datalen + 20 >= limit)
529
+ return 0;
530
+ sha1write(f, header, hdrlen);
531
+ }
532
+
533
if (!pack_to_stdout && p->index_version == 1 &&
534
check_pack_inflate(p, &w_curs, offset, datalen, entry->size))
535
die("corrupt packed object for %s", sha1_to_hex(entry->sha1));
@@ -467,7 +537,7 @@ static unsigned long write_object(struct sha1file *f,
537
unuse_pack(&w_curs);
538
reused++;
539
}
470
- if (entry->delta)
540
+ if (usable_delta)
541
written_delta++;
542
written++;
543
if (!pack_to_stdout)
@@ -486,11 +556,18 @@ static off_t write_one(struct sha1file *f,
556
return offset;
557
558
/* if we are deltified, write out base object first. */
489
- if (e->delta)
559
+ if (e->delta) {
560
offset = write_one(f, e->delta, offset);
561
+ if (!offset)
562
+ return 0;
563
+ }
564
565
e->offset = offset;
493
- size = write_object(f, e);
566
+ size = write_object(f, e, offset);
567
+ if (!size) {
568
+ e->offset = 0;
569
+ return 0;
570
+ }
571
572
/* make sure off_t is sufficiently large not to wrap */
573
if (offset > offset + size)