@matheusbsbs / matheus-bagliano / commits / 17b08f2cd0
Fork 0

git-repack --max-pack-size: write_{object,one}() respect pack limit

With --max-pack-size, generate the appropriate write limit for each object and check against it before each group of writes. Update delta usability rules to handle base being in a previously- written pack. Inline sha1write_compress() so we know the exact size of the written data when it needs to be compressed. Detect and return write "failure". Signed-off-by: Dana L. How <danahow@gmail.com> Signed-off-by: Junio C Hamano <junkio@cox.net>

Dana L. How committed May 13, 2007 at 12:06 UTC 17b08f2cd0000d42bb9f0a36354ab9300d64d420
1 file changed +104 -27
builtin-pack-objects.c
+104 -27
@@ -352,16 +352,31 @@ static void copy_pack_data(struct sha1file *f,
352 }
353
354 static unsigned long write_object(struct sha1file *f,
355 - struct object_entry *entry)
355 + struct object_entry *entry,
356 + off_t write_offset)
357 {
358 unsigned long size;
359 enum object_type type;
360 void *buf;
361 unsigned char header[10];
362 + unsigned char dheader[10];
363 unsigned hdrlen;
364 off_t datalen;
365 enum object_type obj_type;
366 int to_reuse = 0;
367 + /* write limit if limited packsize and not first object */
368 + unsigned long limit = pack_size_limit && nr_written ?
369 + pack_size_limit - write_offset : 0;
370 + /* no if no delta */
371 + int usable_delta = !entry->delta ? 0 :
372 + /* yes if unlimited packfile */
373 + !pack_size_limit ? 1 :
374 + /* no if base written to previous pack */
375 + entry->delta->offset == (off_t)-1 ? 0 :
376 + /* otherwise double-check written to this
377 + * pack, like we do below
378 + */
379 + entry->delta->offset ? 1 : 0;
380
381 if (!pack_to_stdout)
382 crc32_begin(f);
@@ -372,7 +387,9 @@ static unsigned long write_object(struct sha1file *f,
387 else if (!entry->in_pack)
388 to_reuse = 0; /* can't reuse what we don't have */
389 else if (obj_type == OBJ_REF_DELTA || obj_type == OBJ_OFS_DELTA)
375 - to_reuse = 1; /* check_object() decided it for us */
390 + /* check_object() decided it for us ... */
391 + to_reuse = usable_delta;
392 + /* ... but pack split may override that */
393 else if (obj_type != entry->in_pack_type)
394 to_reuse = 0; /* pack has delta which is unusable */
395 else if (entry->delta)
@@ -383,24 +400,48 @@ static unsigned long write_object(struct sha1file *f,
400 */
401
402 if (!to_reuse) {
403 + z_stream stream;
404 + unsigned long maxsize;
405 + void *out;
406 buf = read_sha1_file(entry->sha1, &type, &size);
407 if (!buf)
408 die("unable to read %s", sha1_to_hex(entry->sha1));
409 if (size != entry->size)
410 die("object %s size inconsistency (%lu vs %lu)",
411 sha1_to_hex(entry->sha1), size, entry->size);
392 - if (entry->delta) {
412 + if (usable_delta) {
413 buf = delta_against(buf, size, entry);
414 size = entry->delta_size;
415 obj_type = (allow_ofs_delta && entry->delta->offset) ?
416 OBJ_OFS_DELTA : OBJ_REF_DELTA;
417 + } else {
418 + /*
419 + * recover real object type in case
420 + * check_object() wanted to re-use a delta,
421 + * but we couldn't since base was in previous split pack
422 + */
423 + obj_type = type;
424 }
425 + /* compress the data to store and put compressed length in datalen */
426 + memset(&stream, 0, sizeof(stream));
427 + deflateInit(&stream, pack_compression_level);
428 + maxsize = deflateBound(&stream, size);
429 + out = xmalloc(maxsize);
430 + /* Compress it */
431 + stream.next_in = buf;
432 + stream.avail_in = size;
433 + stream.next_out = out;
434 + stream.avail_out = maxsize;
435 + while (deflate(&stream, Z_FINISH) == Z_OK)
436 + /* nothing */;
437 + deflateEnd(&stream);
438 + datalen = stream.total_out;
439 + deflateEnd(&stream);
440 /*
441 * The object header is a byte of 'type' followed by zero or
442 * more bytes of length.
443 */
444 hdrlen = encode_header(obj_type, size, header);
403 - sha1write(f, header, hdrlen);
445
446 if (obj_type == OBJ_OFS_DELTA) {
447 /*
@@ -409,21 +450,41 @@ static unsigned long write_object(struct sha1file *f,
450 * base from this object's position in the pack.
451 */
452 off_t ofs = entry->offset - entry->delta->offset;
412 - unsigned pos = sizeof(header) - 1;
413 - header[pos] = ofs & 127;
453 + unsigned pos = sizeof(dheader) - 1;
454 + dheader[pos] = ofs & 127;
455 while (ofs >>= 7)
415 - header[--pos] = 128 | (--ofs & 127);
416 - sha1write(f, header + pos, sizeof(header) - pos);
417 - hdrlen += sizeof(header) - pos;
456 + dheader[--pos] = 128 | (--ofs & 127);
457 + if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) {
458 + free(out);
459 + free(buf);
460 + return 0;
461 + }
462 + sha1write(f, header, hdrlen);
463 + sha1write(f, dheader + pos, sizeof(dheader) - pos);
464 + hdrlen += sizeof(dheader) - pos;
465 } else if (obj_type == OBJ_REF_DELTA) {
466 /*
467 * Deltas with a base reference contain
468 * an additional 20 bytes for the base sha1.
469 */
470 + if (limit && hdrlen + 20 + datalen + 20 >= limit) {
471 + free(out);
472 + free(buf);
473 + return 0;
474 + }
475 + sha1write(f, header, hdrlen);
476 sha1write(f, entry->delta->sha1, 20);
477 hdrlen += 20;
478 + } else {
479 + if (limit && hdrlen + datalen + 20 >= limit) {
480 + free(out);
481 + free(buf);
482 + return 0;
483 + }
484 + sha1write(f, header, hdrlen);
485 }
426 - datalen = sha1write_compressed(f, buf, size, pack_compression_level);
486 + sha1write(f, out, datalen);
487 + free(out);
488 free(buf);
489 }
490 else {
@@ -438,20 +499,6 @@ static unsigned long write_object(struct sha1file *f,
499 reused_delta++;
500 }
501 hdrlen = encode_header(obj_type, entry->size, header);
441 - sha1write(f, header, hdrlen);
442 - if (obj_type == OBJ_OFS_DELTA) {
443 - off_t ofs = entry->offset - entry->delta->offset;
444 - unsigned pos = sizeof(header) - 1;
445 - header[pos] = ofs & 127;
446 - while (ofs >>= 7)
447 - header[--pos] = 128 | (--ofs & 127);
448 - sha1write(f, header + pos, sizeof(header) - pos);
449 - hdrlen += sizeof(header) - pos;
450 - } else if (obj_type == OBJ_REF_DELTA) {
451 - sha1write(f, entry->delta->sha1, 20);
452 - hdrlen += 20;
453 - }
454 -
502 offset = entry->in_pack_offset;
503 revidx = find_packed_object(p, offset);
504 datalen = revidx[1].offset - offset;
@@ -460,6 +507,29 @@ static unsigned long write_object(struct sha1file *f,
507 die("bad packed object CRC for %s", sha1_to_hex(entry->sha1));
508 offset += entry->in_pack_header_size;
509 datalen -= entry->in_pack_header_size;
510 + if (obj_type == OBJ_OFS_DELTA) {
511 + off_t ofs = entry->offset - entry->delta->offset;
512 + unsigned pos = sizeof(dheader) - 1;
513 + dheader[pos] = ofs & 127;
514 + while (ofs >>= 7)
515 + dheader[--pos] = 128 | (--ofs & 127);
516 + if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit)
517 + return 0;
518 + sha1write(f, header, hdrlen);
519 + sha1write(f, dheader + pos, sizeof(dheader) - pos);
520 + hdrlen += sizeof(dheader) - pos;
521 + } else if (obj_type == OBJ_REF_DELTA) {
522 + if (limit && hdrlen + 20 + datalen + 20 >= limit)
523 + return 0;
524 + sha1write(f, header, hdrlen);
525 + sha1write(f, entry->delta->sha1, 20);
526 + hdrlen += 20;
527 + } else {
528 + if (limit && hdrlen + datalen + 20 >= limit)
529 + return 0;
530 + sha1write(f, header, hdrlen);
531 + }
532 +
533 if (!pack_to_stdout && p->index_version == 1 &&
534 check_pack_inflate(p, &w_curs, offset, datalen, entry->size))
535 die("corrupt packed object for %s", sha1_to_hex(entry->sha1));
@@ -467,7 +537,7 @@ static unsigned long write_object(struct sha1file *f,
537 unuse_pack(&w_curs);
538 reused++;
539 }
470 - if (entry->delta)
540 + if (usable_delta)
541 written_delta++;
542 written++;
543 if (!pack_to_stdout)
@@ -486,11 +556,18 @@ static off_t write_one(struct sha1file *f,
556 return offset;
557
558 /* if we are deltified, write out base object first. */
489 - if (e->delta)
559 + if (e->delta) {
560 offset = write_one(f, e->delta, offset);
561 + if (!offset)
562 + return 0;
563 + }
564
565 e->offset = offset;
493 - size = write_object(f, e);
566 + size = write_object(f, e, offset);
567 + if (!size) {
568 + e->offset = 0;
569 + return 0;
570 + }
571
572 /* make sure off_t is sufficiently large not to wrap */
573 if (offset > offset + size)