@matheusbsbs / matheus-bagliano / commits / 7a979d99ba
Fork 0

Thin pack - create packfile with missing delta base.

This goes together with "rev-list --object-edge" change, to feed pack-objects list of edge commits in addition to the usual object list. Upon seeing such list, pack-objects loosens the usual "self contained delta" constraints, and can produce delta against blobs and trees contained in the edge commits without storing the delta base objects themselves. The resulting packfile is not usable in .git/object/packs, but is a good way to implement "delta-only" transfer. Signed-off-by: Junio C Hamano <junkio@cox.net>

Junio C Hamano committed Feb 19, 2006 at 14:47 UTC 7a979d99bafae3576d9480007f958eed0c9e0278
1 file changed +219 -81
pack-objects.c
+219 -81
@@ -3,6 +3,7 @@
3 #include "delta.h"
4 #include "pack.h"
5 #include "csum-file.h"
6 +#include "diff.h"
7 #include <sys/time.h>
8
9 static const char pack_usage[] = "git-pack-objects [-q] [--no-reuse-delta] [--non-empty] [--local] [--incremental] [--window=N] [--depth=N] {--stdout | base-name} < object-list";
@@ -26,6 +27,13 @@ struct object_entry {
27 struct object_entry *delta_sibling; /* other deltified objects who
28 * uses the same base as me
29 */
30 + int preferred_base; /* we do not pack this, but is encouraged to
31 + * be used as the base objectto delta huge
32 + * objects against.
33 + */
34 + int based_on_preferred; /* current delta candidate is a preferred
35 + * one, or delta against a preferred one.
36 + */
37 };
38
39 /*
@@ -48,7 +56,7 @@ static int local = 0;
56 static int incremental = 0;
57 static struct object_entry **sorted_by_sha, **sorted_by_type;
58 static struct object_entry *objects = NULL;
51 -static int nr_objects = 0, nr_alloc = 0;
59 +static int nr_objects = 0, nr_alloc = 0, nr_result = 0;
60 static const char *base_name;
61 static unsigned char pack_file_sha1[20];
62 static int progress = 1;
@@ -229,7 +237,8 @@ static int encode_header(enum object_type type, unsigned long size, unsigned cha
237 return n;
238 }
239
232 -static unsigned long write_object(struct sha1file *f, struct object_entry *entry)
240 +static unsigned long write_object(struct sha1file *f,
241 + struct object_entry *entry)
242 {
243 unsigned long size;
244 char type[10];
@@ -239,6 +248,9 @@ static unsigned long write_object(struct sha1file *f, struct object_entry *entry
248 enum object_type obj_type;
249 int to_reuse = 0;
250
251 + if (entry->preferred_base)
252 + return 0;
253 +
254 obj_type = entry->type;
255 if (! entry->in_pack)
256 to_reuse = 0; /* can't reuse what we don't have */
@@ -326,10 +338,11 @@ static void write_pack_file(void)
338 if (!base_name)
339 f = sha1fd(1, "<stdout>");
340 else
329 - f = sha1create("%s-%s.%s", base_name, sha1_to_hex(object_list_sha1), "pack");
341 + f = sha1create("%s-%s.%s", base_name,
342 + sha1_to_hex(object_list_sha1), "pack");
343 hdr.hdr_signature = htonl(PACK_SIGNATURE);
344 hdr.hdr_version = htonl(PACK_VERSION);
332 - hdr.hdr_entries = htonl(nr_objects);
345 + hdr.hdr_entries = htonl(nr_result);
346 sha1write(f, &hdr, sizeof(hdr));
347 offset = sizeof(hdr);
348 for (i = 0; i < nr_objects; i++)
@@ -341,9 +354,10 @@ static void write_pack_file(void)
354 static void write_index_file(void)
355 {
356 int i;
344 - struct sha1file *f = sha1create("%s-%s.%s", base_name, sha1_to_hex(object_list_sha1), "idx");
357 + struct sha1file *f = sha1create("%s-%s.%s", base_name,
358 + sha1_to_hex(object_list_sha1), "idx");
359 struct object_entry **list = sorted_by_sha;
346 - struct object_entry **last = list + nr_objects;
360 + struct object_entry **last = list + nr_result;
361 unsigned int array[256];
362
363 /*
@@ -368,7 +382,7 @@ static void write_index_file(void)
382 * Write the actual SHA1 entries..
383 */
384 list = sorted_by_sha;
371 - for (i = 0; i < nr_objects; i++) {
385 + for (i = 0; i < nr_result; i++) {
386 struct object_entry *entry = *list++;
387 unsigned int offset = htonl(entry->offset);
388 sha1write(f, &offset, 4);
@@ -378,27 +392,87 @@ static void write_index_file(void)
392 sha1close(f, NULL, 1);
393 }
394
381 -static int add_object_entry(unsigned char *sha1, unsigned int hash)
395 +static int locate_object_entry_hash(const unsigned char *sha1)
396 +{
397 + int i;
398 + unsigned int ui;
399 + memcpy(&ui, sha1, sizeof(unsigned int));
400 + i = ui % object_ix_hashsz;
401 + while (0 < object_ix[i]) {
402 + if (!memcmp(sha1, objects[object_ix[i]-1].sha1, 20))
403 + return i;
404 + if (++i == object_ix_hashsz)
405 + i = 0;
406 + }
407 + return -1 - i;
408 +}
409 +
410 +static struct object_entry *locate_object_entry(const unsigned char *sha1)
411 +{
412 + int i;
413 +
414 + if (!object_ix_hashsz)
415 + return NULL;
416 +
417 + i = locate_object_entry_hash(sha1);
418 + if (0 <= i)
419 + return &objects[object_ix[i]-1];
420 + return NULL;
421 +}
422 +
423 +static void rehash_objects(void)
424 {
425 + int i;
426 + struct object_entry *oe;
427 +
428 + object_ix_hashsz = nr_objects * 3;
429 + if (object_ix_hashsz < 1024)
430 + object_ix_hashsz = 1024;
431 + object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz);
432 + object_ix = memset(object_ix, 0, sizeof(int) * object_ix_hashsz);
433 + for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
434 + int ix = locate_object_entry_hash(oe->sha1);
435 + if (0 <= ix)
436 + continue;
437 + ix = -1 - ix;
438 + object_ix[ix] = i + 1;
439 + }
440 +}
441 +
442 +static int add_object_entry(const unsigned char *sha1, const char *name, int exclude)
443 +{
444 + unsigned int hash = 0;
445 unsigned int idx = nr_objects;
446 struct object_entry *entry;
447 struct packed_git *p;
448 unsigned int found_offset = 0;
449 struct packed_git *found_pack = NULL;
388 -
389 - for (p = packed_git; p; p = p->next) {
390 - struct pack_entry e;
391 - if (find_pack_entry_one(sha1, &e, p)) {
392 - if (incremental)
393 - return 0;
394 - if (local && !p->pack_local)
395 - return 0;
396 - if (!found_pack) {
397 - found_offset = e.offset;
398 - found_pack = e.p;
450 + int ix;
451 +
452 + if (!exclude) {
453 + for (p = packed_git; p; p = p->next) {
454 + struct pack_entry e;
455 + if (find_pack_entry_one(sha1, &e, p)) {
456 + if (incremental)
457 + return 0;
458 + if (local && !p->pack_local)
459 + return 0;
460 + if (!found_pack) {
461 + found_offset = e.offset;
462 + found_pack = e.p;
463 + }
464 }
465 }
466 }
467 + if ((entry = locate_object_entry(sha1)) != NULL)
468 + goto already_added;
469 +
470 + while (*name) {
471 + unsigned char c = *name++;
472 + if (isspace(c))
473 + continue;
474 + hash = hash * 11 + c;
475 + }
476
477 if (idx >= nr_alloc) {
478 unsigned int needed = (idx + 1024) * 3 / 2;
@@ -406,45 +480,79 @@ static int add_object_entry(unsigned char *sha1, unsigned int hash)
480 nr_alloc = needed;
481 }
482 entry = objects + idx;
483 + nr_objects = idx + 1;
484 memset(entry, 0, sizeof(*entry));
485 memcpy(entry->sha1, sha1, 20);
486 entry->hash = hash;
412 - if (found_pack) {
413 - entry->in_pack = found_pack;
414 - entry->in_pack_offset = found_offset;
487 +
488 + if (object_ix_hashsz * 3 <= nr_objects * 4)
489 + rehash_objects();
490 + else {
491 + ix = locate_object_entry_hash(entry->sha1);
492 + if (0 <= ix)
493 + die("internal error in object hashing.");
494 + object_ix[-1 - ix] = idx + 1;
495 + }
496 +
497 + already_added:
498 + if (exclude)
499 + entry->preferred_base = 1;
500 + else {
501 + if (found_pack) {
502 + entry->in_pack = found_pack;
503 + entry->in_pack_offset = found_offset;
504 + }
505 }
416 - nr_objects = idx+1;
506 return 1;
507 }
508
420 -static int locate_object_entry_hash(unsigned char *sha1)
509 +static void add_pbase_tree(struct tree_desc *tree)
510 {
422 - int i;
423 - unsigned int ui;
424 - memcpy(&ui, sha1, sizeof(unsigned int));
425 - i = ui % object_ix_hashsz;
426 - while (0 < object_ix[i]) {
427 - if (!memcmp(sha1, objects[object_ix[i]-1].sha1, 20))
428 - return i;
429 - if (++i == object_ix_hashsz)
430 - i = 0;
511 + while (tree->size) {
512 + const unsigned char *sha1;
513 + const char *name;
514 + unsigned mode;
515 + unsigned long size;
516 + char type[20];
517 +
518 + sha1 = tree_entry_extract(tree, &name, &mode);
519 + update_tree_entry(tree);
520 + if (!has_sha1_file(sha1))
521 + continue;
522 + if (sha1_object_info(sha1, type, &size))
523 + continue;
524 + add_object_entry(sha1, name, 1);
525 + if (!strcmp(type, "tree")) {
526 + struct tree_desc sub;
527 + void *elem;
528 + elem = read_sha1_file(sha1, type, &sub.size);
529 + sub.buf = elem;
530 + if (sub.buf) {
531 + add_pbase_tree(&sub);
532 + free(elem);
533 + }
534 + }
535 }
432 - return -1 - i;
536 }
537
435 -static struct object_entry *locate_object_entry(unsigned char *sha1)
538 +static void add_preferred_base(unsigned char *sha1)
539 {
437 - int i = locate_object_entry_hash(sha1);
438 - if (0 <= i)
439 - return &objects[object_ix[i]-1];
440 - return NULL;
540 + struct tree_desc tree;
541 + void *elem;
542 + elem = read_object_with_reference(sha1, "tree", &tree.size, NULL);
543 + tree.buf = elem;
544 + if (!tree.buf)
545 + return;
546 + add_object_entry(sha1, "", 1);
547 + add_pbase_tree(&tree);
548 + free(elem);
549 }
550
551 static void check_object(struct object_entry *entry)
552 {
553 char type[20];
554
447 - if (entry->in_pack) {
555 + if (entry->in_pack && !entry->preferred_base) {
556 unsigned char base[20];
557 unsigned long size;
558 struct object_entry *base_entry;
@@ -463,7 +571,8 @@ static void check_object(struct object_entry *entry)
571 */
572 if (!no_reuse_delta &&
573 entry->in_pack_type == OBJ_DELTA &&
466 - (base_entry = locate_object_entry(base))) {
574 + (base_entry = locate_object_entry(base)) &&
575 + (!base_entry->preferred_base)) {
576
577 /* Depth value does not matter - find_deltas()
578 * will never consider reused delta as the
@@ -501,25 +610,6 @@ static void check_object(struct object_entry *entry)
610 sha1_to_hex(entry->sha1), type);
611 }
612
504 -static void hash_objects(void)
505 -{
506 - int i;
507 - struct object_entry *oe;
508 -
509 - object_ix_hashsz = nr_objects * 2;
510 - object_ix = xcalloc(sizeof(int), object_ix_hashsz);
511 - for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
512 - int ix = locate_object_entry_hash(oe->sha1);
513 - if (0 <= ix) {
514 - error("the same object '%s' added twice",
515 - sha1_to_hex(oe->sha1));
516 - continue;
517 - }
518 - ix = -1 - ix;
519 - object_ix[ix] = i + 1;
520 - }
521 -}
522 -
613 static unsigned int check_delta_limit(struct object_entry *me, unsigned int n)
614 {
615 struct object_entry *child = me->delta_child;
@@ -538,7 +628,6 @@ static void get_object_details(void)
628 int i;
629 struct object_entry *entry;
630
541 - hash_objects();
631 prepare_pack_ix();
632 for (i = 0, entry = objects; i < nr_objects; i++, entry++)
633 check_object(entry);
@@ -576,6 +665,24 @@ static int sha1_sort(const struct object_entry *a, const struct object_entry *b)
665 return memcmp(a->sha1, b->sha1, 20);
666 }
667
668 +static struct object_entry **create_final_object_list()
669 +{
670 + struct object_entry **list;
671 + int i, j;
672 +
673 + for (i = nr_result = 0; i < nr_objects; i++)
674 + if (!objects[i].preferred_base)
675 + nr_result++;
676 + list = xmalloc(nr_result * sizeof(struct object_entry *));
677 + for (i = j = 0; i < nr_objects; i++) {
678 + if (!objects[i].preferred_base)
679 + list[j++] = objects + i;
680 + }
681 + current_sort = sha1_sort;
682 + qsort(list, nr_result, sizeof(struct object_entry *), sort_comparator);
683 + return list;
684 +}
685 +
686 static int type_size_sort(const struct object_entry *a, const struct object_entry *b)
687 {
688 if (a->type < b->type)
@@ -586,6 +693,10 @@ static int type_size_sort(const struct object_entry *a, const struct object_entr
693 return -1;
694 if (a->hash > b->hash)
695 return 1;
696 + if (a->preferred_base < b->preferred_base)
697 + return -1;
698 + if (a->preferred_base > b->preferred_base)
699 + return 1;
700 if (a->size < b->size)
701 return -1;
702 if (a->size > b->size)
@@ -610,6 +721,8 @@ static int try_delta(struct unpacked *cur, struct unpacked *old, unsigned max_de
721 {
722 struct object_entry *cur_entry = cur->entry;
723 struct object_entry *old_entry = old->entry;
724 + int old_preferred = (old_entry->preferred_base ||
725 + old_entry->based_on_preferred);
726 unsigned long size, oldsize, delta_size, sizediff;
727 long max_size;
728 void *delta_buf;
@@ -618,9 +731,15 @@ static int try_delta(struct unpacked *cur, struct unpacked *old, unsigned max_de
731 if (cur_entry->type != old_entry->type)
732 return -1;
733
621 - /* If the current object is at edge, take the depth the objects
622 - * that depend on the current object into account -- otherwise
623 - * they would become too deep.
734 + /* We do not compute delta to *create* objects we are not
735 + * going to pack.
736 + */
737 + if (cur_entry->preferred_base)
738 + return -1;
739 +
740 + /* If the current object is at pack edge, take the depth the
741 + * objects that depend on the current object into account --
742 + * otherwise they would become too deep.
743 */
744 if (cur_entry->delta_child) {
745 if (max_depth <= cur_entry->delta_limit)
@@ -646,8 +765,27 @@ static int try_delta(struct unpacked *cur, struct unpacked *old, unsigned max_de
765 * delete).
766 */
767 max_size = size / 2 - 20;
649 - if (cur_entry->delta)
650 - max_size = cur_entry->delta_size-1;
768 + if (cur_entry->delta) {
769 + if (cur_entry->based_on_preferred) {
770 + if (old_preferred)
771 + max_size = cur_entry->delta_size-1;
772 + else
773 + /* trying with non-preferred one when we
774 + * already have a delta based on preferred
775 + * one is pointless.
776 + */
777 + return 0;
778 + }
779 + else if (!old_preferred)
780 + max_size = cur_entry->delta_size-1;
781 + else
782 + /* otherwise... even if delta with a
783 + * preferred one produces a bigger result than
784 + * what we currently have, which is based on a
785 + * non-preferred one, it is OK.
786 + */
787 + ;
788 + }
789 if (sizediff >= max_size)
790 return -1;
791 delta_buf = diff_delta(old->data, oldsize,
@@ -657,6 +795,7 @@ static int try_delta(struct unpacked *cur, struct unpacked *old, unsigned max_de
795 cur_entry->delta = old_entry;
796 cur_entry->delta_size = delta_size;
797 cur_entry->depth = old_entry->depth + 1;
798 + cur_entry->based_on_preferred = old_preferred;
799 free(delta_buf);
800 return 0;
801 }
@@ -722,7 +861,7 @@ static void find_deltas(struct object_entry **list, int window, int depth)
861 static void prepare_pack(int window, int depth)
862 {
863 if (progress)
725 - fprintf(stderr, "Packing %d objects", nr_objects);
864 + fprintf(stderr, "Packing %d objects", nr_result);
865 get_object_details();
866 if (progress)
867 fputc('.', stderr);
@@ -861,8 +1000,6 @@ int main(int argc, char **argv)
1000 gettimeofday(&prev_tv, NULL);
1001 }
1002 while (fgets(line, sizeof(line), stdin) != NULL) {
864 - unsigned int hash;
865 - char *p;
1003 unsigned char sha1[20];
1004
1005 if (progress && (eye_candy <= nr_objects)) {
@@ -881,31 +1018,32 @@ int main(int argc, char **argv)
1018 }
1019 eye_candy += eye_candy_incr;
1020 }
1021 + if (line[0] == '-') {
1022 + if (get_sha1_hex(line+1, sha1))
1023 + die("expected edge sha1, got garbage:\n %s",
1024 + line+1);
1025 + add_preferred_base(sha1);
1026 + continue;
1027 + }
1028 if (get_sha1_hex(line, sha1))
1029 die("expected sha1, got garbage:\n %s", line);
886 - hash = 0;
887 - p = line+40;
888 - while (*p) {
889 - unsigned char c = *p++;
890 - if (isspace(c))
891 - continue;
892 - hash = hash * 11 + c;
893 - }
894 - add_object_entry(sha1, hash);
1030 + add_object_entry(sha1, line+40, 0);
1031 }
1032 if (progress)
1033 fprintf(stderr, "Done counting %d objects.\n", nr_objects);
1034 if (non_empty && !nr_objects)
1035 return 0;
1036
901 - sorted_by_sha = create_sorted_list(sha1_sort);
1037 + sorted_by_sha = create_final_object_list();
1038 SHA1_Init(&ctx);
1039 list = sorted_by_sha;
904 - for (i = 0; i < nr_objects; i++) {
1040 + for (i = 0; i < nr_result; i++) {
1041 struct object_entry *entry = *list++;
1042 SHA1_Update(&ctx, entry->sha1, 20);
1043 }
1044 SHA1_Final(object_list_sha1, &ctx);
1045 + if (progress && (nr_objects != nr_result))
1046 + fprintf(stderr, "Result has %d objects.\n", nr_result);
1047
1048 if (reuse_cached_pack(object_list_sha1, pack_to_stdout))
1049 ;
@@ -918,6 +1056,6 @@ int main(int argc, char **argv)
1056 }
1057 if (progress)
1058 fprintf(stderr, "Total %d, written %d (delta %d), reused %d (delta %d)\n",
921 - nr_objects, written, written_delta, reused, reused_delta);
1059 + nr_result, written, written_delta, reused, reused_delta);
1060 return 0;
1061 }