@matheusbsbs / matheus-bagliano / commits / 5379a5c5ee
Fork 0

Thin pack generation: optimization.

Jens Axboe noticed that recent "git push" has become very slow since we made --thin transfer the default. Thin pack generation to push a handful revisions that touch relatively small number of paths out of huge tree was stupid; it registered _everything_ from the excluded revisions. As a result, "Counting objects" phase was unnecessarily expensive. This changes the logic to register the blobs and trees from excluded revisions only for paths we are actually going to send to the other end. Signed-off-by: Junio C Hamano <junkio@cox.net>

Junio C Hamano committed Apr 5, 2006 at 23:24 UTC 5379a5c5ee45d1380240a47573c7571de92626bb
1 file changed +236 -48
pack-objects.c
+236 -48
@@ -453,7 +453,7 @@ static void rehash_objects(void)
453 if (object_ix_hashsz < 1024)
454 object_ix_hashsz = 1024;
455 object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz);
456 - object_ix = memset(object_ix, 0, sizeof(int) * object_ix_hashsz);
456 + memset(object_ix, 0, sizeof(int) * object_ix_hashsz);
457 for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
458 int ix = locate_object_entry_hash(oe->sha1);
459 if (0 <= ix)
@@ -505,21 +505,6 @@ static unsigned name_hash(struct name_path *path, const char *name)
505 * but close enough.
506 */
507 hash = (name_hash<<DIRBITS) | (hash & ((1U<<DIRBITS )-1));
508 -
509 - if (0) { /* debug */
510 - n = name + strlen(name);
511 - if (n != name && n[-1] == '\n')
512 - n--;
513 - while (name <= --n)
514 - fputc(*n, stderr);
515 - for (p = path; p; p = p->up) {
516 - fputc('/', stderr);
517 - n = p->elem + p->len;
518 - while (p->elem <= --n)
519 - fputc(*n, stderr);
520 - }
521 - fprintf(stderr, "\t%08x\n", hash);
522 - }
508 return hash;
509 }
510
@@ -587,56 +572,254 @@ static int add_object_entry(const unsigned char *sha1, unsigned hash, int exclud
572 return status;
573 }
574
590 -static void add_pbase_tree(struct tree_desc *tree, struct name_path *up)
575 +struct pbase_tree_cache {
576 + unsigned char sha1[20];
577 + int ref;
578 + int temporary;
579 + void *tree_data;
580 + unsigned long tree_size;
581 +};
582 +
583 +static struct pbase_tree_cache *(pbase_tree_cache[256]);
584 +static int pbase_tree_cache_ix(const unsigned char *sha1)
585 +{
586 + return sha1[0] % ARRAY_SIZE(pbase_tree_cache);
587 +}
588 +static int pbase_tree_cache_ix_incr(int ix)
589 +{
590 + return (ix+1) % ARRAY_SIZE(pbase_tree_cache);
591 +}
592 +
593 +static struct pbase_tree {
594 + struct pbase_tree *next;
595 + /* This is a phony "cache" entry; we are not
596 + * going to evict it nor find it through _get()
597 + * mechanism -- this is for the toplevel node that
598 + * would almost always change with any commit.
599 + */
600 + struct pbase_tree_cache pcache;
601 +} *pbase_tree;
602 +
603 +static struct pbase_tree_cache *pbase_tree_get(const unsigned char *sha1)
604 +{
605 + struct pbase_tree_cache *ent, *nent;
606 + void *data;
607 + unsigned long size;
608 + char type[20];
609 + int neigh;
610 + int my_ix = pbase_tree_cache_ix(sha1);
611 + int available_ix = -1;
612 +
613 + /* pbase-tree-cache acts as a limited hashtable.
614 + * your object will be found at your index or within a few
615 + * slots after that slot if it is cached.
616 + */
617 + for (neigh = 0; neigh < 8; neigh++) {
618 + ent = pbase_tree_cache[my_ix];
619 + if (ent && !memcmp(ent->sha1, sha1, 20)) {
620 + ent->ref++;
621 + return ent;
622 + }
623 + else if (((available_ix < 0) && (!ent || !ent->ref)) ||
624 + ((0 <= available_ix) &&
625 + (!ent && pbase_tree_cache[available_ix])))
626 + available_ix = my_ix;
627 + if (!ent)
628 + break;
629 + my_ix = pbase_tree_cache_ix_incr(my_ix);
630 + }
631 +
632 + /* Did not find one. Either we got a bogus request or
633 + * we need to read and perhaps cache.
634 + */
635 + data = read_sha1_file(sha1, type, &size);
636 + if (!data)
637 + return NULL;
638 + if (strcmp(type, tree_type)) {
639 + free(data);
640 + return NULL;
641 + }
642 +
643 + /* We need to either cache or return a throwaway copy */
644 +
645 + if (available_ix < 0)
646 + ent = NULL;
647 + else {
648 + ent = pbase_tree_cache[available_ix];
649 + my_ix = available_ix;
650 + }
651 +
652 + if (!ent) {
653 + nent = xmalloc(sizeof(*nent));
654 + nent->temporary = (available_ix < 0);
655 + }
656 + else {
657 + /* evict and reuse */
658 + free(ent->tree_data);
659 + nent = ent;
660 + }
661 + memcpy(nent->sha1, sha1, 20);
662 + nent->tree_data = data;
663 + nent->tree_size = size;
664 + nent->ref = 1;
665 + if (!nent->temporary)
666 + pbase_tree_cache[my_ix] = nent;
667 + return nent;
668 +}
669 +
670 +static void pbase_tree_put(struct pbase_tree_cache *cache)
671 +{
672 + if (!cache->temporary) {
673 + cache->ref--;
674 + return;
675 + }
676 + free(cache->tree_data);
677 + free(cache);
678 +}
679 +
680 +static int name_cmp_len(const char *name)
681 +{
682 + int i;
683 + for (i = 0; name[i] && name[i] != '\n' && name[i] != '/'; i++)
684 + ;
685 + return i;
686 +}
687 +
688 +static void add_pbase_object(struct tree_desc *tree,
689 + struct name_path *up,
690 + const char *name,
691 + int cmplen)
692 {
693 while (tree->size) {
694 const unsigned char *sha1;
594 - const char *name;
595 - unsigned mode, hash;
695 + const char *entry_name;
696 + int entry_len;
697 + unsigned mode;
698 unsigned long size;
699 char type[20];
700
599 - sha1 = tree_entry_extract(tree, &name, &mode);
701 + sha1 = tree_entry_extract(tree, &entry_name, &mode);
702 update_tree_entry(tree);
601 - if (!has_sha1_file(sha1))
602 - continue;
603 - if (sha1_object_info(sha1, type, &size))
703 + entry_len = strlen(entry_name);
704 + if (entry_len != cmplen ||
705 + memcmp(entry_name, name, cmplen) ||
706 + !has_sha1_file(sha1) ||
707 + sha1_object_info(sha1, type, &size))
708 continue;
605 -
606 - hash = name_hash(up, name);
607 - if (!add_object_entry(sha1, hash, 1))
608 - continue;
609 -
709 + if (name[cmplen] != '/') {
710 + unsigned hash = name_hash(up, name);
711 + add_object_entry(sha1, hash, 1);
712 + return;
713 + }
714 if (!strcmp(type, tree_type)) {
715 struct tree_desc sub;
612 - void *elem;
716 struct name_path me;
717 + struct pbase_tree_cache *tree;
718 + const char *down = name+cmplen+1;
719 + int downlen = name_cmp_len(down);
720 +
721 + tree = pbase_tree_get(sha1);
722 + if (!tree)
723 + return;
724 + sub.buf = tree->tree_data;
725 + sub.size = tree->tree_size;
726 +
727 + me.up = up;
728 + me.elem = entry_name;
729 + me.len = entry_len;
730 + add_pbase_object(&sub, &me, down, downlen);
731 + pbase_tree_put(tree);
732 + }
733 + }
734 +}
735
615 - elem = read_sha1_file(sha1, type, &sub.size);
616 - sub.buf = elem;
617 - if (sub.buf) {
618 - me.up = up;
619 - me.elem = name;
620 - me.len = strlen(name);
621 - add_pbase_tree(&sub, &me);
622 - free(elem);
623 - }
736 +static unsigned *done_pbase_paths;
737 +static int done_pbase_paths_num;
738 +static int done_pbase_paths_alloc;
739 +static int done_pbase_path_pos(unsigned hash)
740 +{
741 + int lo = 0;
742 + int hi = done_pbase_paths_num;
743 + while (lo < hi) {
744 + int mi = (hi + lo) / 2;
745 + if (done_pbase_paths[mi] == hash)
746 + return mi;
747 + if (done_pbase_paths[mi] < hash)
748 + hi = mi;
749 + else
750 + lo = mi + 1;
751 + }
752 + return -lo-1;
753 +}
754 +
755 +static int check_pbase_path(unsigned hash)
756 +{
757 + int pos = (!done_pbase_paths) ? -1 : done_pbase_path_pos(hash);
758 + if (0 <= pos)
759 + return 1;
760 + pos = -pos - 1;
761 + if (done_pbase_paths_alloc <= done_pbase_paths_num) {
762 + done_pbase_paths_alloc = alloc_nr(done_pbase_paths_alloc);
763 + done_pbase_paths = xrealloc(done_pbase_paths,
764 + done_pbase_paths_alloc *
765 + sizeof(unsigned));
766 + }
767 + done_pbase_paths_num++;
768 + if (pos < done_pbase_paths_num)
769 + memmove(done_pbase_paths + pos + 1,
770 + done_pbase_paths + pos,
771 + (done_pbase_paths_num - pos - 1) * sizeof(unsigned));
772 + done_pbase_paths[pos] = hash;
773 + return 0;
774 +}
775 +
776 +static void add_preferred_base_object(char *name, unsigned hash)
777 +{
778 + struct pbase_tree *it;
779 + int cmplen = name_cmp_len(name);
780 +
781 + if (check_pbase_path(hash))
782 + return;
783 +
784 + for (it = pbase_tree; it; it = it->next) {
785 + if (cmplen == 0) {
786 + hash = name_hash(NULL, "");
787 + add_object_entry(it->pcache.sha1, hash, 1);
788 + }
789 + else {
790 + struct tree_desc tree;
791 + tree.buf = it->pcache.tree_data;
792 + tree.size = it->pcache.tree_size;
793 + add_pbase_object(&tree, NULL, name, cmplen);
794 }
795 }
796 }
797
798 static void add_preferred_base(unsigned char *sha1)
799 {
630 - struct tree_desc tree;
631 - void *elem;
800 + struct pbase_tree *it;
801 + void *data;
802 + unsigned long size;
803 + unsigned char tree_sha1[20];
804
633 - elem = read_object_with_reference(sha1, tree_type, &tree.size, NULL);
634 - tree.buf = elem;
635 - if (!tree.buf)
805 + data = read_object_with_reference(sha1, tree_type, &size, tree_sha1);
806 + if (!data)
807 return;
637 - if (add_object_entry(sha1, name_hash(NULL, ""), 1))
638 - add_pbase_tree(&tree, NULL);
639 - free(elem);
808 +
809 + for (it = pbase_tree; it; it = it->next) {
810 + if (!memcmp(it->pcache.sha1, tree_sha1, 20)) {
811 + free(data);
812 + return;
813 + }
814 + }
815 +
816 + it = xcalloc(1, sizeof(*it));
817 + it->next = pbase_tree;
818 + pbase_tree = it;
819 +
820 + memcpy(it->pcache.sha1, tree_sha1, 20);
821 + it->pcache.tree_data = data;
822 + it->pcache.tree_size = size;
823 }
824
825 static void check_object(struct object_entry *entry)
@@ -1051,6 +1234,7 @@ int main(int argc, char **argv)
1234 char line[PATH_MAX + 20];
1235 int window = 10, depth = 10, pack_to_stdout = 0;
1236 struct object_entry **list;
1237 + int num_preferred_base = 0;
1238 int i;
1239
1240 setup_git_directory();
@@ -1116,6 +1300,7 @@ int main(int argc, char **argv)
1300
1301 for (;;) {
1302 unsigned char sha1[20];
1303 + unsigned hash;
1304
1305 if (!fgets(line, sizeof(line), stdin)) {
1306 if (feof(stdin))
@@ -1132,12 +1317,15 @@ int main(int argc, char **argv)
1317 if (get_sha1_hex(line+1, sha1))
1318 die("expected edge sha1, got garbage:\n %s",
1319 line+1);
1135 - add_preferred_base(sha1);
1320 + if (num_preferred_base++ < window)
1321 + add_preferred_base(sha1);
1322 continue;
1323 }
1324 if (get_sha1_hex(line, sha1))
1325 die("expected sha1, got garbage:\n %s", line);
1140 - add_object_entry(sha1, name_hash(NULL, line+41), 0);
1326 + hash = name_hash(NULL, line+41);
1327 + add_preferred_base_object(line+41, hash);
1328 + add_object_entry(sha1, hash, 0);
1329 }
1330 if (progress)
1331 fprintf(stderr, "Done counting %d objects.\n", nr_objects);