commit-reach: make can_all_from_reach... linear

The can_all_from_reach_with_flags() algorithm is currently quadratic in the worst case, because it calls the reachable() method for every 'from' without tracking which commits have already been walked or which can already reach a commit in 'to'. Rewrite the algorithm to walk each commit a constant number of times. We also add some optimizations that should work for the main consumer of this method: fetch negotitation (haves/wants). The first step includes using a depth-first-search (DFS) from each 'from' commit, sorted by ascending generation number. We do not walk beyond the minimum generation number or the minimum commit date. This DFS is likely to be faster than the existing reachable() method because we expect previous ref values to be along the first-parent history. If we find a target commit, then we mark everything in the DFS stack as a RESULT. This expands the set of targets for the other 'from' commits. We also mark the visited commits using 'assign_flag' to prevent re- walking the same commits. We still need to clear our flags at the end, which is why we will have a total of three visits to each commit. Performance was measured on the Linux repository using 'test-tool reach can_all_from_reach'. The input included rows seeded by tag values. The "small" case included X-rows as v4.[0-9]* and Y-rows as v3.[0-9]*. This mimics a (very large) fetch that says "I have all major v3 releases and want all major v4 releases." The "large" case included X-rows as "v4.*" and Y-rows as "v3.*". This adds all release-candidate tags to the set, which does not greatly increase the number of objects that are considered, but does increase the number of 'from' commits, demonstrating the quadratic nature of the previous code. Small Case: Before: 1.52 s After: 0.26 s Large Case: Before: 3.50 s After: 0.27 s Note how the time increases between the two cases in the two versions. The new code increases relative to the number of commits that need to be walked, but not directly relative to the number of 'from' commits. Signed-off-by: Derrick Stolee <dstolee@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Derrick Stolee committed Jul 20, 2018 at 16:33 UTC 4fbcca4effc1c6f8431120f88f5a4bd1c8e38ca3
3 files changed +83 -53
commit-reach.c
+75 -47
@@ -514,66 +514,87 @@ int commit_contains(struct ref_filter *filter, struct commit *commit,
514 return is_descendant_of(commit, list);
515 }
516
517 -int reachable(struct commit *from, unsigned int with_flag,
518 - unsigned int assign_flag, time_t min_commit_date)
517 +static int compare_commits_by_gen(const void *_a, const void *_b)
518 {
520 - struct prio_queue work = { compare_commits_by_commit_date };
519 + const struct commit *a = (const struct commit *)_a;
520 + const struct commit *b = (const struct commit *)_b;
521
522 - prio_queue_put(&work, from);
523 - while (work.nr) {
524 - struct commit_list *list;
525 - struct commit *commit = prio_queue_get(&work);
526 -
527 - if (commit->object.flags & with_flag) {
528 - from->object.flags |= assign_flag;
529 - break;
530 - }
531 - if (!commit->object.parsed)
532 - parse_object(the_repository, &commit->object.oid);
533 - if (commit->object.flags & REACHABLE)
534 - continue;
535 - commit->object.flags |= REACHABLE;
536 - if (commit->date < min_commit_date)
537 - continue;
538 - for (list = commit->parents; list; list = list->next) {
539 - struct commit *parent = list->item;
540 - if (!(parent->object.flags & REACHABLE))
541 - prio_queue_put(&work, parent);
542 - }
543 - }
544 - from->object.flags |= REACHABLE;
545 - clear_commit_marks(from, REACHABLE);
546 - clear_prio_queue(&work);
547 - return (from->object.flags & assign_flag);
522 + if (a->generation < b->generation)
523 + return -1;
524 + if (a->generation > b->generation)
525 + return 1;
526 + return 0;
527 }
528
529 int can_all_from_reach_with_flag(struct object_array *from,
530 unsigned int with_flag,
531 unsigned int assign_flag,
553 - time_t min_commit_date)
532 + time_t min_commit_date,
533 + uint32_t min_generation)
534 {
535 + struct commit **list = NULL;
536 int i;
537 + int result = 1;
538
539 + ALLOC_ARRAY(list, from->nr);
540 for (i = 0; i < from->nr; i++) {
558 - struct object *from_one = from->objects[i].item;
541 + list[i] = (struct commit *)from->objects[i].item;
542
560 - if (from_one->flags & assign_flag)
561 - continue;
562 - from_one = deref_tag(the_repository, from_one, "a from object", 0);
563 - if (!from_one || from_one->type != OBJ_COMMIT) {
564 - /* no way to tell if this is reachable by
565 - * looking at the ancestry chain alone, so
566 - * leave a note to ourselves not to worry about
567 - * this object anymore.
568 - */
569 - from->objects[i].item->flags |= assign_flag;
570 - continue;
571 - }
572 - if (!reachable((struct commit *)from_one, with_flag, assign_flag,
573 - min_commit_date))
543 + if (parse_commit(list[i]) ||
544 + list[i]->generation < min_generation)
545 return 0;
546 }
576 - return 1;
547 +
548 + QSORT(list, from->nr, compare_commits_by_gen);
549 +
550 + for (i = 0; i < from->nr; i++) {
551 + /* DFS from list[i] */
552 + struct commit_list *stack = NULL;
553 +
554 + list[i]->object.flags |= assign_flag;
555 + commit_list_insert(list[i], &stack);
556 +
557 + while (stack) {
558 + struct commit_list *parent;
559 +
560 + if (stack->item->object.flags & with_flag) {
561 + pop_commit(&stack);
562 + continue;
563 + }
564 +
565 + for (parent = stack->item->parents; parent; parent = parent->next) {
566 + if (parent->item->object.flags & (with_flag | RESULT))
567 + stack->item->object.flags |= RESULT;
568 +
569 + if (!(parent->item->object.flags & assign_flag)) {
570 + parent->item->object.flags |= assign_flag;
571 +
572 + if (parse_commit(parent->item) ||
573 + parent->item->date < min_commit_date ||
574 + parent->item->generation < min_generation)
575 + continue;
576 +
577 + commit_list_insert(parent->item, &stack);
578 + break;
579 + }
580 + }
581 +
582 + if (!parent)
583 + pop_commit(&stack);
584 + }
585 +
586 + if (!(list[i]->object.flags & (with_flag | RESULT))) {
587 + result = 0;
588 + goto cleanup;
589 + }
590 + }
591 +
592 +cleanup:
593 + for (i = 0; i < from->nr; i++) {
594 + clear_commit_marks(list[i], RESULT);
595 + clear_commit_marks(list[i], assign_flag);
596 + }
597 + return result;
598 }
599
600 int can_all_from_reach(struct commit_list *from, struct commit_list *to,
@@ -583,6 +604,7 @@ int can_all_from_reach(struct commit_list *from, struct commit_list *to,
604 time_t min_commit_date = cutoff_by_min_date ? from->item->date : 0;
605 struct commit_list *from_iter = from, *to_iter = to;
606 int result;
607 + uint32_t min_generation = GENERATION_NUMBER_INFINITY;
608
609 while (from_iter) {
610 add_object_array(&from_iter->item->object, NULL, &from_objs);
@@ -590,6 +612,9 @@ int can_all_from_reach(struct commit_list *from, struct commit_list *to,
612 if (!parse_commit(from_iter->item)) {
613 if (from_iter->item->date < min_commit_date)
614 min_commit_date = from_iter->item->date;
615 +
616 + if (from_iter->item->generation < min_generation)
617 + min_generation = from_iter->item->generation;
618 }
619
620 from_iter = from_iter->next;
@@ -599,6 +624,9 @@ int can_all_from_reach(struct commit_list *from, struct commit_list *to,
624 if (!parse_commit(to_iter->item)) {
625 if (to_iter->item->date < min_commit_date)
626 min_commit_date = to_iter->item->date;
627 +
628 + if (to_iter->item->generation < min_generation)
629 + min_generation = to_iter->item->generation;
630 }
631
632 to_iter->item->object.flags |= PARENT2;
@@ -607,7 +635,7 @@ int can_all_from_reach(struct commit_list *from, struct commit_list *to,
635 }
636
637 result = can_all_from_reach_with_flag(&from_objs, PARENT2, PARENT1,
610 - min_commit_date);
638 + min_commit_date, min_generation);
639
640 while (from) {
641 clear_commit_marks(from->item, PARENT1);
commit-reach.h
+4 -5
@@ -59,19 +59,18 @@ define_commit_slab(contains_cache, enum contains_result);
59 int commit_contains(struct ref_filter *filter, struct commit *commit,
60 struct commit_list *list, struct contains_cache *cache);
61
62 -int reachable(struct commit *from, unsigned int with_flag,
63 - unsigned int assign_flag, time_t min_commit_date);
64 -
62 /*
63 * Determine if every commit in 'from' can reach at least one commit
64 * that is marked with 'with_flag'. As we traverse, use 'assign_flag'
65 * as a marker for commits that are already visited. Do not walk
69 - * commits with date below 'min_commit_date'.
66 + * commits with date below 'min_commit_date' or generation below
67 + * 'min_generation'.
68 */
69 int can_all_from_reach_with_flag(struct object_array *from,
70 unsigned int with_flag,
71 unsigned int assign_flag,
74 - time_t min_commit_date);
72 + time_t min_commit_date,
73 + uint32_t min_generation);
74 int can_all_from_reach(struct commit_list *from, struct commit_list *to,
75 int commit_date_cutoff);
76
upload-pack.c
+4 -1
@@ -338,11 +338,14 @@ static int got_oid(const char *hex, struct object_id *oid)
338
339 static int ok_to_give_up(void)
340 {
341 + uint32_t min_generation = GENERATION_NUMBER_ZERO;
342 +
343 if (!have_obj.nr)
344 return 0;
345
346 return can_all_from_reach_with_flag(&want_obj, THEY_HAVE,
345 - COMMON_KNOWN, oldest_have);
347 + COMMON_KNOWN, oldest_have,
348 + min_generation);
349 }
350
351 static int get_common_commits(void)