@matheusbsbs / matheus-bagliano / commits / 6d24ad971c
Fork 0

Optimize rename detection for a huge diff

When there are N deleted paths and M created paths, we used to allocate (N x M) "struct diff_score" that record how similar each of the pair is, and picked the <src,dst> pair that gives the best match first, and then went on to process worse matches. This sorting is done so that when two new files in the postimage that are similar to the same file deleted from the preimage, we can process the more similar one first, and when processing the second one, it can notice "Ah, the source I was planning to say I am a copy of is already taken by somebody else" and continue on to match itself with another file in the preimage with a lessor match. This matters to a change introduced between 1.5.3.X series and 1.5.4-rc, that lets the code to favor unused matches first and then falls back to using already used matches. This instead allocates and keeps only a handful rename source candidates per new files in the postimage. I.e. it makes the memory requirement from O(N x M) to O(M). For each dst, we compute similarlity with all sources (i.e. the number of similarity estimate computations is still O(N x M)), but we keep handful best src candidates for each dst. Signed-off-by: Junio C Hamano <gitster@pobox.com>

Junio C Hamano committed Jan 29, 2008 at 20:54 UTC 6d24ad971c8195b00cd9678fbff7c2aaddb00908
1 file changed +58 -22
diffcore-rename.c
+58 -22
@@ -112,8 +112,8 @@ static int basename_same(struct diff_filespec *src, struct diff_filespec *dst)
112 struct diff_score {
113 int src; /* index in rename_src */
114 int dst; /* index in rename_dst */
115 - int score;
116 - int name_score;
115 + unsigned short score;
116 + short name_score;
117 };
118
119 static int estimate_similarity(struct diff_filespec *src,
@@ -223,6 +223,12 @@ static int score_compare(const void *a_, const void *b_)
223 {
224 const struct diff_score *a = a_, *b = b_;
225
226 + /* sink the unused ones to the bottom */
227 + if (a->dst < 0)
228 + return (0 <= b->dst);
229 + else if (b->dst < 0)
230 + return -1;
231 +
232 if (a->score == b->score)
233 return b->name_score - a->name_score;
234
@@ -387,6 +393,22 @@ static int find_exact_renames(void)
393 return i;
394 }
395
396 +#define NUM_CANDIDATE_PER_DST 4
397 +static void record_if_better(struct diff_score m[], struct diff_score *o)
398 +{
399 + int i, worst;
400 +
401 + /* find the worst one */
402 + worst = 0;
403 + for (i = 1; i < NUM_CANDIDATE_PER_DST; i++)
404 + if (score_compare(&m[i], &m[worst]) > 0)
405 + worst = i;
406 +
407 + /* is it better than the worst one? */
408 + if (score_compare(&m[worst], o) > 0)
409 + m[worst] = *o;
410 +}
411 +
412 void diffcore_rename(struct diff_options *options)
413 {
414 int detect_rename = options->detect_rename;
@@ -473,47 +495,61 @@ void diffcore_rename(struct diff_options *options)
495 if (num_create * num_src > rename_limit * rename_limit)
496 goto cleanup;
497
476 - mx = xmalloc(sizeof(*mx) * num_create * num_src);
498 + mx = xcalloc(num_create * NUM_CANDIDATE_PER_DST, sizeof(*mx));
499 for (dst_cnt = i = 0; i < rename_dst_nr; i++) {
478 - int base = dst_cnt * num_src;
500 struct diff_filespec *two = rename_dst[i].two;
501 + struct diff_score *m;
502 +
503 if (rename_dst[i].pair)
504 continue; /* dealt with exact match already. */
505 +
506 + m = &mx[dst_cnt * NUM_CANDIDATE_PER_DST];
507 + for (j = 0; j < NUM_CANDIDATE_PER_DST; j++)
508 + m[j].dst = -1;
509 +
510 for (j = 0; j < rename_src_nr; j++) {
511 struct diff_filespec *one = rename_src[j].one;
484 - struct diff_score *m = &mx[base+j];
485 - m->src = j;
486 - m->dst = i;
487 - m->score = estimate_similarity(one, two,
488 - minimum_score);
489 - m->name_score = basename_same(one, two);
512 + struct diff_score this_src;
513 + this_src.score = estimate_similarity(one, two,
514 + minimum_score);
515 + this_src.name_score = basename_same(one, two);
516 + this_src.dst = i;
517 + this_src.src = j;
518 + record_if_better(m, &this_src);
519 diff_free_filespec_blob(one);
520 }
521 /* We do not need the text anymore */
522 diff_free_filespec_blob(two);
523 dst_cnt++;
524 }
525 +
526 /* cost matrix sorted by most to least similar pair */
497 - qsort(mx, num_create * num_src, sizeof(*mx), score_compare);
498 - for (i = 0; i < num_create * num_src; i++) {
499 - struct diff_rename_dst *dst = &rename_dst[mx[i].dst];
500 - struct diff_filespec *src;
527 + qsort(mx, dst_cnt * NUM_CANDIDATE_PER_DST, sizeof(*mx), score_compare);
528 +
529 + for (i = 0; i < dst_cnt * NUM_CANDIDATE_PER_DST; i++) {
530 + struct diff_rename_dst *dst;
531 +
532 + if ((mx[i].dst < 0) ||
533 + (mx[i].score < minimum_score))
534 + break; /* there is no more usable pair. */
535 + dst = &rename_dst[mx[i].dst];
536 if (dst->pair)
537 continue; /* already done, either exact or fuzzy. */
503 - if (mx[i].score < minimum_score)
504 - break; /* there is no more usable pair. */
505 - src = rename_src[mx[i].src].one;
506 - if (src->rename_used)
538 + if (rename_src[mx[i].src].one->rename_used)
539 continue;
540 record_rename_pair(mx[i].dst, mx[i].src, mx[i].score);
541 rename_count++;
542 }
511 - for (i = 0; i < num_create * num_src; i++) {
512 - struct diff_rename_dst *dst = &rename_dst[mx[i].dst];
543 +
544 + for (i = 0; i < dst_cnt * NUM_CANDIDATE_PER_DST; i++) {
545 + struct diff_rename_dst *dst;
546 +
547 + if ((mx[i].dst < 0) ||
548 + (mx[i].score < minimum_score))
549 + break; /* there is no more usable pair. */
550 + dst = &rename_dst[mx[i].dst];
551 if (dst->pair)
552 continue; /* already done, either exact or fuzzy. */
515 - if (mx[i].score < minimum_score)
516 - break; /* there is no more usable pair. */
553 record_rename_pair(mx[i].dst, mx[i].src, mx[i].score);
554 rename_count++;
555 }