@matheusbsbs / matheus-bagliano / commits / c06c79667c
Fork 0

diffcore-rename: somewhat optimized.

This changes diffcore-rename to reuse statistics information gathered during similarity estimation, and updates the hashtable implementation used to keep track of the statistics to be denser. This seems to give better performance. Signed-off-by: Junio C Hamano <junkio@cox.net>

Junio C Hamano committed Mar 12, 2006 at 03:22 UTC c06c79667c9514aed00d29bcd80bd0cee7cc5a25
5 files changed +149 -21
diff.c
+2
@@ -463,6 +463,8 @@ void diff_free_filespec_data(struct diff_filespec *s)
463 munmap(s->data, s->size);
464 s->should_free = s->should_munmap = 0;
465 s->data = NULL;
466 + free(s->cnt_data);
467 + s->cnt_data = NULL;
468 }
469
470 static void prep_temp_blob(struct diff_tempfile *temp,
diffcore-break.c
+1
@@ -68,6 +68,7 @@ static int should_break(struct diff_filespec *src,
68
69 if (diffcore_count_changes(src->data, src->size,
70 dst->data, dst->size,
71 + NULL, NULL,
72 0,
73 &src_copied, &literal_added))
74 return 0;
diffcore-delta.c
+140 -21
@@ -24,11 +24,106 @@
24 * The length of the sequence is arbitrarily set to 8 for now.
25 */
26
27 +/* Wild guess at the initial hash size */
28 +#define INITIAL_HASH_SIZE 10
29 #define HASHBASE 65537 /* next_prime(2^16) */
30
29 -static void hash_chars(unsigned char *buf, unsigned long sz, int *count)
31 +struct spanhash {
32 + unsigned long hashval;
33 + unsigned long cnt;
34 +};
35 +struct spanhash_top {
36 + int alloc_log2;
37 + int free;
38 + struct spanhash data[FLEX_ARRAY];
39 +};
40 +
41 +static struct spanhash *spanhash_find(struct spanhash_top *top, unsigned long hashval)
42 +{
43 + int sz = 1 << top->alloc_log2;
44 + int bucket = hashval & (sz - 1);
45 + while (1) {
46 + struct spanhash *h = &(top->data[bucket++]);
47 + if (!h->cnt)
48 + return NULL;
49 + if (h->hashval == hashval)
50 + return h;
51 + if (sz <= bucket)
52 + bucket = 0;
53 + }
54 +}
55 +
56 +static struct spanhash_top *spanhash_rehash(struct spanhash_top *orig)
57 +{
58 + struct spanhash_top *new;
59 + int i;
60 + int osz = 1 << orig->alloc_log2;
61 + int sz = osz << 1;
62 +
63 + new = xmalloc(sizeof(*orig) + sizeof(struct spanhash) * sz);
64 + new->alloc_log2 = orig->alloc_log2 + 1;
65 + new->free = osz;
66 + memset(new->data, 0, sizeof(struct spanhash) * sz);
67 + for (i = 0; i < osz; i++) {
68 + struct spanhash *o = &(orig->data[i]);
69 + int bucket;
70 + if (!o->cnt)
71 + continue;
72 + bucket = o->hashval & (sz - 1);
73 + while (1) {
74 + struct spanhash *h = &(new->data[bucket++]);
75 + if (!h->cnt) {
76 + h->hashval = o->hashval;
77 + h->cnt = o->cnt;
78 + new->free--;
79 + break;
80 + }
81 + if (sz <= bucket)
82 + bucket = 0;
83 + }
84 + }
85 + free(orig);
86 + return new;
87 +}
88 +
89 +static struct spanhash_top *add_spanhash(struct spanhash_top *top,
90 + unsigned long hashval)
91 +{
92 + int bucket, lim;
93 + struct spanhash *h;
94 +
95 + lim = (1 << top->alloc_log2);
96 + bucket = hashval & (lim - 1);
97 + while (1) {
98 + h = &(top->data[bucket++]);
99 + if (!h->cnt) {
100 + h->hashval = hashval;
101 + h->cnt = 1;
102 + top->free--;
103 + if (top->free < 0)
104 + return spanhash_rehash(top);
105 + return top;
106 + }
107 + if (h->hashval == hashval) {
108 + h->cnt++;
109 + return top;
110 + }
111 + if (lim <= bucket)
112 + bucket = 0;
113 + }
114 +}
115 +
116 +static struct spanhash_top *hash_chars(unsigned char *buf, unsigned long sz)
117 {
31 - unsigned int accum1, accum2, i;
118 + int i;
119 + unsigned long accum1, accum2, hashval;
120 + struct spanhash_top *hash;
121 +
122 + i = INITIAL_HASH_SIZE;
123 + hash = xmalloc(sizeof(*hash) + sizeof(struct spanhash) * (1<<i));
124 + hash->alloc_log2 = i;
125 + hash->free = (1<<i)/2;
126 + memset(hash->data, 0, sizeof(struct spanhash) * (1<<i));
127
128 /* an 8-byte shift register made of accum1 and accum2. New
129 * bytes come at LSB of accum2, and shifted up to accum1
@@ -40,44 +135,68 @@ static void hash_chars(unsigned char *buf, unsigned long sz, int *count)
135 while (sz) {
136 accum1 = (accum1 << 8) | (accum2 >> 24);
137 accum2 = (accum2 << 8) | *buf++;
43 - /* We want something that hashes permuted byte
44 - * sequences nicely; simpler hash like (accum1 ^
45 - * accum2) does not perform as well.
46 - */
47 - i = (accum1 + accum2 * 0x61) % HASHBASE;
48 - count[i]++;
138 + hashval = (accum1 + accum2 * 0x61) % HASHBASE;
139 + hash = add_spanhash(hash, hashval);
140 sz--;
141 }
142 + return hash;
143 }
144
145 int diffcore_count_changes(void *src, unsigned long src_size,
146 void *dst, unsigned long dst_size,
147 + void **src_count_p,
148 + void **dst_count_p,
149 unsigned long delta_limit,
150 unsigned long *src_copied,
151 unsigned long *literal_added)
152 {
59 - int *src_count, *dst_count, i;
153 + int i, ssz;
154 + struct spanhash_top *src_count, *dst_count;
155 unsigned long sc, la;
156
157 if (src_size < 8 || dst_size < 8)
158 return -1;
159
65 - src_count = xcalloc(HASHBASE * 2, sizeof(int));
66 - dst_count = src_count + HASHBASE;
67 - hash_chars(src, src_size, src_count);
68 - hash_chars(dst, dst_size, dst_count);
69 -
160 + src_count = dst_count = NULL;
161 + if (src_count_p)
162 + src_count = *src_count_p;
163 + if (!src_count) {
164 + src_count = hash_chars(src, src_size);
165 + if (src_count_p)
166 + *src_count_p = src_count;
167 + }
168 + if (dst_count_p)
169 + dst_count = *dst_count_p;
170 + if (!dst_count) {
171 + dst_count = hash_chars(dst, dst_size);
172 + if (dst_count_p)
173 + *dst_count_p = dst_count;
174 + }
175 sc = la = 0;
71 - for (i = 0; i < HASHBASE; i++) {
72 - if (src_count[i] < dst_count[i]) {
73 - la += dst_count[i] - src_count[i];
74 - sc += src_count[i];
176 +
177 + ssz = 1 << src_count->alloc_log2;
178 + for (i = 0; i < ssz; i++) {
179 + struct spanhash *s = &(src_count->data[i]);
180 + struct spanhash *d;
181 + unsigned dst_cnt, src_cnt;
182 + if (!s->cnt)
183 + continue;
184 + src_cnt = s->cnt;
185 + d = spanhash_find(dst_count, s->hashval);
186 + dst_cnt = d ? d->cnt : 0;
187 + if (src_cnt < dst_cnt) {
188 + la += dst_cnt - src_cnt;
189 + sc += src_cnt;
190 }
76 - else /* i.e. if (dst_count[i] <= src_count[i]) */
77 - sc += dst_count[i];
191 + else
192 + sc += dst_cnt;
193 }
194 +
195 + if (!src_count_p)
196 + free(src_count);
197 + if (!dst_count_p)
198 + free(dst_count);
199 *src_copied = sc;
200 *literal_added = la;
81 - free(src_count);
201 return 0;
202 }
diffcore-rename.c
+3
@@ -166,6 +166,7 @@ static int estimate_similarity(struct diff_filespec *src,
166 delta_limit = base_size * (MAX_SCORE-minimum_score) / MAX_SCORE;
167 if (diffcore_count_changes(src->data, src->size,
168 dst->data, dst->size,
169 + &src->cnt_data, &dst->cnt_data,
170 delta_limit,
171 &src_copied, &literal_added))
172 return 0;
@@ -306,6 +307,8 @@ void diffcore_rename(struct diff_options *options)
307 m->score = estimate_similarity(one, two,
308 minimum_score);
309 }
310 + free(two->cnt_data);
311 + two->cnt_data = NULL;
312 dst_cnt++;
313 }
314 /* cost matrix sorted by most to least similar pair */
diffcore.h
+3
@@ -26,6 +26,7 @@ struct diff_filespec {
26 unsigned char sha1[20];
27 char *path;
28 void *data;
29 + void *cnt_data;
30 unsigned long size;
31 int xfrm_flags; /* for use by the xfrm */
32 unsigned short mode; /* file mode */
@@ -103,6 +104,8 @@ void diff_debug_queue(const char *, struct diff_queue_struct *);
104
105 extern int diffcore_count_changes(void *src, unsigned long src_size,
106 void *dst, unsigned long dst_size,
107 + void **src_count_p,
108 + void **dst_count_p,
109 unsigned long delta_limit,
110 unsigned long *src_copied,
111 unsigned long *literal_added);