@matheusbsbs / matheus-bagliano / commits / 5bb86b82ba
Fork 0

diff-delta: bound hash list length to avoid O(m*n) behavior

The diff-delta code can exhibit O(m*n) behavior with some patological data set where most hash entries end up in the same hash bucket. The latest code rework reduced the block size making it particularly vulnerable to this issue, but the issue was always there and can be triggered regardless of the block size. This patch does two things: 1) the hashing has been reworked to offer a better distribution to atenuate the problem a bit, and 2) a limit is imposed to the number of entries that can exist in the same hash bucket. Because of the above the code is a bit more expensive on average, but the problematic samples used to diagnoze the issue are now orders of magnitude less expensive to process with only a slight loss in compression. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Nicolas Pitre committed Feb 27, 2006 at 23:09 UTC 5bb86b82ba18dd2eb736c4f5565f9c920f815b8f
1 file changed +56 -13
diff-delta.c
+56 -13
@@ -30,19 +30,20 @@ struct index {
30
31 static struct index ** delta_index(const unsigned char *buf,
32 unsigned long bufsize,
33 + unsigned long trg_bufsize,
34 unsigned int *hash_shift)
35 {
36 unsigned long hsize;
36 - unsigned int hshift, i;
37 + unsigned int i, hshift, hlimit, *hash_count;
38 const unsigned char *data;
39 struct index *entry, **hash;
40 void *mem;
41
42 /* determine index hash size */
43 hsize = bufsize / 4;
43 - for (i = 8; (1 << i) < hsize && i < 16; i++);
44 + for (i = 8; (1 << i) < hsize && i < 24; i += 2);
45 hsize = 1 << i;
45 - hshift = i - 8;
46 + hshift = (i - 8) / 2;
47 *hash_shift = hshift;
48
49 /* allocate lookup index */
@@ -53,15 +54,59 @@ static struct index ** delta_index(const unsigned char *buf,
54 entry = mem + hsize * sizeof(*hash);
55 memset(hash, 0, hsize * sizeof(*hash));
56
56 - /* then populate it */
57 + /* allocate an array to count hash entries */
58 + hash_count = calloc(hsize, sizeof(*hash_count));
59 + if (!hash_count) {
60 + free(hash);
61 + return NULL;
62 + }
63 +
64 + /* then populate the index */
65 data = buf + bufsize - 2;
66 while (data > buf) {
67 entry->ptr = --data;
60 - i = data[0] ^ data[1] ^ (data[2] << hshift);
68 + i = data[0] ^ ((data[1] ^ (data[2] << hshift)) << hshift);
69 entry->next = hash[i];
70 hash[i] = entry++;
71 + hash_count[i]++;
72 }
73
74 + /*
75 + * Determine a limit on the number of entries in the same hash
76 + * bucket. This guard us against patological data sets causing
77 + * really bad hash distribution with most entries in the same hash
78 + * bucket that would bring us to O(m*n) computing costs (m and n
79 + * corresponding to reference and target buffer sizes).
80 + *
81 + * The more the target buffer is large, the more it is important to
82 + * have small entry lists for each hash buckets. With such a limit
83 + * the cost is bounded to something more like O(m+n).
84 + */
85 + hlimit = (1 << 26) / trg_bufsize;
86 + if (hlimit < 16)
87 + hlimit = 16;
88 +
89 + /*
90 + * Now make sure none of the hash buckets has more entries than
91 + * we're willing to test. Otherwise we short-circuit the entry
92 + * list uniformly to still preserve a good repartition across
93 + * the reference buffer.
94 + */
95 + for (i = 0; i < hsize; i++) {
96 + if (hash_count[i] < hlimit)
97 + continue;
98 + entry = hash[i];
99 + do {
100 + struct index *keep = entry;
101 + int skip = hash_count[i] / hlimit / 2;
102 + do {
103 + entry = entry->next;
104 + } while(--skip && entry);
105 + keep->next = entry;
106 + } while(entry);
107 + }
108 + free(hash_count);
109 +
110 return hash;
111 }
112
@@ -85,7 +130,7 @@ void *diff_delta(void *from_buf, unsigned long from_size,
130
131 if (!from_size || !to_size)
132 return NULL;
88 - hash = delta_index(from_buf, from_size, &hash_shift);
133 + hash = delta_index(from_buf, from_size, to_size, &hash_shift);
134 if (!hash)
135 return NULL;
136
@@ -126,8 +171,8 @@ void *diff_delta(void *from_buf, unsigned long from_size,
171
172 while (data < top) {
173 unsigned int moff = 0, msize = 0;
129 - if (data + 2 < top) {
130 - i = data[0] ^ data[1] ^ (data[2] << hash_shift);
174 + if (data + 3 <= top) {
175 + i = data[0] ^ ((data[1] ^ (data[2] << hash_shift)) << hash_shift);
176 for (entry = hash[i]; entry; entry = entry->next) {
177 const unsigned char *ref = entry->ptr;
178 const unsigned char *src = data;
@@ -138,11 +183,9 @@ void *diff_delta(void *from_buf, unsigned long from_size,
183 ref_size = 0x10000;
184 if (ref_size <= msize)
185 break;
141 - while (ref_size && *src++ == *ref) {
142 - ref++;
143 - ref_size--;
144 - }
145 - ref_size = ref - entry->ptr;
186 + if (*ref != *src)
187 + continue;
188 + while (ref_size-- && *++src == *++ref);
189 if (msize < ref - entry->ptr) {
190 /* this is our best match so far */
191 msize = ref - entry->ptr;