@matheusbsbs / matheus-bagliano / commits / c13c6bf758
Fork 0

diff-delta: bound hash list length to avoid O(m*n) behavior

The diff-delta code can exhibit O(m*n) behavior with some patological data set where most hash entries end up in the same hash bucket. To prevent this, a limit is imposed to the number of entries that can exist in the same hash bucket. Because of the above the code is a tiny bit more expensive on average, even if some small optimizations were added as well to atenuate the overhead. But the problematic samples used to diagnoze the issue are now orders of magnitude less expensive to process with only a slight loss in compression. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Nicolas Pitre committed Mar 8, 2006 at 14:32 UTC c13c6bf758457a3e7293b2adf63cc47aec8d83ef
1 file changed +71 -30
diff-delta.c
+71 -30
@@ -40,17 +40,18 @@ struct index {
40
41 static struct index ** delta_index(const unsigned char *buf,
42 unsigned long bufsize,
43 + unsigned long trg_bufsize,
44 unsigned int *hash_shift)
45 {
45 - unsigned int hsize, hshift, entries, blksize, i;
46 + unsigned int i, hsize, hshift, hlimit, entries, *hash_count;
47 const unsigned char *data;
48 struct index *entry, **hash;
49 void *mem;
50
51 /* determine index hash size */
51 - entries = (bufsize + BLK_SIZE - 1) / BLK_SIZE;
52 + entries = bufsize / BLK_SIZE;
53 hsize = entries / 4;
53 - for (i = 4; (1 << i) < hsize && i < 16; i++);
54 + for (i = 4; (1 << i) < hsize && i < 31; i++);
55 hsize = 1 << i;
56 hshift = 32 - i;
57 *hash_shift = hshift;
@@ -63,20 +64,62 @@ static struct index ** delta_index(const unsigned char *buf,
64 entry = mem + hsize * sizeof(*hash);
65 memset(hash, 0, hsize * sizeof(*hash));
66
66 - /* then populate it */
67 + /* allocate an array to count hash entries */
68 + hash_count = calloc(hsize, sizeof(*hash_count));
69 + if (!hash_count) {
70 + free(hash);
71 + return NULL;
72 + }
73 +
74 + /* then populate the index */
75 data = buf + entries * BLK_SIZE - BLK_SIZE;
68 - blksize = bufsize - (data - buf);
76 while (data >= buf) {
70 - unsigned int val = adler32(0, data, blksize);
77 + unsigned int val = adler32(0, data, BLK_SIZE);
78 i = HASH(val, hshift);
79 entry->ptr = data;
80 entry->val = val;
81 entry->next = hash[i];
82 hash[i] = entry++;
76 - blksize = BLK_SIZE;
83 + hash_count[i]++;
84 data -= BLK_SIZE;
85 }
86
87 + /*
88 + * Determine a limit on the number of entries in the same hash
89 + * bucket. This guard us against patological data sets causing
90 + * really bad hash distribution with most entries in the same hash
91 + * bucket that would bring us to O(m*n) computing costs (m and n
92 + * corresponding to reference and target buffer sizes).
93 + *
94 + * The more the target buffer is large, the more it is important to
95 + * have small entry lists for each hash buckets. With such a limit
96 + * the cost is bounded to something more like O(m+n).
97 + */
98 + hlimit = (1 << 26) / trg_bufsize;
99 + if (hlimit < 4*BLK_SIZE)
100 + hlimit = 4*BLK_SIZE;
101 +
102 + /*
103 + * Now make sure none of the hash buckets has more entries than
104 + * we're willing to test. Otherwise we cull the entry list
105 + * uniformly to still preserve a good repartition across
106 + * the reference buffer.
107 + */
108 + for (i = 0; i < hsize; i++) {
109 + if (hash_count[i] < hlimit)
110 + continue;
111 + entry = hash[i];
112 + do {
113 + struct index *keep = entry;
114 + int skip = hash_count[i] / hlimit / 2;
115 + do {
116 + entry = entry->next;
117 + } while(--skip && entry);
118 + keep->next = entry;
119 + } while(entry);
120 + }
121 + free(hash_count);
122 +
123 return hash;
124 }
125
@@ -100,7 +143,7 @@ void *diff_delta(void *from_buf, unsigned long from_size,
143
144 if (!from_size || !to_size)
145 return NULL;
103 - hash = delta_index(from_buf, from_size, &hash_shift);
146 + hash = delta_index(from_buf, from_size, to_size, &hash_shift);
147 if (!hash)
148 return NULL;
149
@@ -141,29 +184,27 @@ void *diff_delta(void *from_buf, unsigned long from_size,
184
185 while (data < top) {
186 unsigned int moff = 0, msize = 0;
144 - unsigned int blksize = MIN(top - data, BLK_SIZE);
145 - unsigned int val = adler32(0, data, blksize);
146 - i = HASH(val, hash_shift);
147 - for (entry = hash[i]; entry; entry = entry->next) {
148 - const unsigned char *ref = entry->ptr;
149 - const unsigned char *src = data;
150 - unsigned int ref_size = ref_top - ref;
151 - if (entry->val != val)
152 - continue;
153 - if (ref_size > top - src)
154 - ref_size = top - src;
155 - while (ref_size && *src++ == *ref) {
156 - ref++;
157 - ref_size--;
158 - }
159 - ref_size = ref - entry->ptr;
160 - if (ref_size > msize) {
161 - /* this is our best match so far */
162 - moff = entry->ptr - ref_data;
163 - msize = ref_size;
164 - if (msize >= 0x10000) {
165 - msize = 0x10000;
187 + if (data + BLK_SIZE <= top) {
188 + unsigned int val = adler32(0, data, BLK_SIZE);
189 + i = HASH(val, hash_shift);
190 + for (entry = hash[i]; entry; entry = entry->next) {
191 + const unsigned char *ref = entry->ptr;
192 + const unsigned char *src = data;
193 + unsigned int ref_size = ref_top - ref;
194 + if (entry->val != val)
195 + continue;
196 + if (ref_size > top - src)
197 + ref_size = top - src;
198 + if (ref_size > 0x10000)
199 + ref_size = 0x10000;
200 + if (ref_size <= msize)
201 break;
202 + while (ref_size-- && *src++ == *ref)
203 + ref++;
204 + if (msize < ref - entry->ptr) {
205 + /* this is our best match so far */
206 + msize = ref - entry->ptr;
207 + moff = entry->ptr - ref_data;
208 }
209 }
210 }