improve diff-delta with sparse and/or repetitive data
It is useless to preserve multiple hash entries for consecutive blocks with the same hash. Keeping only the first one will allow for matching the longest string of identical bytes while subsequent blocks will only allow for shorter matches. The backward matching code will match the end of it as necessary. This improves both performances (no repeated string compare with long successions of identical bytes, or even small group of bytes), as well as compression (less likely to need random hash bucket entry culling), especially with sparse files. With well behaved data sets this patch doesn't change much. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
Nicolas Pitre committed
May 2, 2006 at 23:31 UTC
06a9f9203570d21f9ef5fe219cdde527dcdf0990
1 file changed
+27
-13
diff-delta.c
+27
-13
@@ -136,11 +136,12 @@ struct delta_index {
136
137
struct delta_index * create_delta_index(const void *buf, unsigned long bufsize)
138
{
139
- unsigned int i, hsize, hmask, entries, *hash_count;
139
+ unsigned int i, hsize, hmask, entries, prev_val, *hash_count;
140
const unsigned char *data, *buffer = buf;
141
struct delta_index *index;
142
struct index_entry *entry, **hash;
143
void *mem;
144
+ unsigned long memsize;
145
146
if (!buf || !bufsize)
147
return NULL;
@@ -155,9 +156,10 @@ struct delta_index * create_delta_index(const void *buf, unsigned long bufsize)
156
hmask = hsize - 1;
157
158
/* allocate lookup index */
158
- mem = malloc(sizeof(*index) +
159
- sizeof(*hash) * hsize +
160
- sizeof(*entry) * entries);
159
+ memsize = sizeof(*index) +
160
+ sizeof(*hash) * hsize +
161
+ sizeof(*entry) * entries;
162
+ mem = malloc(memsize);
163
if (!mem)
164
return NULL;
165
index = mem;
@@ -179,18 +181,26 @@ struct delta_index * create_delta_index(const void *buf, unsigned long bufsize)
181
}
182
183
/* then populate the index */
182
- data = buffer + entries * RABIN_WINDOW - RABIN_WINDOW;
183
- while (data >= buffer) {
184
+ prev_val = ~0;
185
+ for (data = buffer + entries * RABIN_WINDOW - RABIN_WINDOW;
186
+ data >= buffer;
187
+ data -= RABIN_WINDOW) {
188
unsigned int val = 0;
189
for (i = 1; i <= RABIN_WINDOW; i++)
190
val = ((val << 8) | data[i]) ^ T[val >> RABIN_SHIFT];
187
- i = val & hmask;
188
- entry->ptr = data + RABIN_WINDOW;
189
- entry->val = val;
190
- entry->next = hash[i];
191
- hash[i] = entry++;
192
- hash_count[i]++;
193
- data -= RABIN_WINDOW;
191
+ if (val == prev_val) {
192
+ /* keep the lowest of consecutive identical blocks */
193
+ entry[-1].ptr = data + RABIN_WINDOW;
194
+ } else {
195
+ prev_val = val;
196
+ i = val & hmask;
197
+ entry->ptr = data + RABIN_WINDOW;
198
+ entry->val = val;
199
+ entry->next = hash[i];
200
+ hash[i] = entry++;
201
+ hash_count[i]++;
202
+ entries--;
203
+ }
204
}
205
206
/*
@@ -220,6 +230,10 @@ struct delta_index * create_delta_index(const void *buf, unsigned long bufsize)
230
}
231
free(hash_count);
232
233
+ /* If we didn't use all hash entries, free the unused memory. */
234
+ if (entries)
235
+ index = realloc(index, memsize - entries * sizeof(*entry));
236
+
237
return index;
238
}
239