@matheusbsbs / matheus-bagliano / commits / 06a9f92035
Fork 0

improve diff-delta with sparse and/or repetitive data

It is useless to preserve multiple hash entries for consecutive blocks with the same hash. Keeping only the first one will allow for matching the longest string of identical bytes while subsequent blocks will only allow for shorter matches. The backward matching code will match the end of it as necessary. This improves both performances (no repeated string compare with long successions of identical bytes, or even small group of bytes), as well as compression (less likely to need random hash bucket entry culling), especially with sparse files. With well behaved data sets this patch doesn't change much. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Nicolas Pitre committed May 2, 2006 at 23:31 UTC 06a9f9203570d21f9ef5fe219cdde527dcdf0990
1 file changed +27 -13
diff-delta.c
+27 -13
@@ -136,11 +136,12 @@ struct delta_index {
136
137 struct delta_index * create_delta_index(const void *buf, unsigned long bufsize)
138 {
139 - unsigned int i, hsize, hmask, entries, *hash_count;
139 + unsigned int i, hsize, hmask, entries, prev_val, *hash_count;
140 const unsigned char *data, *buffer = buf;
141 struct delta_index *index;
142 struct index_entry *entry, **hash;
143 void *mem;
144 + unsigned long memsize;
145
146 if (!buf || !bufsize)
147 return NULL;
@@ -155,9 +156,10 @@ struct delta_index * create_delta_index(const void *buf, unsigned long bufsize)
156 hmask = hsize - 1;
157
158 /* allocate lookup index */
158 - mem = malloc(sizeof(*index) +
159 - sizeof(*hash) * hsize +
160 - sizeof(*entry) * entries);
159 + memsize = sizeof(*index) +
160 + sizeof(*hash) * hsize +
161 + sizeof(*entry) * entries;
162 + mem = malloc(memsize);
163 if (!mem)
164 return NULL;
165 index = mem;
@@ -179,18 +181,26 @@ struct delta_index * create_delta_index(const void *buf, unsigned long bufsize)
181 }
182
183 /* then populate the index */
182 - data = buffer + entries * RABIN_WINDOW - RABIN_WINDOW;
183 - while (data >= buffer) {
184 + prev_val = ~0;
185 + for (data = buffer + entries * RABIN_WINDOW - RABIN_WINDOW;
186 + data >= buffer;
187 + data -= RABIN_WINDOW) {
188 unsigned int val = 0;
189 for (i = 1; i <= RABIN_WINDOW; i++)
190 val = ((val << 8) | data[i]) ^ T[val >> RABIN_SHIFT];
187 - i = val & hmask;
188 - entry->ptr = data + RABIN_WINDOW;
189 - entry->val = val;
190 - entry->next = hash[i];
191 - hash[i] = entry++;
192 - hash_count[i]++;
193 - data -= RABIN_WINDOW;
191 + if (val == prev_val) {
192 + /* keep the lowest of consecutive identical blocks */
193 + entry[-1].ptr = data + RABIN_WINDOW;
194 + } else {
195 + prev_val = val;
196 + i = val & hmask;
197 + entry->ptr = data + RABIN_WINDOW;
198 + entry->val = val;
199 + entry->next = hash[i];
200 + hash[i] = entry++;
201 + hash_count[i]++;
202 + entries--;
203 + }
204 }
205
206 /*
@@ -220,6 +230,10 @@ struct delta_index * create_delta_index(const void *buf, unsigned long bufsize)
230 }
231 free(hash_count);
232
233 + /* If we didn't use all hash entries, free the unused memory. */
234 + if (entries)
235 + index = realloc(index, memsize - entries * sizeof(*entry));
236 +
237 return index;
238 }
239