@matheusbsbs / matheus-bagliano / commits / d2100860fd
Fork 0

diff-delta.c: pack the index structure

In normal use cases, the performance wins are not overly impressive: we get something like 5-10% due to the slightly better locality of memory accesses using the packed structure. However, since the data structure for index entries saves 33% of memory on 32-bit platforms and 40% on 64-bit platforms, the behavior when memory gets limited should be nicer. This is a rather well-contained change. One obvious improvement would be sorting the elements in one bucket according to their hash, then using binary probing to find the elements with the right hash value. As it stands, the output should be strictly the same as previously unless one uses the option for limiting the amount of used memory, in which case the created packs might be better. Signed-off-by: David Kastrup <dak@gnu.org> Acked-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>

David Kastrup committed Sep 8, 2007 at 23:17 UTC d2100860fd67dec6474157697888caaa0a0f51d0
1 file changed +58 -16
diff-delta.c
+58 -16
@@ -115,7 +115,11 @@ static const unsigned int U[256] = {
115 struct index_entry {
116 const unsigned char *ptr;
117 unsigned int val;
118 - struct index_entry *next;
118 +};
119 +
120 +struct unpacked_index_entry {
121 + struct index_entry entry;
122 + struct unpacked_index_entry *next;
123 };
124
125 struct delta_index {
@@ -131,7 +135,8 @@ struct delta_index * create_delta_index(const void *buf, unsigned long bufsize)
135 unsigned int i, hsize, hmask, entries, prev_val, *hash_count;
136 const unsigned char *data, *buffer = buf;
137 struct delta_index *index;
134 - struct index_entry *entry, **hash;
138 + struct unpacked_index_entry *entry, **hash;
139 + struct index_entry *packed_entry, **packed_hash;
140 void *mem;
141 unsigned long memsize;
142
@@ -148,28 +153,21 @@ struct delta_index * create_delta_index(const void *buf, unsigned long bufsize)
153 hmask = hsize - 1;
154
155 /* allocate lookup index */
151 - memsize = sizeof(*index) +
152 - sizeof(*hash) * hsize +
156 + memsize = sizeof(*hash) * hsize +
157 sizeof(*entry) * entries;
158 mem = malloc(memsize);
159 if (!mem)
160 return NULL;
157 - index = mem;
158 - mem = index + 1;
161 hash = mem;
162 mem = hash + hsize;
163 entry = mem;
164
163 - index->memsize = memsize;
164 - index->src_buf = buf;
165 - index->src_size = bufsize;
166 - index->hash_mask = hmask;
165 memset(hash, 0, hsize * sizeof(*hash));
166
167 /* allocate an array to count hash entries */
168 hash_count = calloc(hsize, sizeof(*hash_count));
169 if (!hash_count) {
172 - free(index);
170 + free(hash);
171 return NULL;
172 }
173
@@ -183,12 +181,13 @@ struct delta_index * create_delta_index(const void *buf, unsigned long bufsize)
181 val = ((val << 8) | data[i]) ^ T[val >> RABIN_SHIFT];
182 if (val == prev_val) {
183 /* keep the lowest of consecutive identical blocks */
186 - entry[-1].ptr = data + RABIN_WINDOW;
184 + entry[-1].entry.ptr = data + RABIN_WINDOW;
185 + --entries;
186 } else {
187 prev_val = val;
188 i = val & hmask;
190 - entry->ptr = data + RABIN_WINDOW;
191 - entry->val = val;
189 + entry->entry.ptr = data + RABIN_WINDOW;
190 + entry->entry.val = val;
191 entry->next = hash[i];
192 hash[i] = entry++;
193 hash_count[i]++;
@@ -212,16 +211,59 @@ struct delta_index * create_delta_index(const void *buf, unsigned long bufsize)
211 continue;
212 entry = hash[i];
213 do {
215 - struct index_entry *keep = entry;
214 + struct unpacked_index_entry *keep = entry;
215 int skip = hash_count[i] / HASH_LIMIT;
216 do {
217 + --entries;
218 entry = entry->next;
219 } while(--skip && entry);
220 + ++entries;
221 keep->next = entry;
222 } while(entry);
223 }
224 free(hash_count);
225
226 + /* Now create the packed index in array form rather than
227 + * linked lists */
228 +
229 + memsize = sizeof(*index)
230 + + sizeof(*packed_hash) * (hsize+1)
231 + + sizeof(*packed_entry) * entries;
232 +
233 + mem = malloc(memsize);
234 +
235 + if (!mem) {
236 + free(hash);
237 + return NULL;
238 + }
239 +
240 + index = mem;
241 + index->memsize = memsize;
242 + index->src_buf = buf;
243 + index->src_size = bufsize;
244 + index->hash_mask = hmask;
245 +
246 + mem = index + 1;
247 + packed_hash = mem;
248 + mem = packed_hash + (hsize+1);
249 + packed_entry = mem;
250 +
251 + /* Coalesce all entries belonging to one linked list into
252 + * consecutive array entries */
253 +
254 + for (i = 0; i < hsize; i++) {
255 + packed_hash[i] = packed_entry;
256 + for (entry = hash[i]; entry; entry = entry->next)
257 + *packed_entry++ = entry->entry;
258 + }
259 +
260 + /* Sentinel value to indicate the length of the last hash
261 + * bucket */
262 +
263 + packed_hash[hsize] = packed_entry;
264 + assert(packed_entry - (struct index_entry *)mem == entries);
265 + free(hash);
266 +
267 return index;
268 }
269
@@ -302,7 +344,7 @@ create_delta(const struct delta_index *index,
344 val ^= U[data[-RABIN_WINDOW]];
345 val = ((val << 8) | *data) ^ T[val >> RABIN_SHIFT];
346 i = val & index->hash_mask;
305 - for (entry = index->hash[i]; entry; entry = entry->next) {
347 + for (entry = index->hash[i]; entry < index->hash[i+1]; entry++) {
348 const unsigned char *ref = entry->ptr;
349 const unsigned char *src = data;
350 unsigned int ref_size = ref_top - ref;