@matheusbsbs / matheus-bagliano / commits / 6d016c9c7f
Fork 0

git-pack-redundant: speed and memory usage improvements

Slab allocation of llist entries gives some speed improvements. Not computing the pack_list permutaions all at once reduces memory usage greatly on repositories with many packs. Signed-off-by: Lukas Sandström <lukass@etek.chalmers.se> Signed-off-by: Junio C Hamano <junkio@cox.net>

Lukas Sandström committed Dec 20, 2005 at 21:05 UTC 6d016c9c7f21a97249c4571e8d7265da1fd2f083
1 file changed +69 -80
pack-redundant.c
+69 -80
@@ -8,6 +8,8 @@
8
9 #include "cache.h"
10
11 +#define BLKSIZE 512
12 +
13 static const char pack_redundant_usage[] =
14 "git-pack-redundant [ --verbose ] [ --alt-odb ] < --all | <.pack filename> ...>";
15
@@ -33,29 +35,32 @@ static struct pack_list {
35 struct pll {
36 struct pll *next;
37 struct pack_list *pl;
36 - size_t pl_size;
38 };
39
40 static struct llist_item *free_nodes = NULL;
41
42 +static inline void llist_item_put(struct llist_item *item)
43 +{
44 + item->next = free_nodes;
45 + free_nodes = item;
46 +}
47 +
48 static inline struct llist_item *llist_item_get()
49 {
50 struct llist_item *new;
51 if ( free_nodes ) {
52 new = free_nodes;
53 free_nodes = free_nodes->next;
47 - } else
48 - new = xmalloc(sizeof(struct llist_item));
49 -
54 + } else {
55 + int i = 1;
56 + new = xmalloc(sizeof(struct llist_item) * BLKSIZE);
57 + for(;i < BLKSIZE; i++) {
58 + llist_item_put(&new[i]);
59 + }
60 + }
61 return new;
62 }
63
53 -static inline void llist_item_put(struct llist_item *item)
54 -{
55 - item->next = free_nodes;
56 - free_nodes = item;
57 -}
58 -
64 static void llist_free(struct llist *list)
65 {
66 while((list->back = list->front)) {
@@ -270,77 +275,58 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
275 }
276 }
277
273 -static void pll_insert(struct pll **pll, struct pll **hint_table)
278 +void pll_free(struct pll *l)
279 {
275 - struct pll *prev;
276 - int i = (*pll)->pl_size - 1;
277 -
278 - if (hint_table[i] == NULL) {
279 - hint_table[i--] = *pll;
280 - for (; i >= 0; --i) {
281 - if (hint_table[i] != NULL)
282 - break;
280 + struct pll *old;
281 + struct pack_list *opl;
282 +
283 + while (l) {
284 + old = l;
285 + while (l->pl) {
286 + opl = l->pl;
287 + l->pl = opl->next;
288 + free(opl);
289 }
284 - if (hint_table[i] == NULL) /* no elements in list */
285 - die("Why did this happen?");
290 + l = l->next;
291 + free(old);
292 }
287 -
288 - prev = hint_table[i];
289 - while (prev->next && prev->next->pl_size < (*pll)->pl_size)
290 - prev = prev->next;
291 -
292 - (*pll)->next = prev->next;
293 - prev->next = *pll;
293 }
294
295 /* all the permutations have to be free()d at the same time,
296 * since they refer to each other
297 */
299 -static struct pll * get_all_permutations(struct pack_list *list)
298 +static struct pll * get_permutations(struct pack_list *list, int n)
299 {
301 - struct pll *subset, *pll, *new_pll = NULL; /*silence warning*/
302 - static struct pll **hint = NULL;
303 - if (hint == NULL)
304 - hint = xcalloc(pack_list_size(list), sizeof(struct pll *));
305 -
306 - if (list == NULL)
300 + struct pll *subset, *ret = NULL, *new_pll = NULL, *pll;
301 +
302 + if (list == NULL || pack_list_size(list) < n || n == 0)
303 return NULL;
304
309 - if (list->next == NULL) {
310 - new_pll = xmalloc(sizeof(struct pll));
311 - hint[0] = new_pll;
312 - new_pll->next = NULL;
313 - new_pll->pl = list;
314 - new_pll->pl_size = 1;
315 - return new_pll;
305 + if (n == 1) {
306 + while (list) {
307 + new_pll = xmalloc(sizeof(pll));
308 + new_pll->pl = NULL;
309 + pack_list_insert(&new_pll->pl, list);
310 + new_pll->next = ret;
311 + ret = new_pll;
312 + list = list->next;
313 + }
314 + return ret;
315 }
316
318 - pll = subset = get_all_permutations(list->next);
319 - while (pll) {
320 - if (pll->pl->pack == list->pack) {
321 - pll = pll->next;
322 - continue;
317 + while (list->next) {
318 + subset = get_permutations(list->next, n - 1);
319 + while (subset) {
320 + new_pll = xmalloc(sizeof(pll));
321 + new_pll->pl = subset->pl;
322 + pack_list_insert(&new_pll->pl, list);
323 + new_pll->next = ret;
324 + ret = new_pll;
325 + subset = subset->next;
326 }
324 - new_pll = xmalloc(sizeof(struct pll));
325 -
326 - new_pll->pl = xmalloc(sizeof(struct pack_list));
327 - memcpy(new_pll->pl, list, sizeof(struct pack_list));
328 - new_pll->pl->next = pll->pl;
329 - new_pll->pl_size = pll->pl_size + 1;
330 -
331 - pll_insert(&new_pll, hint);
332 -
333 - pll = pll->next;
334 - }
335 - /* add ourself */
336 - new_pll = xmalloc(sizeof(struct pll));
337 - new_pll->pl = xmalloc(sizeof(struct pack_list));
338 - memcpy(new_pll->pl, list, sizeof(struct pack_list));
339 - new_pll->pl->next = NULL;
340 - new_pll->pl_size = 1;
341 - pll_insert(&new_pll, hint);
342 -
343 - return hint[0];
327 + list = list->next;
328 + }
329 + return ret;
330 }
331
332 static int is_superset(struct pack_list *pl, struct llist *list)
@@ -428,6 +414,7 @@ static void minimize(struct pack_list **min)
414 struct pll *perm, *perm_all, *perm_ok = NULL, *new_perm;
415 struct llist *missing;
416 size_t min_perm_size = (size_t)-1, perm_size;
417 + int n;
418
419 pl = local_packs;
420 while (pl) {
@@ -441,8 +428,7 @@ static void minimize(struct pack_list **min)
428 missing = llist_copy(all_objects);
429 pl = unique;
430 while (pl) {
444 - llist_sorted_difference_inplace(missing,
445 - pl->all_objects);
431 + llist_sorted_difference_inplace(missing, pl->all_objects);
432 pl = pl->next;
433 }
434
@@ -453,19 +439,21 @@ static void minimize(struct pack_list **min)
439 }
440
441 /* find the permutations which contain all missing objects */
456 - perm_all = perm = get_all_permutations(non_unique);
457 - while (perm) {
458 - if (perm_ok && perm->pl_size > perm_ok->pl_size)
459 - break; /* ignore all larger permutations */
460 - if (is_superset(perm->pl, missing)) {
461 - new_perm = xmalloc(sizeof(struct pll));
462 - memcpy(new_perm, perm, sizeof(struct pll));
463 - new_perm->next = perm_ok;
464 - perm_ok = new_perm;
442 + for (n = 1; n <= pack_list_size(non_unique) && !perm_ok; n++) {
443 + perm_all = perm = get_permutations(non_unique, n);
444 + while (perm) {
445 + if (is_superset(perm->pl, missing)) {
446 + new_perm = xmalloc(sizeof(struct pll));
447 + memcpy(new_perm, perm, sizeof(struct pll));
448 + new_perm->next = perm_ok;
449 + perm_ok = new_perm;
450 + }
451 + perm = perm->next;
452 }
466 - perm = perm->next;
453 + if (perm_ok)
454 + break;
455 + pll_free(perm_all);
456 }
468 -
457 if (perm_ok == NULL)
458 die("Internal error: No complete sets found!\n");
459
@@ -537,6 +525,7 @@ static void scan_alt_odb_packs(void)
525 alt->all_objects);
526 local = local->next;
527 }
528 + llist_sorted_difference_inplace(all_objects, alt->all_objects);
529 alt = alt->next;
530 }
531 }