git-pack-redundant: speed and memory usage improvements
Slab allocation of llist entries gives some speed improvements. Not computing the pack_list permutaions all at once reduces memory usage greatly on repositories with many packs. Signed-off-by: Lukas Sandström <lukass@etek.chalmers.se> Signed-off-by: Junio C Hamano <junkio@cox.net>
Lukas Sandström committed
Dec 20, 2005 at 21:05 UTC
6d016c9c7f21a97249c4571e8d7265da1fd2f083
1 file changed
+69
-80
pack-redundant.c
+69
-80
@@ -8,6 +8,8 @@
8
9
#include "cache.h"
10
11
+#define BLKSIZE 512
12
+
13
static const char pack_redundant_usage[] =
14
"git-pack-redundant [ --verbose ] [ --alt-odb ] < --all | <.pack filename> ...>";
15
@@ -33,29 +35,32 @@ static struct pack_list {
35
struct pll {
36
struct pll *next;
37
struct pack_list *pl;
36
- size_t pl_size;
38
};
39
40
static struct llist_item *free_nodes = NULL;
41
42
+static inline void llist_item_put(struct llist_item *item)
43
+{
44
+ item->next = free_nodes;
45
+ free_nodes = item;
46
+}
47
+
48
static inline struct llist_item *llist_item_get()
49
{
50
struct llist_item *new;
51
if ( free_nodes ) {
52
new = free_nodes;
53
free_nodes = free_nodes->next;
47
- } else
48
- new = xmalloc(sizeof(struct llist_item));
49
-
54
+ } else {
55
+ int i = 1;
56
+ new = xmalloc(sizeof(struct llist_item) * BLKSIZE);
57
+ for(;i < BLKSIZE; i++) {
58
+ llist_item_put(&new[i]);
59
+ }
60
+ }
61
return new;
62
}
63
53
-static inline void llist_item_put(struct llist_item *item)
54
-{
55
- item->next = free_nodes;
56
- free_nodes = item;
57
-}
58
-
64
static void llist_free(struct llist *list)
65
{
66
while((list->back = list->front)) {
@@ -270,77 +275,58 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
275
}
276
}
277
273
-static void pll_insert(struct pll **pll, struct pll **hint_table)
278
+void pll_free(struct pll *l)
279
{
275
- struct pll *prev;
276
- int i = (*pll)->pl_size - 1;
277
-
278
- if (hint_table[i] == NULL) {
279
- hint_table[i--] = *pll;
280
- for (; i >= 0; --i) {
281
- if (hint_table[i] != NULL)
282
- break;
280
+ struct pll *old;
281
+ struct pack_list *opl;
282
+
283
+ while (l) {
284
+ old = l;
285
+ while (l->pl) {
286
+ opl = l->pl;
287
+ l->pl = opl->next;
288
+ free(opl);
289
}
284
- if (hint_table[i] == NULL) /* no elements in list */
285
- die("Why did this happen?");
290
+ l = l->next;
291
+ free(old);
292
}
287
-
288
- prev = hint_table[i];
289
- while (prev->next && prev->next->pl_size < (*pll)->pl_size)
290
- prev = prev->next;
291
-
292
- (*pll)->next = prev->next;
293
- prev->next = *pll;
293
}
294
295
/* all the permutations have to be free()d at the same time,
296
* since they refer to each other
297
*/
299
-static struct pll * get_all_permutations(struct pack_list *list)
298
+static struct pll * get_permutations(struct pack_list *list, int n)
299
{
301
- struct pll *subset, *pll, *new_pll = NULL; /*silence warning*/
302
- static struct pll **hint = NULL;
303
- if (hint == NULL)
304
- hint = xcalloc(pack_list_size(list), sizeof(struct pll *));
305
-
306
- if (list == NULL)
300
+ struct pll *subset, *ret = NULL, *new_pll = NULL, *pll;
301
+
302
+ if (list == NULL || pack_list_size(list) < n || n == 0)
303
return NULL;
304
309
- if (list->next == NULL) {
310
- new_pll = xmalloc(sizeof(struct pll));
311
- hint[0] = new_pll;
312
- new_pll->next = NULL;
313
- new_pll->pl = list;
314
- new_pll->pl_size = 1;
315
- return new_pll;
305
+ if (n == 1) {
306
+ while (list) {
307
+ new_pll = xmalloc(sizeof(pll));
308
+ new_pll->pl = NULL;
309
+ pack_list_insert(&new_pll->pl, list);
310
+ new_pll->next = ret;
311
+ ret = new_pll;
312
+ list = list->next;
313
+ }
314
+ return ret;
315
}
316
318
- pll = subset = get_all_permutations(list->next);
319
- while (pll) {
320
- if (pll->pl->pack == list->pack) {
321
- pll = pll->next;
322
- continue;
317
+ while (list->next) {
318
+ subset = get_permutations(list->next, n - 1);
319
+ while (subset) {
320
+ new_pll = xmalloc(sizeof(pll));
321
+ new_pll->pl = subset->pl;
322
+ pack_list_insert(&new_pll->pl, list);
323
+ new_pll->next = ret;
324
+ ret = new_pll;
325
+ subset = subset->next;
326
}
324
- new_pll = xmalloc(sizeof(struct pll));
325
-
326
- new_pll->pl = xmalloc(sizeof(struct pack_list));
327
- memcpy(new_pll->pl, list, sizeof(struct pack_list));
328
- new_pll->pl->next = pll->pl;
329
- new_pll->pl_size = pll->pl_size + 1;
330
-
331
- pll_insert(&new_pll, hint);
332
-
333
- pll = pll->next;
334
- }
335
- /* add ourself */
336
- new_pll = xmalloc(sizeof(struct pll));
337
- new_pll->pl = xmalloc(sizeof(struct pack_list));
338
- memcpy(new_pll->pl, list, sizeof(struct pack_list));
339
- new_pll->pl->next = NULL;
340
- new_pll->pl_size = 1;
341
- pll_insert(&new_pll, hint);
342
-
343
- return hint[0];
327
+ list = list->next;
328
+ }
329
+ return ret;
330
}
331
332
static int is_superset(struct pack_list *pl, struct llist *list)
@@ -428,6 +414,7 @@ static void minimize(struct pack_list **min)
414
struct pll *perm, *perm_all, *perm_ok = NULL, *new_perm;
415
struct llist *missing;
416
size_t min_perm_size = (size_t)-1, perm_size;
417
+ int n;
418
419
pl = local_packs;
420
while (pl) {
@@ -441,8 +428,7 @@ static void minimize(struct pack_list **min)
428
missing = llist_copy(all_objects);
429
pl = unique;
430
while (pl) {
444
- llist_sorted_difference_inplace(missing,
445
- pl->all_objects);
431
+ llist_sorted_difference_inplace(missing, pl->all_objects);
432
pl = pl->next;
433
}
434
@@ -453,19 +439,21 @@ static void minimize(struct pack_list **min)
439
}
440
441
/* find the permutations which contain all missing objects */
456
- perm_all = perm = get_all_permutations(non_unique);
457
- while (perm) {
458
- if (perm_ok && perm->pl_size > perm_ok->pl_size)
459
- break; /* ignore all larger permutations */
460
- if (is_superset(perm->pl, missing)) {
461
- new_perm = xmalloc(sizeof(struct pll));
462
- memcpy(new_perm, perm, sizeof(struct pll));
463
- new_perm->next = perm_ok;
464
- perm_ok = new_perm;
442
+ for (n = 1; n <= pack_list_size(non_unique) && !perm_ok; n++) {
443
+ perm_all = perm = get_permutations(non_unique, n);
444
+ while (perm) {
445
+ if (is_superset(perm->pl, missing)) {
446
+ new_perm = xmalloc(sizeof(struct pll));
447
+ memcpy(new_perm, perm, sizeof(struct pll));
448
+ new_perm->next = perm_ok;
449
+ perm_ok = new_perm;
450
+ }
451
+ perm = perm->next;
452
}
466
- perm = perm->next;
453
+ if (perm_ok)
454
+ break;
455
+ pll_free(perm_all);
456
}
468
-
457
if (perm_ok == NULL)
458
die("Internal error: No complete sets found!\n");
459
@@ -537,6 +525,7 @@ static void scan_alt_odb_packs(void)
525
alt->all_objects);
526
local = local->next;
527
}
528
+ llist_sorted_difference_inplace(all_objects, alt->all_objects);
529
alt = alt->next;
530
}
531
}