1479
return freed_mem;
1480
}
1481
1482
-static void find_deltas(struct object_entry **list, unsigned list_size,
1482
+static void find_deltas(struct object_entry **list, unsigned *list_size,
1483
int window, int depth, unsigned *processed)
1484
{
1485
- uint32_t i = 0, idx = 0, count = 0;
1485
+ uint32_t i, idx = 0, count = 0;
1486
unsigned int array_size = window * sizeof(struct unpacked);
1487
struct unpacked *array;
1488
unsigned long mem_usage = 0;
1490
array = xmalloc(array_size);
1491
memset(array, 0, array_size);
1492
1493
- do {
1494
- struct object_entry *entry = list[i++];
1493
+ for (;;) {
1494
+ struct object_entry *entry = *list++;
1495
struct unpacked *n = array + idx;
1496
int j, max_depth, best_base = -1;
1497
1498
+ progress_lock();
1499
+ if (!*list_size) {
1500
+ progress_unlock();
1501
+ break;
1502
+ }
1503
+ (*list_size)--;
1504
+ if (!entry->preferred_base) {
1505
+ (*processed)++;
1506
+ display_progress(progress_state, *processed);
1507
+ }
1508
+ progress_unlock();
1509
+
1510
mem_usage -= free_unpacked(n);
1511
n->entry = entry;
1512
1524
if (entry->preferred_base)
1525
goto next;
1526
1515
- progress_lock();
1516
- (*processed)++;
1517
- display_progress(progress_state, *processed);
1518
- progress_unlock();
1519
-
1527
/*
1528
* If the current object is at pack edge, take the depth the
1529
* objects that depend on the current object into account
1583
count++;
1584
if (idx >= window)
1585
idx = 0;
1579
- } while (i < list_size);
1586
+ }
1587
1588
for (i = 0; i < window; ++i) {
1589
free_delta_index(array[i].index);
1598
pthread_t thread;
1599
struct object_entry **list;
1600
unsigned list_size;
1601
+ unsigned remaining;
1602
int window;
1603
int depth;
1604
unsigned *processed;
1620
pthread_mutex_lock(&data_ready);
1621
pthread_mutex_unlock(&data_request);
1622
1615
- if (!me->list_size)
1623
+ if (!me->remaining)
1624
return NULL;
1625
1618
- find_deltas(me->list, me->list_size,
1626
+ find_deltas(me->list, &me->remaining,
1627
me->window, me->depth, me->processed);
1628
}
1629
}
1632
int window, int depth, unsigned *processed)
1633
{
1634
struct thread_params *target, p[delta_search_threads];
1627
- int i, ret;
1628
- unsigned chunk_size;
1635
+ int i, ret, active_threads = 0;
1636
1637
if (delta_search_threads <= 1) {
1631
- find_deltas(list, list_size, window, depth, processed);
1638
+ find_deltas(list, &list_size, window, depth, processed);
1639
return;
1640
}
1641
1642
pthread_mutex_lock(&data_provider);
1643
pthread_mutex_lock(&data_ready);
1644
1645
+ /* Start work threads. */
1646
for (i = 0; i < delta_search_threads; i++) {
1647
p[i].window = window;
1648
p[i].depth = depth;
1649
p[i].processed = processed;
1650
+ p[i].remaining = 0;
1651
ret = pthread_create(&p[i].thread, NULL,
1652
threaded_find_deltas, &p[i]);
1653
if (ret)
1654
die("unable to create thread: %s", strerror(ret));
1655
+ active_threads++;
1656
}
1657
1648
- /* this should be auto-tuned somehow */
1649
- chunk_size = window * 1000;
1658
+ /* Then partition the work amongst them. */
1659
+ for (i = 0; i < delta_search_threads; i++) {
1660
+ unsigned sub_size = list_size / (delta_search_threads - i);
1661
1651
- do {
1652
- unsigned sublist_size = chunk_size;
1653
- if (sublist_size > list_size)
1654
- sublist_size = list_size;
1662
+ pthread_mutex_lock(&data_provider);
1663
+ target = data_requester;
1664
+ if (!sub_size) {
1665
+ pthread_mutex_unlock(&data_ready);
1666
+ pthread_join(target->thread, NULL);
1667
+ active_threads--;
1668
+ continue;
1669
+ }
1670
1671
/* try to split chunks on "path" boundaries */
1657
- while (sublist_size < list_size && list[sublist_size]->hash &&
1658
- list[sublist_size]->hash == list[sublist_size-1]->hash)
1659
- sublist_size++;
1672
+ while (sub_size < list_size && list[sub_size]->hash &&
1673
+ list[sub_size]->hash == list[sub_size-1]->hash)
1674
+ sub_size++;
1675
+
1676
+ target->list = list;
1677
+ target->list_size = sub_size;
1678
+ target->remaining = sub_size;
1679
+ pthread_mutex_unlock(&data_ready);
1680
1681
+ list += sub_size;
1682
+ list_size -= sub_size;
1683
+ }
1684
+
1685
+ /*
1686
+ * Now let's wait for work completion. Each time a thread is done
1687
+ * with its work, we steal half of the remaining work from the
1688
+ * thread with the largest number of unprocessed objects and give
1689
+ * it to that newly idle thread. This ensure good load balancing
1690
+ * until the remaining object list segments are simply too short
1691
+ * to be worth splitting anymore.
1692
+ */
1693
+ do {
1694
+ struct thread_params *victim = NULL;
1695
+ unsigned sub_size = 0;
1696
pthread_mutex_lock(&data_provider);
1697
target = data_requester;
1663
- target->list = list;
1664
- target->list_size = sublist_size;
1698
+
1699
+ progress_lock();
1700
+ for (i = 0; i < delta_search_threads; i++)
1701
+ if (p[i].remaining > 2*window &&
1702
+ (!victim || victim->remaining < p[i].remaining))
1703
+ victim = &p[i];
1704
+ if (victim) {
1705
+ sub_size = victim->remaining / 2;
1706
+ list = victim->list + victim->list_size - sub_size;
1707
+ while (sub_size && list[0]->hash &&
1708
+ list[0]->hash == list[-1]->hash) {
1709
+ list++;
1710
+ sub_size--;
1711
+ }
1712
+ target->list = list;
1713
+ victim->list_size -= sub_size;
1714
+ victim->remaining -= sub_size;
1715
+ }
1716
+ progress_unlock();
1717
+
1718
+ target->list_size = sub_size;
1719
+ target->remaining = sub_size;
1720
pthread_mutex_unlock(&data_ready);
1721
1667
- list += sublist_size;
1668
- list_size -= sublist_size;
1669
- if (!sublist_size) {
1722
+ if (!sub_size) {
1723
pthread_join(target->thread, NULL);
1671
- i--;
1724
+ active_threads--;
1725
}
1673
- } while (i);
1726
+ } while (active_threads);
1727
}
1728
1729
#else
1677
-#define ll_find_deltas find_deltas
1730
+#define ll_find_deltas(l, s, w, d, p) find_deltas(l, &s, w, d, p)
1731
#endif
1732
1733
static void prepare_pack(int window, int depth)