threaded delta search: refine work allocation
With this, each thread get repeatedly assigned the next available chunk of objects to process until the whole list is done. The idea is to have reasonably small chunks so that all CPUs remain busy with a minimum number of threads for as long as there is data to process. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Nicolas Pitre committed
Sep 10, 2007 at 00:06 UTC
c2a33679a726aeb75529540c2b295f21023ddbc7
1 file changed
+45
-14
builtin-pack-objects.c
+45
-14
@@ -1582,27 +1582,42 @@ struct thread_params {
1582
unsigned *processed;
1583
};
1584
1585
+static pthread_mutex_t data_request = PTHREAD_MUTEX_INITIALIZER;
1586
+static pthread_mutex_t data_ready = PTHREAD_MUTEX_INITIALIZER;
1587
+static pthread_mutex_t data_provider = PTHREAD_MUTEX_INITIALIZER;
1588
+static struct thread_params *data_requester;
1589
+
1590
static void *threaded_find_deltas(void *arg)
1591
{
1587
- struct thread_params *p = arg;
1588
- if (p->list_size)
1589
- find_deltas(p->list, p->list_size,
1590
- p->window, p->depth, p->processed);
1591
- return NULL;
1592
+ struct thread_params *me = arg;
1593
+
1594
+ for (;;) {
1595
+ pthread_mutex_lock(&data_request);
1596
+ data_requester = me;
1597
+ pthread_mutex_unlock(&data_provider);
1598
+ pthread_mutex_lock(&data_ready);
1599
+
1600
+ if (!me->list_size)
1601
+ return NULL;
1602
+
1603
+ find_deltas(me->list, me->list_size,
1604
+ me->window, me->depth, me->processed);
1605
+ }
1606
}
1607
1594
-#define NR_THREADS 8
1608
+#define NR_THREADS 4
1609
1610
static void ll_find_deltas(struct object_entry **list, unsigned list_size,
1611
int window, int depth, unsigned *processed)
1612
{
1613
struct thread_params p[NR_THREADS];
1614
int i, ret;
1615
+ unsigned chunk_size;
1616
+
1617
+ pthread_mutex_lock(&data_provider);
1618
+ pthread_mutex_lock(&data_ready);
1619
1620
for (i = 0; i < NR_THREADS; i++) {
1603
- unsigned sublist_size = list_size / (NR_THREADS - i);
1604
- p[i].list = list;
1605
- p[i].list_size = sublist_size;
1621
p[i].window = window;
1622
p[i].depth = depth;
1623
p[i].processed = processed;
@@ -1610,13 +1625,29 @@ static void ll_find_deltas(struct object_entry **list, unsigned list_size,
1625
threaded_find_deltas, &p[i]);
1626
if (ret)
1627
die("unable to create thread: %s", strerror(ret));
1613
- list += sublist_size;
1614
- list_size -= sublist_size;
1628
}
1629
1617
- for (i = 0; i < NR_THREADS; i++) {
1618
- pthread_join(p[i].thread, NULL);
1619
- }
1630
+ /* this should be auto-tuned somehow */
1631
+ chunk_size = window * 1000;
1632
+
1633
+ do {
1634
+ unsigned sublist_size = chunk_size;
1635
+ if (sublist_size > list_size)
1636
+ sublist_size = list_size;
1637
+
1638
+ pthread_mutex_lock(&data_provider);
1639
+ data_requester->list = list;
1640
+ data_requester->list_size = sublist_size;
1641
+ pthread_mutex_unlock(&data_ready);
1642
+
1643
+ list += sublist_size;
1644
+ list_size -= sublist_size;
1645
+ if (!sublist_size) {
1646
+ pthread_join(data_requester->thread, NULL);
1647
+ i--;
1648
+ }
1649
+ pthread_mutex_unlock(&data_request);
1650
+ } while (i);
1651
}
1652
1653
#else