basic threaded delta search
this is still rough, hence it is disabled by default. You need to compile with "make THREADED_DELTA_SEARCH=1 ..." at the moment. Threading is done on different portions of the object list to be deltified. This is currently done by spliting the list into n parts and then a thread is spawned for each of them. A better method would consist of spliting the list into more smaller parts and have the n threads pick the next part available. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Nicolas Pitre committed
Sep 6, 2007 at 02:13 UTC
8ecce684a38f7cea084abe9eef80bda04d7c77be
2 files changed
+90
-1
Makefile
+8
@@ -122,6 +122,9 @@ all::
122
# If not set it defaults to the bare 'wish'. If it is set to the empty
123
# string then NO_TCLTK will be forced (this is used by configure script).
124
#
125
+# Define THREADED_DELTA_SEARCH if you have pthreads and wish to exploit
126
+# parallel delta searching when packing objects.
127
+#
128
129
GIT-VERSION-FILE: .FORCE-GIT-VERSION-FILE
130
@$(SHELL_PATH) ./GIT-VERSION-GEN
@@ -662,6 +665,11 @@ ifdef NO_HSTRERROR
665
COMPAT_OBJS += compat/hstrerror.o
666
endif
667
668
+ifdef THREADED_DELTA_SEARCH
669
+ BASIC_CFLAGS += -DTHREADED_DELTA_SEARCH
670
+ EXTLIBS += -lpthread
671
+endif
672
+
673
ifeq ($(TCLTK_PATH),)
674
NO_TCLTK=NoThanks
675
endif
builtin-pack-objects.c
+82
-1
@@ -15,6 +15,10 @@
15
#include "list-objects.h"
16
#include "progress.h"
17
18
+#ifdef THREADED_DELTA_SEARCH
19
+#include <pthread.h>
20
+#endif
21
+
22
static const char pack_usage[] = "\
23
git-pack-objects [{ -q | --progress | --all-progress }] \n\
24
[--max-pack-size=N] [--local] [--incremental] \n\
@@ -1290,6 +1294,25 @@ static int delta_cacheable(unsigned long src_size, unsigned long trg_size,
1294
return 0;
1295
}
1296
1297
+#ifdef THREADED_DELTA_SEARCH
1298
+
1299
+static pthread_mutex_t read_mutex = PTHREAD_MUTEX_INITIALIZER;
1300
+#define read_lock() pthread_mutex_lock(&read_mutex)
1301
+#define read_unlock() pthread_mutex_unlock(&read_mutex)
1302
+
1303
+static pthread_mutex_t progress_mutex = PTHREAD_MUTEX_INITIALIZER;
1304
+#define progress_lock() pthread_mutex_lock(&progress_mutex)
1305
+#define progress_unlock() pthread_mutex_unlock(&progress_mutex)
1306
+
1307
+#else
1308
+
1309
+#define read_lock() 0
1310
+#define read_unlock() 0
1311
+#define progress_lock() 0
1312
+#define progress_unlock() 0
1313
+
1314
+#endif
1315
+
1316
/*
1317
* We search for deltas _backwards_ in a list sorted by type and
1318
* by size, so that we see progressively smaller and smaller files.
@@ -1348,7 +1371,9 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
1371
1372
/* Load data if not already done */
1373
if (!trg->data) {
1374
+ read_lock();
1375
trg->data = read_sha1_file(trg_entry->idx.sha1, &type, &sz);
1376
+ read_unlock();
1377
if (!trg->data)
1378
die("object %s cannot be read",
1379
sha1_to_hex(trg_entry->idx.sha1));
@@ -1358,7 +1383,9 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
1383
*mem_usage += sz;
1384
}
1385
if (!src->data) {
1386
+ read_lock();
1387
src->data = read_sha1_file(src_entry->idx.sha1, &type, &sz);
1388
+ read_unlock();
1389
if (!src->data)
1390
die("object %s cannot be read",
1391
sha1_to_hex(src_entry->idx.sha1));
@@ -1470,9 +1497,11 @@ static void find_deltas(struct object_entry **list, unsigned list_size,
1497
if (entry->preferred_base)
1498
goto next;
1499
1500
+ progress_lock();
1501
(*processed)++;
1502
if (progress)
1503
display_progress(&progress_state, *processed);
1504
+ progress_unlock();
1505
1506
/*
1507
* If the current object is at pack edge, take the depth the
@@ -1542,6 +1571,58 @@ static void find_deltas(struct object_entry **list, unsigned list_size,
1571
free(array);
1572
}
1573
1574
+#ifdef THREADED_DELTA_SEARCH
1575
+
1576
+struct thread_params {
1577
+ pthread_t thread;
1578
+ struct object_entry **list;
1579
+ unsigned list_size;
1580
+ int window;
1581
+ int depth;
1582
+ unsigned *processed;
1583
+};
1584
+
1585
+static void *threaded_find_deltas(void *arg)
1586
+{
1587
+ struct thread_params *p = arg;
1588
+ if (p->list_size)
1589
+ find_deltas(p->list, p->list_size,
1590
+ p->window, p->depth, p->processed);
1591
+ return NULL;
1592
+}
1593
+
1594
+#define NR_THREADS 8
1595
+
1596
+static void ll_find_deltas(struct object_entry **list, unsigned list_size,
1597
+ int window, int depth, unsigned *processed)
1598
+{
1599
+ struct thread_params p[NR_THREADS];
1600
+ int i, ret;
1601
+
1602
+ for (i = 0; i < NR_THREADS; i++) {
1603
+ unsigned sublist_size = list_size / (NR_THREADS - i);
1604
+ p[i].list = list;
1605
+ p[i].list_size = sublist_size;
1606
+ p[i].window = window;
1607
+ p[i].depth = depth;
1608
+ p[i].processed = processed;
1609
+ ret = pthread_create(&p[i].thread, NULL,
1610
+ threaded_find_deltas, &p[i]);
1611
+ if (ret)
1612
+ die("unable to create thread: %s", strerror(ret));
1613
+ list += sublist_size;
1614
+ list_size -= sublist_size;
1615
+ }
1616
+
1617
+ for (i = 0; i < NR_THREADS; i++) {
1618
+ pthread_join(p[i].thread, NULL);
1619
+ }
1620
+}
1621
+
1622
+#else
1623
+#define ll_find_deltas find_deltas
1624
+#endif
1625
+
1626
static void prepare_pack(int window, int depth)
1627
{
1628
struct object_entry **delta_list;
@@ -1583,7 +1664,7 @@ static void prepare_pack(int window, int depth)
1664
"Deltifying %u objects...", "",
1665
nr_deltas);
1666
qsort(delta_list, n, sizeof(*delta_list), type_size_sort);
1586
- find_deltas(delta_list, n, window+1, depth, &nr_done);
1667
+ ll_find_deltas(delta_list, n, window+1, depth, &nr_done);
1668
if (progress)
1669
stop_progress(&progress_state);
1670
if (nr_done != nr_deltas)