@matheusbsbs / matheus-bagliano / commits / 8ecce684a3
Fork 0

basic threaded delta search

this is still rough, hence it is disabled by default. You need to compile with "make THREADED_DELTA_SEARCH=1 ..." at the moment. Threading is done on different portions of the object list to be deltified. This is currently done by spliting the list into n parts and then a thread is spawned for each of them. A better method would consist of spliting the list into more smaller parts and have the n threads pick the next part available. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Nicolas Pitre committed Sep 6, 2007 at 02:13 UTC 8ecce684a38f7cea084abe9eef80bda04d7c77be
2 files changed +90 -1
Makefile
+8
@@ -122,6 +122,9 @@ all::
122 # If not set it defaults to the bare 'wish'. If it is set to the empty
123 # string then NO_TCLTK will be forced (this is used by configure script).
124 #
125 +# Define THREADED_DELTA_SEARCH if you have pthreads and wish to exploit
126 +# parallel delta searching when packing objects.
127 +#
128
129 GIT-VERSION-FILE: .FORCE-GIT-VERSION-FILE
130 @$(SHELL_PATH) ./GIT-VERSION-GEN
@@ -662,6 +665,11 @@ ifdef NO_HSTRERROR
665 COMPAT_OBJS += compat/hstrerror.o
666 endif
667
668 +ifdef THREADED_DELTA_SEARCH
669 + BASIC_CFLAGS += -DTHREADED_DELTA_SEARCH
670 + EXTLIBS += -lpthread
671 +endif
672 +
673 ifeq ($(TCLTK_PATH),)
674 NO_TCLTK=NoThanks
675 endif
builtin-pack-objects.c
+82 -1
@@ -15,6 +15,10 @@
15 #include "list-objects.h"
16 #include "progress.h"
17
18 +#ifdef THREADED_DELTA_SEARCH
19 +#include <pthread.h>
20 +#endif
21 +
22 static const char pack_usage[] = "\
23 git-pack-objects [{ -q | --progress | --all-progress }] \n\
24 [--max-pack-size=N] [--local] [--incremental] \n\
@@ -1290,6 +1294,25 @@ static int delta_cacheable(unsigned long src_size, unsigned long trg_size,
1294 return 0;
1295 }
1296
1297 +#ifdef THREADED_DELTA_SEARCH
1298 +
1299 +static pthread_mutex_t read_mutex = PTHREAD_MUTEX_INITIALIZER;
1300 +#define read_lock() pthread_mutex_lock(&read_mutex)
1301 +#define read_unlock() pthread_mutex_unlock(&read_mutex)
1302 +
1303 +static pthread_mutex_t progress_mutex = PTHREAD_MUTEX_INITIALIZER;
1304 +#define progress_lock() pthread_mutex_lock(&progress_mutex)
1305 +#define progress_unlock() pthread_mutex_unlock(&progress_mutex)
1306 +
1307 +#else
1308 +
1309 +#define read_lock() 0
1310 +#define read_unlock() 0
1311 +#define progress_lock() 0
1312 +#define progress_unlock() 0
1313 +
1314 +#endif
1315 +
1316 /*
1317 * We search for deltas _backwards_ in a list sorted by type and
1318 * by size, so that we see progressively smaller and smaller files.
@@ -1348,7 +1371,9 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
1371
1372 /* Load data if not already done */
1373 if (!trg->data) {
1374 + read_lock();
1375 trg->data = read_sha1_file(trg_entry->idx.sha1, &type, &sz);
1376 + read_unlock();
1377 if (!trg->data)
1378 die("object %s cannot be read",
1379 sha1_to_hex(trg_entry->idx.sha1));
@@ -1358,7 +1383,9 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
1383 *mem_usage += sz;
1384 }
1385 if (!src->data) {
1386 + read_lock();
1387 src->data = read_sha1_file(src_entry->idx.sha1, &type, &sz);
1388 + read_unlock();
1389 if (!src->data)
1390 die("object %s cannot be read",
1391 sha1_to_hex(src_entry->idx.sha1));
@@ -1470,9 +1497,11 @@ static void find_deltas(struct object_entry **list, unsigned list_size,
1497 if (entry->preferred_base)
1498 goto next;
1499
1500 + progress_lock();
1501 (*processed)++;
1502 if (progress)
1503 display_progress(&progress_state, *processed);
1504 + progress_unlock();
1505
1506 /*
1507 * If the current object is at pack edge, take the depth the
@@ -1542,6 +1571,58 @@ static void find_deltas(struct object_entry **list, unsigned list_size,
1571 free(array);
1572 }
1573
1574 +#ifdef THREADED_DELTA_SEARCH
1575 +
1576 +struct thread_params {
1577 + pthread_t thread;
1578 + struct object_entry **list;
1579 + unsigned list_size;
1580 + int window;
1581 + int depth;
1582 + unsigned *processed;
1583 +};
1584 +
1585 +static void *threaded_find_deltas(void *arg)
1586 +{
1587 + struct thread_params *p = arg;
1588 + if (p->list_size)
1589 + find_deltas(p->list, p->list_size,
1590 + p->window, p->depth, p->processed);
1591 + return NULL;
1592 +}
1593 +
1594 +#define NR_THREADS 8
1595 +
1596 +static void ll_find_deltas(struct object_entry **list, unsigned list_size,
1597 + int window, int depth, unsigned *processed)
1598 +{
1599 + struct thread_params p[NR_THREADS];
1600 + int i, ret;
1601 +
1602 + for (i = 0; i < NR_THREADS; i++) {
1603 + unsigned sublist_size = list_size / (NR_THREADS - i);
1604 + p[i].list = list;
1605 + p[i].list_size = sublist_size;
1606 + p[i].window = window;
1607 + p[i].depth = depth;
1608 + p[i].processed = processed;
1609 + ret = pthread_create(&p[i].thread, NULL,
1610 + threaded_find_deltas, &p[i]);
1611 + if (ret)
1612 + die("unable to create thread: %s", strerror(ret));
1613 + list += sublist_size;
1614 + list_size -= sublist_size;
1615 + }
1616 +
1617 + for (i = 0; i < NR_THREADS; i++) {
1618 + pthread_join(p[i].thread, NULL);
1619 + }
1620 +}
1621 +
1622 +#else
1623 +#define ll_find_deltas find_deltas
1624 +#endif
1625 +
1626 static void prepare_pack(int window, int depth)
1627 {
1628 struct object_entry **delta_list;
@@ -1583,7 +1664,7 @@ static void prepare_pack(int window, int depth)
1664 "Deltifying %u objects...", "",
1665 nr_deltas);
1666 qsort(delta_list, n, sizeof(*delta_list), type_size_sort);
1586 - find_deltas(delta_list, n, window+1, depth, &nr_done);
1667 + ll_find_deltas(delta_list, n, window+1, depth, &nr_done);
1668 if (progress)
1669 stop_progress(&progress_state);
1670 if (nr_done != nr_deltas)