pack-objects: Add runtime detection of online CPU's
Packing objects can be done in parallell nowadays, but it's only done if the config option pack.threads is set to a value above 1. Because of that, the code-path used is often not the most optimal one. This patch adds a routine to detect the number of online CPU's at runtime (online_cpus()). When pack.threads (or --threads=) is given a value of 0, the number of threads is set to the number of online CPU's. This feature is also documented. As per Nicolas Pitre's recommendations, the default is still to run pack-objects single-threaded unless explicitly activated, either by configuration or by command line parameter. The routine online_cpus() is a rework of "numcpus.c", written by one Philip Willoughby <pgw99@doc.ic.ac.uk>. numcpus.c is in the public domain and can presently be downloaded from http://csgsoft.doc.ic.ac.uk/numcpus/ Signed-off-by: Andreas Ericsson <ae@op5.se> Signed-off-by: Brandon Casey <casey@nrlssc.navy.mil> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Andreas Ericsson committed
Feb 22, 2008 at 20:11 UTC
833e3df17188e48b2d31db0966a4051dad5e03b9
6 files changed
+69
-4
Documentation/config.txt
+2
@@ -808,6 +808,8 @@ pack.threads::
808
warning. This is meant to reduce packing time on multiprocessor
809
machines. The required amount of memory for the delta search window
810
is however multiplied by the number of threads.
811
+ Specifying 0 will cause git to auto-detect the number of CPU's
812
+ and set the number of threads accordingly.
813
814
pack.indexVersion::
815
Specify the default pack index version. Valid values are 1 for
Documentation/git-pack-objects.txt
+2
@@ -177,6 +177,8 @@ base-name::
177
This is meant to reduce packing time on multiprocessor machines.
178
The required amount of memory for the delta search window is
179
however multiplied by the number of threads.
180
+ Specifying 0 will cause git to auto-detect the number of CPU's
181
+ and set the number of threads accordingly.
182
183
--index-version=<version>[,<offset>]::
184
This is intended to be used by the test suite only. It allows
Makefile
+1
@@ -741,6 +741,7 @@ endif
741
ifdef THREADED_DELTA_SEARCH
742
BASIC_CFLAGS += -DTHREADED_DELTA_SEARCH
743
EXTLIBS += -lpthread
744
+ LIB_OBJS += thread-utils.o
745
endif
746
747
ifeq ($(TCLTK_PATH),)
builtin-pack-objects.c
+10
-4
@@ -16,6 +16,7 @@
16
#include "progress.h"
17
18
#ifdef THREADED_DELTA_SEARCH
19
+#include "thread-utils.h"
20
#include <pthread.h>
21
#endif
22
@@ -1852,11 +1853,11 @@ static int git_pack_config(const char *k, const char *v)
1853
}
1854
if (!strcmp(k, "pack.threads")) {
1855
delta_search_threads = git_config_int(k, v);
1855
- if (delta_search_threads < 1)
1856
+ if (delta_search_threads < 0)
1857
die("invalid number of threads specified (%d)",
1858
delta_search_threads);
1859
#ifndef THREADED_DELTA_SEARCH
1859
- if (delta_search_threads > 1)
1860
+ if (delta_search_threads != 1)
1861
warning("no threads support, ignoring %s", k);
1862
#endif
1863
return 0;
@@ -2122,10 +2123,10 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
2123
if (!prefixcmp(arg, "--threads=")) {
2124
char *end;
2125
delta_search_threads = strtoul(arg+10, &end, 0);
2125
- if (!arg[10] || *end || delta_search_threads < 1)
2126
+ if (!arg[10] || *end || delta_search_threads < 0)
2127
usage(pack_usage);
2128
#ifndef THREADED_DELTA_SEARCH
2128
- if (delta_search_threads > 1)
2129
+ if (delta_search_threads != 1)
2130
warning("no threads support, "
2131
"ignoring %s", arg);
2132
#endif
@@ -2235,6 +2236,11 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
2236
if (!pack_to_stdout && thin)
2237
die("--thin cannot be used to build an indexable pack.");
2238
2239
+#ifdef THREADED_DELTA_SEARCH
2240
+ if (!delta_search_threads) /* --threads=0 means autodetect */
2241
+ delta_search_threads = online_cpus();
2242
+#endif
2243
+
2244
prepare_packed_git();
2245
2246
if (progress)
thread-utils.c
new
+48
@@ -0,0 +1,48 @@
1
+#include "cache.h"
2
+
3
+#ifdef _WIN32
4
+# define WIN32_LEAN_AND_MEAN
5
+# include <windows.h>
6
+#elif defined(hpux) || defined(__hpux) || defined(_hpux)
7
+# include <sys/pstat.h>
8
+#endif
9
+
10
+/*
11
+ * By doing this in two steps we can at least get
12
+ * the function to be somewhat coherent, even
13
+ * with this disgusting nest of #ifdefs.
14
+ */
15
+#ifndef _SC_NPROCESSORS_ONLN
16
+# ifdef _SC_NPROC_ONLN
17
+# define _SC_NPROCESSORS_ONLN _SC_NPROC_ONLN
18
+# elif defined _SC_CRAY_NCPU
19
+# define _SC_NPROCESSORS_ONLN _SC_CRAY_NCPU
20
+# endif
21
+#endif
22
+
23
+int online_cpus(void)
24
+{
25
+#ifdef _SC_NPROCESSORS_ONLN
26
+ long ncpus;
27
+#endif
28
+
29
+#ifdef _WIN32
30
+ SYSTEM_INFO info;
31
+ GetSystemInfo(&info);
32
+
33
+ if ((int)info.dwNumberOfProcessors > 0)
34
+ return (int)info.dwNumberOfProcessors;
35
+#elif defined(hpux) || defined(__hpux) || defined(_hpux)
36
+ struct pst_dynamic psd;
37
+
38
+ if (!pstat_getdynamic(&psd, sizeof(psd), (size_t)1, 0))
39
+ return (int)psd.psd_proc_cnt;
40
+#endif
41
+
42
+#ifdef _SC_NPROCESSORS_ONLN
43
+ if ((ncpus = (long)sysconf(_SC_NPROCESSORS_ONLN)) > 0)
44
+ return (int)ncpus;
45
+#endif
46
+
47
+ return 1;
48
+}
thread-utils.h
new
+6
@@ -0,0 +1,6 @@
1
+#ifndef THREAD_COMPAT_H
2
+#define THREAD_COMPAT_H
3
+
4
+extern int online_cpus(void);
5
+
6
+#endif /* THREAD_COMPAT_H */