read-cache: load cache extensions on a worker thread
This patch helps address the CPU cost of loading the index by loading the cache extensions on a worker thread in parallel with loading the cache entries. In some cases, loading the extensions takes longer than loading the cache entries so this patch utilizes the new EOIE to start the thread to load the extensions before loading all the cache entries in parallel. This is possible because the current extensions don't access the cache entries in the index_state structure so are OK that they don't all exist yet. The CACHE_EXT_TREE, CACHE_EXT_RESOLVE_UNDO, and CACHE_EXT_UNTRACKED extensions don't even get a pointer to the index so don't have access to the cache entries. CACHE_EXT_LINK only uses the index_state to initialize the split index. CACHE_EXT_FSMONITOR only uses the index_state to save the fsmonitor last update and dirty flags. I used p0002-read-cache.sh to generate some performance data: Test w/100,000 files reduced the time by 0.53% Test w/1,000,000 files reduced the time by 27.78% Signed-off-by: Ben Peart <benpeart@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Ben Peart committed
Oct 10, 2018 at 11:59 UTC
abb4bb83845dc012ffe1c04750d1a09edd598a82
1 file changed
+79
-16
read-cache.c
+79
-16
@@ -23,6 +23,7 @@
23
#include "split-index.h"
24
#include "utf8.h"
25
#include "fsmonitor.h"
26
+#include "thread-utils.h"
27
28
/* Mask for the name length in ce_flags in the on-disk index */
29
@@ -1890,6 +1891,44 @@ static size_t estimate_cache_size(size_t ondisk_size, unsigned int entries)
1891
static size_t read_eoie_extension(const char *mmap, size_t mmap_size);
1892
static void write_eoie_extension(struct strbuf *sb, git_hash_ctx *eoie_context, size_t offset);
1893
1894
+struct load_index_extensions
1895
+{
1896
+#ifndef NO_PTHREADS
1897
+ pthread_t pthread;
1898
+#endif
1899
+ struct index_state *istate;
1900
+ const char *mmap;
1901
+ size_t mmap_size;
1902
+ unsigned long src_offset;
1903
+};
1904
+
1905
+static void *load_index_extensions(void *_data)
1906
+{
1907
+ struct load_index_extensions *p = _data;
1908
+ unsigned long src_offset = p->src_offset;
1909
+
1910
+ while (src_offset <= p->mmap_size - the_hash_algo->rawsz - 8) {
1911
+ /* After an array of active_nr index entries,
1912
+ * there can be arbitrary number of extended
1913
+ * sections, each of which is prefixed with
1914
+ * extension name (4-byte) and section length
1915
+ * in 4-byte network byte order.
1916
+ */
1917
+ uint32_t extsize = get_be32(p->mmap + src_offset + 4);
1918
+ if (read_index_extension(p->istate,
1919
+ p->mmap + src_offset,
1920
+ p->mmap + src_offset + 8,
1921
+ extsize) < 0) {
1922
+ munmap((void *)p->mmap, p->mmap_size);
1923
+ die(_("index file corrupt"));
1924
+ }
1925
+ src_offset += 8;
1926
+ src_offset += extsize;
1927
+ }
1928
+
1929
+ return NULL;
1930
+}
1931
+
1932
/* remember to discard_cache() before reading a different cache! */
1933
int do_read_index(struct index_state *istate, const char *path, int must_exist)
1934
{
@@ -1900,6 +1939,11 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)
1939
const char *mmap;
1940
size_t mmap_size;
1941
const struct cache_entry *previous_ce = NULL;
1942
+ struct load_index_extensions p;
1943
+ size_t extension_offset = 0;
1944
+#ifndef NO_PTHREADS
1945
+ int nr_threads;
1946
+#endif
1947
1948
if (istate->initialized)
1949
return istate->cache_nr;
@@ -1936,6 +1980,30 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)
1980
istate->cache = xcalloc(istate->cache_alloc, sizeof(*istate->cache));
1981
istate->initialized = 1;
1982
1983
+ p.istate = istate;
1984
+ p.mmap = mmap;
1985
+ p.mmap_size = mmap_size;
1986
+
1987
+#ifndef NO_PTHREADS
1988
+ nr_threads = git_config_get_index_threads();
1989
+ if (!nr_threads)
1990
+ nr_threads = online_cpus();
1991
+
1992
+ if (nr_threads > 1) {
1993
+ extension_offset = read_eoie_extension(mmap, mmap_size);
1994
+ if (extension_offset) {
1995
+ int err;
1996
+
1997
+ p.src_offset = extension_offset;
1998
+ err = pthread_create(&p.pthread, NULL, load_index_extensions, &p);
1999
+ if (err)
2000
+ die(_("unable to create load_index_extensions thread: %s"), strerror(err));
2001
+
2002
+ nr_threads--;
2003
+ }
2004
+ }
2005
+#endif
2006
+
2007
if (istate->version == 4) {
2008
mem_pool_init(&istate->ce_mem_pool,
2009
estimate_cache_size_from_compressed(istate->cache_nr));
@@ -1960,22 +2028,17 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)
2028
istate->timestamp.sec = st.st_mtime;
2029
istate->timestamp.nsec = ST_MTIME_NSEC(st);
2030
1963
- while (src_offset <= mmap_size - the_hash_algo->rawsz - 8) {
1964
- /* After an array of active_nr index entries,
1965
- * there can be arbitrary number of extended
1966
- * sections, each of which is prefixed with
1967
- * extension name (4-byte) and section length
1968
- * in 4-byte network byte order.
1969
- */
1970
- uint32_t extsize;
1971
- extsize = get_be32(mmap + src_offset + 4);
1972
- if (read_index_extension(istate,
1973
- mmap + src_offset,
1974
- mmap + src_offset + 8,
1975
- extsize) < 0)
1976
- goto unmap;
1977
- src_offset += 8;
1978
- src_offset += extsize;
2031
+ /* if we created a thread, join it otherwise load the extensions on the primary thread */
2032
+#ifndef NO_PTHREADS
2033
+ if (extension_offset) {
2034
+ int ret = pthread_join(p.pthread, NULL);
2035
+ if (ret)
2036
+ die(_("unable to join load_index_extensions thread: %s"), strerror(ret));
2037
+ }
2038
+#endif
2039
+ if (!extension_offset) {
2040
+ p.src_offset = src_offset;
2041
+ load_index_extensions(&p);
2042
}
2043
munmap((void *)mmap, mmap_size);
2044
return istate->cache_nr;