Add cache preload facility
This can do the lstat() storm in parallel, giving potentially much improved performance for cold-cache cases or things like NFS that have weak metadata caching. Just use "read_cache_preload()" instead of "read_cache()" to force an optimistic preload of the index stat data. The function takes a pathspec as its argument, allowing us to preload only the relevant portion of the index. Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Linus Torvalds committed
Nov 13, 2008 at 16:36 UTC
671c9b7e315db89081cc69f83a8f405e4aca37bc
9 files changed
+122
-10
Documentation/config.txt
+9
@@ -413,6 +413,15 @@ data writes properly, but can be useful for filesystems that do not use
413
journalling (traditional UNIX filesystems) or that only journal metadata
414
and not file contents (OS X's HFS+, or Linux ext3 with "data=writeback").
415
416
+core.preloadindex::
417
+ Enable parallel index preload for operations like 'git diff'
418
++
419
+This can speed up operations like 'git diff' and 'git status' especially
420
+on filesystems like NFS that have weak caching semantics and thus
421
+relatively high IO latencies. With this set to 'true', git will do the
422
+index comparison to the filesystem data in parallel, allowing
423
+overlapping IO's.
424
+
425
alias.*::
426
Command aliases for the linkgit:git[1] command wrapper - e.g.
427
after defining "alias.last = cat-file commit HEAD", the invocation
Makefile
+1
@@ -496,6 +496,7 @@ LIB_OBJS += write_or_die.o
496
LIB_OBJS += ws.o
497
LIB_OBJS += wt-status.o
498
LIB_OBJS += xdiff-interface.o
499
+LIB_OBJS += preload-index.o
500
501
BUILTIN_OBJS += builtin-add.o
502
BUILTIN_OBJS += builtin-annotate.o
builtin-commit.c
+4
-4
@@ -225,18 +225,18 @@ static char *prepare_index(int argc, const char **argv, const char *prefix)
225
226
if (interactive) {
227
interactive_add(argc, argv, prefix);
228
- if (read_cache() < 0)
228
+ if (read_cache_preload(NULL) < 0)
229
die("index file corrupt");
230
commit_style = COMMIT_AS_IS;
231
return get_index_file();
232
}
233
234
- if (read_cache() < 0)
235
- die("index file corrupt");
236
-
234
if (*argv)
235
pathspec = get_pathspec(prefix, argv);
236
237
+ if (read_cache_preload(pathspec) < 0)
238
+ die("index file corrupt");
239
+
240
/*
241
* Non partial, non as-is commit.
242
*
builtin-diff-files.c
+2
-2
@@ -59,8 +59,8 @@ int cmd_diff_files(int argc, const char **argv, const char *prefix)
59
(rev.diffopt.output_format & DIFF_FORMAT_PATCH))
60
rev.combine_merges = rev.dense_combined_merges = 1;
61
62
- if (read_cache() < 0) {
63
- perror("read_cache");
62
+ if (read_cache_preload(rev.diffopt.paths) < 0) {
63
+ perror("read_cache_preload");
64
return -1;
65
}
66
result = run_diff_files(&rev, options);
builtin-diff.c
+4
-4
@@ -134,8 +134,8 @@ static int builtin_diff_index(struct rev_info *revs,
134
revs->max_count != -1 || revs->min_age != -1 ||
135
revs->max_age != -1)
136
usage(builtin_diff_usage);
137
- if (read_cache() < 0) {
138
- perror("read_cache");
137
+ if (read_cache_preload(revs->diffopt.paths) < 0) {
138
+ perror("read_cache_preload");
139
return -1;
140
}
141
return run_diff_index(revs, cached);
@@ -234,8 +234,8 @@ static int builtin_diff_files(struct rev_info *revs, int argc, const char **argv
234
revs->combine_merges = revs->dense_combined_merges = 1;
235
236
setup_work_tree();
237
- if (read_cache() < 0) {
238
- perror("read_cache");
237
+ if (read_cache_preload(revs->diffopt.paths) < 0) {
238
+ perror("read_cache_preload");
239
return -1;
240
}
241
result = run_diff_files(revs, options);
cache.h
+3
@@ -262,6 +262,7 @@ static inline void remove_name_hash(struct cache_entry *ce)
262
263
#define read_cache() read_index(&the_index)
264
#define read_cache_from(path) read_index_from(&the_index, (path))
265
+#define read_cache_preload(pathspec) read_index_preload(&the_index, (pathspec))
266
#define is_cache_unborn() is_index_unborn(&the_index)
267
#define read_cache_unmerged() read_index_unmerged(&the_index)
268
#define write_cache(newfd, cache, entries) write_index(&the_index, (newfd))
@@ -368,6 +369,7 @@ extern int init_db(const char *template_dir, unsigned int flags);
369
370
/* Initialize and use the cache information */
371
extern int read_index(struct index_state *);
372
+extern int read_index_preload(struct index_state *, const char **pathspec);
373
extern int read_index_from(struct index_state *, const char *path);
374
extern int is_index_unborn(struct index_state *);
375
extern int read_index_unmerged(struct index_state *);
@@ -458,6 +460,7 @@ extern size_t packed_git_limit;
460
extern size_t delta_base_cache_limit;
461
extern int auto_crlf;
462
extern int fsync_object_files;
463
+extern int core_preload_index;
464
465
enum safe_crlf {
466
SAFE_CRLF_FALSE = 0,
config.c
+5
@@ -490,6 +490,11 @@ static int git_default_core_config(const char *var, const char *value)
490
return 0;
491
}
492
493
+ if (!strcmp(var, "core.preloadindex")) {
494
+ core_preload_index = git_config_bool(var, value);
495
+ return 0;
496
+ }
497
+
498
/* Add other config variables here and to Documentation/config.txt. */
499
return 0;
500
}
environment.c
+3
@@ -43,6 +43,9 @@ unsigned whitespace_rule_cfg = WS_DEFAULT_RULE;
43
enum branch_track git_branch_track = BRANCH_TRACK_REMOTE;
44
enum rebase_setup_type autorebase = AUTOREBASE_NEVER;
45
46
+/* Parallel index stat data preload? */
47
+int core_preload_index = 0;
48
+
49
/* This is set by setup_git_dir_gently() and/or git_default_config() */
50
char *git_work_tree_cfg;
51
static char *work_tree;
preload-index.c
new
+91
@@ -0,0 +1,91 @@
1
+/*
2
+ * Copyright (C) 2008 Linus Torvalds
3
+ */
4
+#include "cache.h"
5
+#include <pthread.h>
6
+
7
+/*
8
+ * Mostly randomly chosen maximum thread counts: we
9
+ * cap the parallelism to 20 threads, and we want
10
+ * to have at least 500 lstat's per thread for it to
11
+ * be worth starting a thread.
12
+ */
13
+#define MAX_PARALLEL (20)
14
+#define THREAD_COST (500)
15
+
16
+struct thread_data {
17
+ pthread_t pthread;
18
+ struct index_state *index;
19
+ const char **pathspec;
20
+ int offset, nr;
21
+};
22
+
23
+static void *preload_thread(void *_data)
24
+{
25
+ int nr;
26
+ struct thread_data *p = _data;
27
+ struct index_state *index = p->index;
28
+ struct cache_entry **cep = index->cache + p->offset;
29
+
30
+ nr = p->nr;
31
+ if (nr + p->offset > index->cache_nr)
32
+ nr = index->cache_nr - p->offset;
33
+
34
+ do {
35
+ struct cache_entry *ce = *cep++;
36
+ struct stat st;
37
+
38
+ if (ce_stage(ce))
39
+ continue;
40
+ if (ce_uptodate(ce))
41
+ continue;
42
+ if (!ce_path_match(ce, p->pathspec))
43
+ continue;
44
+ if (lstat(ce->name, &st))
45
+ continue;
46
+ if (ie_match_stat(index, ce, &st, 0))
47
+ continue;
48
+ ce_mark_uptodate(ce);
49
+ } while (--nr > 0);
50
+ return NULL;
51
+}
52
+
53
+static void preload_index(struct index_state *index, const char **pathspec)
54
+{
55
+ int threads, i, work, offset;
56
+ struct thread_data data[MAX_PARALLEL];
57
+
58
+ if (!core_preload_index)
59
+ return;
60
+
61
+ threads = index->cache_nr / THREAD_COST;
62
+ if (threads < 2)
63
+ return;
64
+ if (threads > MAX_PARALLEL)
65
+ threads = MAX_PARALLEL;
66
+ offset = 0;
67
+ work = (index->cache_nr + threads - 1) / threads;
68
+ for (i = 0; i < threads; i++) {
69
+ struct thread_data *p = data+i;
70
+ p->index = index;
71
+ p->pathspec = pathspec;
72
+ p->offset = offset;
73
+ p->nr = work;
74
+ offset += work;
75
+ if (pthread_create(&p->pthread, NULL, preload_thread, p))
76
+ die("unable to create threaded lstat");
77
+ }
78
+ for (i = 0; i < threads; i++) {
79
+ struct thread_data *p = data+i;
80
+ if (pthread_join(p->pthread, NULL))
81
+ die("unable to join threaded lstat");
82
+ }
83
+}
84
+
85
+int read_index_preload(struct index_state *index, const char **pathspec)
86
+{
87
+ int retval = read_index(index);
88
+
89
+ preload_index(index, pathspec);
90
+ return retval;
91
+}