name-hash: add test-lazy-init-name-hash
Add t/helper/test-lazy-init-name-hash.c test code to demonstrate performance times for lazy_init_name_hash() using the original single-threaded and the new multi-threaded code paths. Includes a --dump option to dump the created hashmaps to stdout. You can use this to run both code paths and confirm that they generate the same hashmaps. Includes a --analyze option to analyze performance of both code paths over a range of index sizes to help you find a lower bound for the LAZY_THREAD_COST in name-hash.c. For example, passing "-a 4000" will set "istate.cache_nr" to 4000 and then try the multi-threaded code -- probably giving 2 threads with 2000 entries each. It will then run both the single-threaded (1x4000) and the multi-threaded (2x2000) and compare the times. It will then repeat the test with 8000, 12000, and etc. so that you can see the cross over. Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jeff Hostetler committed
Mar 23, 2017 at 13:47 UTC
ea194895326d10f6e255eee05ac25ab5ae096dd9
3 files changed
+266
Makefile
+1
@@ -614,6 +614,7 @@ TEST_PROGRAMS_NEED_X += test-fake-ssh
614
TEST_PROGRAMS_NEED_X += test-genrandom
615
TEST_PROGRAMS_NEED_X += test-hashmap
616
TEST_PROGRAMS_NEED_X += test-index-version
617
+TEST_PROGRAMS_NEED_X += test-lazy-init-name-hash
618
TEST_PROGRAMS_NEED_X += test-line-buffer
619
TEST_PROGRAMS_NEED_X += test-match-trees
620
TEST_PROGRAMS_NEED_X += test-mergesort
cache.h
+1
@@ -343,6 +343,7 @@ struct index_state {
343
extern struct index_state the_index;
344
345
/* Name hashing */
346
+extern int test_lazy_init_name_hash(struct index_state *istate, int try_threaded);
347
extern void add_name_hash(struct index_state *istate, struct cache_entry *ce);
348
extern void remove_name_hash(struct index_state *istate, struct cache_entry *ce);
349
extern void free_name_hash(struct index_state *istate);
t/helper/test-lazy-init-name-hash.c
new
+264
@@ -0,0 +1,264 @@
1
+#include "cache.h"
2
+#include "parse-options.h"
3
+
4
+static int single;
5
+static int multi;
6
+static int count = 1;
7
+static int dump;
8
+static int perf;
9
+static int analyze;
10
+static int analyze_step;
11
+
12
+/*
13
+ * Dump the contents of the "dir" and "name" hash tables to stdout.
14
+ * If you sort the result, you can compare it with the other type
15
+ * mode and verify that both single and multi produce the same set.
16
+ */
17
+static void dump_run(void)
18
+{
19
+ struct hashmap_iter iter_dir;
20
+ struct hashmap_iter iter_cache;
21
+
22
+ /* Stolen from name-hash.c */
23
+ struct dir_entry {
24
+ struct hashmap_entry ent;
25
+ struct dir_entry *parent;
26
+ int nr;
27
+ unsigned int namelen;
28
+ char name[FLEX_ARRAY];
29
+ };
30
+
31
+ struct dir_entry *dir;
32
+ struct cache_entry *ce;
33
+
34
+ read_cache();
35
+ if (single) {
36
+ test_lazy_init_name_hash(&the_index, 0);
37
+ } else {
38
+ int nr_threads_used = test_lazy_init_name_hash(&the_index, 1);
39
+ if (!nr_threads_used)
40
+ die("non-threaded code path used");
41
+ }
42
+
43
+ dir = hashmap_iter_first(&the_index.dir_hash, &iter_dir);
44
+ while (dir) {
45
+ printf("dir %08x %7d %s\n", dir->ent.hash, dir->nr, dir->name);
46
+ dir = hashmap_iter_next(&iter_dir);
47
+ }
48
+
49
+ ce = hashmap_iter_first(&the_index.name_hash, &iter_cache);
50
+ while (ce) {
51
+ printf("name %08x %s\n", ce->ent.hash, ce->name);
52
+ ce = hashmap_iter_next(&iter_cache);
53
+ }
54
+
55
+ discard_cache();
56
+}
57
+
58
+/*
59
+ * Run the single or multi threaded version "count" times and
60
+ * report on the time taken.
61
+ */
62
+static uint64_t time_runs(int try_threaded)
63
+{
64
+ uint64_t t0, t1, t2;
65
+ uint64_t sum = 0;
66
+ uint64_t avg;
67
+ int nr_threads_used;
68
+ int i;
69
+
70
+ for (i = 0; i < count; i++) {
71
+ t0 = getnanotime();
72
+ read_cache();
73
+ t1 = getnanotime();
74
+ nr_threads_used = test_lazy_init_name_hash(&the_index, try_threaded);
75
+ t2 = getnanotime();
76
+
77
+ sum += (t2 - t1);
78
+
79
+ if (try_threaded && !nr_threads_used)
80
+ die("non-threaded code path used");
81
+
82
+ if (nr_threads_used)
83
+ printf("%f %f %d multi %d\n",
84
+ ((double)(t1 - t0))/1000000000,
85
+ ((double)(t2 - t1))/1000000000,
86
+ the_index.cache_nr,
87
+ nr_threads_used);
88
+ else
89
+ printf("%f %f %d single\n",
90
+ ((double)(t1 - t0))/1000000000,
91
+ ((double)(t2 - t1))/1000000000,
92
+ the_index.cache_nr);
93
+ fflush(stdout);
94
+
95
+ discard_cache();
96
+ }
97
+
98
+ avg = sum / count;
99
+ if (count > 1)
100
+ printf("avg %f %s\n",
101
+ (double)avg/1000000000,
102
+ (try_threaded) ? "multi" : "single");
103
+
104
+ return avg;
105
+}
106
+
107
+/*
108
+ * Try a series of runs varying the "istate->cache_nr" and
109
+ * try to find a good value for the multi-threaded criteria.
110
+ */
111
+static void analyze_run(void)
112
+{
113
+ uint64_t t1s, t1m, t2s, t2m;
114
+ int cache_nr_limit;
115
+ int nr_threads_used;
116
+ int i;
117
+ int nr;
118
+
119
+ read_cache();
120
+ cache_nr_limit = the_index.cache_nr;
121
+ discard_cache();
122
+
123
+ nr = analyze;
124
+ while (1) {
125
+ uint64_t sum_single = 0;
126
+ uint64_t sum_multi = 0;
127
+ uint64_t avg_single;
128
+ uint64_t avg_multi;
129
+
130
+ if (nr > cache_nr_limit)
131
+ nr = cache_nr_limit;
132
+
133
+ for (i = 0; i < count; i++) {
134
+ read_cache();
135
+ the_index.cache_nr = nr; /* cheap truncate of index */
136
+ t1s = getnanotime();
137
+ test_lazy_init_name_hash(&the_index, 0);
138
+ t2s = getnanotime();
139
+ sum_single += (t2s - t1s);
140
+ the_index.cache_nr = cache_nr_limit;
141
+ discard_cache();
142
+
143
+ read_cache();
144
+ the_index.cache_nr = nr; /* cheap truncate of index */
145
+ t1m = getnanotime();
146
+ nr_threads_used = test_lazy_init_name_hash(&the_index, 1);
147
+ t2m = getnanotime();
148
+ sum_multi += (t2m - t1m);
149
+ the_index.cache_nr = cache_nr_limit;
150
+ discard_cache();
151
+
152
+ if (!nr_threads_used)
153
+ printf(" [size %8d] [single %f] non-threaded code path used\n",
154
+ nr, ((double)(t2s - t1s))/1000000000);
155
+ else
156
+ printf(" [size %8d] [single %f] %c [multi %f %d]\n",
157
+ nr,
158
+ ((double)(t2s - t1s))/1000000000,
159
+ (((t2s - t1s) < (t2m - t1m)) ? '<' : '>'),
160
+ ((double)(t2m - t1m))/1000000000,
161
+ nr_threads_used);
162
+ fflush(stdout);
163
+ }
164
+ if (count > 1) {
165
+ avg_single = sum_single / count;
166
+ avg_multi = sum_multi / count;
167
+ if (!nr_threads_used)
168
+ printf("avg [size %8d] [single %f]\n",
169
+ nr,
170
+ (double)avg_single/1000000000);
171
+ else
172
+ printf("avg [size %8d] [single %f] %c [multi %f %d]\n",
173
+ nr,
174
+ (double)avg_single/1000000000,
175
+ (avg_single < avg_multi ? '<' : '>'),
176
+ (double)avg_multi/1000000000,
177
+ nr_threads_used);
178
+ fflush(stdout);
179
+ }
180
+
181
+ if (nr >= cache_nr_limit)
182
+ return;
183
+ nr += analyze_step;
184
+ }
185
+}
186
+
187
+int cmd_main(int argc, const char **argv)
188
+{
189
+ const char *usage[] = {
190
+ "test-lazy-init-name-hash -d (-s | -m)",
191
+ "test-lazy-init-name-hash -p [-c c]",
192
+ "test-lazy-init-name-hash -a a [--step s] [-c c]",
193
+ "test-lazy-init-name-hash (-s | -m) [-c c]",
194
+ "test-lazy-init-name-hash -s -m [-c c]",
195
+ NULL
196
+ };
197
+ struct option options[] = {
198
+ OPT_BOOL('s', "single", &single, "run single-threaded code"),
199
+ OPT_BOOL('m', "multi", &multi, "run multi-threaded code"),
200
+ OPT_INTEGER('c', "count", &count, "number of passes"),
201
+ OPT_BOOL('d', "dump", &dump, "dump hash tables"),
202
+ OPT_BOOL('p', "perf", &perf, "compare single vs multi"),
203
+ OPT_INTEGER('a', "analyze", &analyze, "analyze different multi sizes"),
204
+ OPT_INTEGER(0, "step", &analyze_step, "analyze step factor"),
205
+ OPT_END(),
206
+ };
207
+ const char *prefix;
208
+ uint64_t avg_single, avg_multi;
209
+
210
+ prefix = setup_git_directory();
211
+
212
+ argc = parse_options(argc, argv, prefix, options, usage, 0);
213
+
214
+ /*
215
+ * istate->dir_hash is only created when ignore_case is set.
216
+ */
217
+ ignore_case = 1;
218
+
219
+ if (dump) {
220
+ if (perf || analyze > 0)
221
+ die("cannot combine dump, perf, or analyze");
222
+ if (count > 1)
223
+ die("count not valid with dump");
224
+ if (single && multi)
225
+ die("cannot use both single and multi with dump");
226
+ if (!single && !multi)
227
+ die("dump requires either single or multi");
228
+ dump_run();
229
+ return 0;
230
+ }
231
+
232
+ if (perf) {
233
+ if (analyze > 0)
234
+ die("cannot combine dump, perf, or analyze");
235
+ if (single || multi)
236
+ die("cannot use single or multi with perf");
237
+ avg_single = time_runs(0);
238
+ avg_multi = time_runs(1);
239
+ if (avg_multi > avg_single)
240
+ die("multi is slower");
241
+ return 0;
242
+ }
243
+
244
+ if (analyze) {
245
+ if (analyze < 500)
246
+ die("analyze must be at least 500");
247
+ if (!analyze_step)
248
+ analyze_step = analyze;
249
+ if (single || multi)
250
+ die("cannot use single or multi with analyze");
251
+ analyze_run();
252
+ return 0;
253
+ }
254
+
255
+ if (!single && !multi)
256
+ die("require either -s or -m or both");
257
+
258
+ if (single)
259
+ time_runs(0);
260
+ if (multi)
261
+ time_runs(1);
262
+
263
+ return 0;
264
+}