name-hash: perf improvement for lazy_init_name_hash
Improve performance of lazy_init_name_hash() when ignore_case is set. Teach name-hash to build the istate.name_hash and istate.dir_hash simultaneously using a forward-diving technique on the pathname of the index_entry, rather than adding name_hash entries and then searching backwards in the pathname for parent directories. This borrows algorithm ideas from clear_ce_flags_{1,dir}. Multiple threads are used with the new algorithm to speed hashmap construction. This new code path is only used when threads are present (a compiler settings) and when the index is large enough to warrant the pthread complexity. The code in clear_ce_flags_dir() uses a linear search to find the adjacent index entries with the same prefix; a binary search is used here handle_range_dir() to further speed things up. The size of LAZY_THREAD_COST was determined from rough analysis using: t/helper/test-lazy-init-name-hash --analyze Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jeff Hostetler committed
Mar 23, 2017 at 13:47 UTC
846df809bc671d6487fa2be7550e3ed2b62e62f9
1 file changed
+485
-7
name-hash.c
+485
-7
@@ -23,15 +23,21 @@ static int dir_entry_cmp(const struct dir_entry *e1,
23
name ? name : e2->name, e1->namelen);
24
}
25
26
-static struct dir_entry *find_dir_entry(struct index_state *istate,
27
- const char *name, unsigned int namelen)
26
+static struct dir_entry *find_dir_entry__hash(struct index_state *istate,
27
+ const char *name, unsigned int namelen, unsigned int hash)
28
{
29
struct dir_entry key;
30
- hashmap_entry_init(&key, memihash(name, namelen));
30
+ hashmap_entry_init(&key, hash);
31
key.namelen = namelen;
32
return hashmap_get(&istate->dir_hash, &key, name);
33
}
34
35
+static struct dir_entry *find_dir_entry(struct index_state *istate,
36
+ const char *name, unsigned int namelen)
37
+{
38
+ return find_dir_entry__hash(istate, name, namelen, memihash(name, namelen));
39
+}
40
+
41
static struct dir_entry *hash_dir_entry(struct index_state *istate,
42
struct cache_entry *ce, int namelen)
43
{
@@ -112,21 +118,493 @@ static int cache_entry_cmp(const struct cache_entry *ce1,
118
return remove ? !(ce1 == ce2) : 0;
119
}
120
115
-static void lazy_init_name_hash(struct index_state *istate)
121
+static int lazy_try_threaded = 1;
122
+static int lazy_nr_dir_threads;
123
+
124
+#ifdef NO_PTHREADS
125
+
126
+static inline int lookup_lazy_params(struct index_state *istate)
127
{
117
- int nr;
128
+ return 0;
129
+}
130
+
131
+static inline void threaded_lazy_init_name_hash(
132
+ struct index_state *istate)
133
+{
134
+}
135
+
136
+#else
137
+
138
+#include "thread-utils.h"
139
+
140
+/*
141
+ * Set a minimum number of cache_entries that we will handle per
142
+ * thread and use that to decide how many threads to run (upto
143
+ * the number on the system).
144
+ *
145
+ * For guidance setting the lower per-thread bound, see:
146
+ * t/helper/test-lazy-init-name-hash --analyze
147
+ */
148
+#define LAZY_THREAD_COST (2000)
149
+
150
+/*
151
+ * We use n mutexes to guard n partitions of the "istate->dir_hash"
152
+ * hashtable. Since "find" and "insert" operations will hash to a
153
+ * particular bucket and modify/search a single chain, we can say
154
+ * that "all chains mod n" are guarded by the same mutex -- rather
155
+ * than having a single mutex to guard the entire table. (This does
156
+ * require that we disable "rehashing" on the hashtable.)
157
+ *
158
+ * So, a larger value here decreases the probability of a collision
159
+ * and the time that each thread must wait for the mutex.
160
+ */
161
+#define LAZY_MAX_MUTEX (32)
162
+
163
+static pthread_mutex_t *lazy_dir_mutex_array;
164
+
165
+/*
166
+ * An array of lazy_entry items is used by the n threads in
167
+ * the directory parse (first) phase to (lock-free) store the
168
+ * intermediate results. These values are then referenced by
169
+ * the 2 threads in the second phase.
170
+ */
171
+struct lazy_entry {
172
+ struct dir_entry *dir;
173
+ unsigned int hash_dir;
174
+ unsigned int hash_name;
175
+};
176
+
177
+/*
178
+ * Decide if we want to use threads (if available) to load
179
+ * the hash tables. We set "lazy_nr_dir_threads" to zero when
180
+ * it is not worth it.
181
+ */
182
+static int lookup_lazy_params(struct index_state *istate)
183
+{
184
+ int nr_cpus;
185
+
186
+ lazy_nr_dir_threads = 0;
187
+
188
+ if (!lazy_try_threaded)
189
+ return 0;
190
+
191
+ /*
192
+ * If we are respecting case, just use the original
193
+ * code to build the "istate->name_hash". We don't
194
+ * need the complexity here.
195
+ */
196
+ if (!ignore_case)
197
+ return 0;
198
+
199
+ nr_cpus = online_cpus();
200
+ if (nr_cpus < 2)
201
+ return 0;
202
+
203
+ if (istate->cache_nr < 2 * LAZY_THREAD_COST)
204
+ return 0;
205
+
206
+ if (istate->cache_nr < nr_cpus * LAZY_THREAD_COST)
207
+ nr_cpus = istate->cache_nr / LAZY_THREAD_COST;
208
+ lazy_nr_dir_threads = nr_cpus;
209
+ return lazy_nr_dir_threads;
210
+}
211
+
212
+/*
213
+ * Initialize n mutexes for use when searching and inserting
214
+ * into "istate->dir_hash". All "dir" threads are trying
215
+ * to insert partial pathnames into the hash as they iterate
216
+ * over their portions of the index, so lock contention is
217
+ * high.
218
+ *
219
+ * However, the hashmap is going to put items into bucket
220
+ * chains based on their hash values. Use that to create n
221
+ * mutexes and lock on mutex[bucket(hash) % n]. This will
222
+ * decrease the collision rate by (hopefully) by a factor of n.
223
+ */
224
+static void init_dir_mutex(void)
225
+{
226
+ int j;
227
+
228
+ lazy_dir_mutex_array = xcalloc(LAZY_MAX_MUTEX, sizeof(pthread_mutex_t));
229
+
230
+ for (j = 0; j < LAZY_MAX_MUTEX; j++)
231
+ init_recursive_mutex(&lazy_dir_mutex_array[j]);
232
+}
233
+
234
+static void cleanup_dir_mutex(void)
235
+{
236
+ int j;
237
+
238
+ for (j = 0; j < LAZY_MAX_MUTEX; j++)
239
+ pthread_mutex_destroy(&lazy_dir_mutex_array[j]);
240
+
241
+ free(lazy_dir_mutex_array);
242
+}
243
+
244
+static void lock_dir_mutex(int j)
245
+{
246
+ pthread_mutex_lock(&lazy_dir_mutex_array[j]);
247
+}
248
+
249
+static void unlock_dir_mutex(int j)
250
+{
251
+ pthread_mutex_unlock(&lazy_dir_mutex_array[j]);
252
+}
253
+
254
+static inline int compute_dir_lock_nr(
255
+ const struct hashmap *map,
256
+ unsigned int hash)
257
+{
258
+ return hashmap_bucket(map, hash) % LAZY_MAX_MUTEX;
259
+}
260
+
261
+static struct dir_entry *hash_dir_entry_with_parent_and_prefix(
262
+ struct index_state *istate,
263
+ struct dir_entry *parent,
264
+ struct strbuf *prefix)
265
+{
266
+ struct dir_entry *dir;
267
+ unsigned int hash;
268
+ int lock_nr;
269
+
270
+ /*
271
+ * Either we have a parent directory and path with slash(es)
272
+ * or the directory is an immediate child of the root directory.
273
+ */
274
+ assert((parent != NULL) ^ (strchr(prefix->buf, '/') == NULL));
275
+
276
+ if (parent)
277
+ hash = memihash_cont(parent->ent.hash,
278
+ prefix->buf + parent->namelen,
279
+ prefix->len - parent->namelen);
280
+ else
281
+ hash = memihash(prefix->buf, prefix->len);
282
+
283
+ lock_nr = compute_dir_lock_nr(&istate->dir_hash, hash);
284
+ lock_dir_mutex(lock_nr);
285
+
286
+ dir = find_dir_entry__hash(istate, prefix->buf, prefix->len, hash);
287
+ if (!dir) {
288
+ FLEX_ALLOC_MEM(dir, name, prefix->buf, prefix->len);
289
+ hashmap_entry_init(dir, hash);
290
+ dir->namelen = prefix->len;
291
+ dir->parent = parent;
292
+ hashmap_add(&istate->dir_hash, dir);
293
+
294
+ if (parent) {
295
+ unlock_dir_mutex(lock_nr);
296
+
297
+ /* All I really need here is an InterlockedIncrement(&(parent->nr)) */
298
+ lock_nr = compute_dir_lock_nr(&istate->dir_hash, parent->ent.hash);
299
+ lock_dir_mutex(lock_nr);
300
+ parent->nr++;
301
+ }
302
+ }
303
304
+ unlock_dir_mutex(lock_nr);
305
+
306
+ return dir;
307
+}
308
+
309
+/*
310
+ * handle_range_1() and handle_range_dir() are derived from
311
+ * clear_ce_flags_1() and clear_ce_flags_dir() in unpack-trees.c
312
+ * and handle the iteration over the entire array of index entries.
313
+ * They use recursion for adjacent entries in the same parent
314
+ * directory.
315
+ */
316
+static int handle_range_1(
317
+ struct index_state *istate,
318
+ int k_start,
319
+ int k_end,
320
+ struct dir_entry *parent,
321
+ struct strbuf *prefix,
322
+ struct lazy_entry *lazy_entries);
323
+
324
+static int handle_range_dir(
325
+ struct index_state *istate,
326
+ int k_start,
327
+ int k_end,
328
+ struct dir_entry *parent,
329
+ struct strbuf *prefix,
330
+ struct lazy_entry *lazy_entries,
331
+ struct dir_entry **dir_new_out)
332
+{
333
+ int rc, k;
334
+ int input_prefix_len = prefix->len;
335
+ struct dir_entry *dir_new;
336
+
337
+ dir_new = hash_dir_entry_with_parent_and_prefix(istate, parent, prefix);
338
+
339
+ strbuf_addch(prefix, '/');
340
+
341
+ /*
342
+ * Scan forward in the index array for index entries having the same
343
+ * path prefix (that are also in this directory).
344
+ */
345
+ if (strncmp(istate->cache[k_start + 1]->name, prefix->buf, prefix->len) > 0)
346
+ k = k_start + 1;
347
+ else if (strncmp(istate->cache[k_end - 1]->name, prefix->buf, prefix->len) == 0)
348
+ k = k_end;
349
+ else {
350
+ int begin = k_start;
351
+ int end = k_end;
352
+ while (begin < end) {
353
+ int mid = (begin + end) >> 1;
354
+ int cmp = strncmp(istate->cache[mid]->name, prefix->buf, prefix->len);
355
+ if (cmp == 0) /* mid has same prefix; look in second part */
356
+ begin = mid + 1;
357
+ else if (cmp > 0) /* mid is past group; look in first part */
358
+ end = mid;
359
+ else
360
+ die("cache entry out of order");
361
+ }
362
+ k = begin;
363
+ }
364
+
365
+ /*
366
+ * Recurse and process what we can of this subset [k_start, k).
367
+ */
368
+ rc = handle_range_1(istate, k_start, k, dir_new, prefix, lazy_entries);
369
+
370
+ strbuf_setlen(prefix, input_prefix_len);
371
+
372
+ *dir_new_out = dir_new;
373
+ return rc;
374
+}
375
+
376
+static int handle_range_1(
377
+ struct index_state *istate,
378
+ int k_start,
379
+ int k_end,
380
+ struct dir_entry *parent,
381
+ struct strbuf *prefix,
382
+ struct lazy_entry *lazy_entries)
383
+{
384
+ int input_prefix_len = prefix->len;
385
+ int k = k_start;
386
+
387
+ while (k < k_end) {
388
+ struct cache_entry *ce_k = istate->cache[k];
389
+ const char *name, *slash;
390
+
391
+ if (prefix->len && strncmp(ce_k->name, prefix->buf, prefix->len))
392
+ break;
393
+
394
+ name = ce_k->name + prefix->len;
395
+ slash = strchr(name, '/');
396
+
397
+ if (slash) {
398
+ int len = slash - name;
399
+ int processed;
400
+ struct dir_entry *dir_new;
401
+
402
+ strbuf_add(prefix, name, len);
403
+ processed = handle_range_dir(istate, k, k_end, parent, prefix, lazy_entries, &dir_new);
404
+ if (processed) {
405
+ k += processed;
406
+ strbuf_setlen(prefix, input_prefix_len);
407
+ continue;
408
+ }
409
+
410
+ strbuf_addch(prefix, '/');
411
+ processed = handle_range_1(istate, k, k_end, dir_new, prefix, lazy_entries);
412
+ k += processed;
413
+ strbuf_setlen(prefix, input_prefix_len);
414
+ continue;
415
+ }
416
+
417
+ /*
418
+ * It is too expensive to take a lock to insert "ce_k"
419
+ * into "istate->name_hash" and increment the ref-count
420
+ * on the "parent" dir. So we defer actually updating
421
+ * permanent data structures until phase 2 (where we
422
+ * can change the locking requirements) and simply
423
+ * accumulate our current results into the lazy_entries
424
+ * data array).
425
+ *
426
+ * We do not need to lock the lazy_entries array because
427
+ * we have exclusive access to the cells in the range
428
+ * [k_start,k_end) that this thread was given.
429
+ */
430
+ lazy_entries[k].dir = parent;
431
+ if (parent) {
432
+ lazy_entries[k].hash_name = memihash_cont(
433
+ parent->ent.hash,
434
+ ce_k->name + parent->namelen,
435
+ ce_namelen(ce_k) - parent->namelen);
436
+ lazy_entries[k].hash_dir = parent->ent.hash;
437
+ } else {
438
+ lazy_entries[k].hash_name = memihash(ce_k->name, ce_namelen(ce_k));
439
+ }
440
+
441
+ k++;
442
+ }
443
+
444
+ return k - k_start;
445
+}
446
+
447
+struct lazy_dir_thread_data {
448
+ pthread_t pthread;
449
+ struct index_state *istate;
450
+ struct lazy_entry *lazy_entries;
451
+ int k_start;
452
+ int k_end;
453
+};
454
+
455
+static void *lazy_dir_thread_proc(void *_data)
456
+{
457
+ struct lazy_dir_thread_data *d = _data;
458
+ struct strbuf prefix = STRBUF_INIT;
459
+ handle_range_1(d->istate, d->k_start, d->k_end, NULL, &prefix, d->lazy_entries);
460
+ strbuf_release(&prefix);
461
+ return NULL;
462
+}
463
+
464
+struct lazy_name_thread_data {
465
+ pthread_t pthread;
466
+ struct index_state *istate;
467
+ struct lazy_entry *lazy_entries;
468
+};
469
+
470
+static void *lazy_name_thread_proc(void *_data)
471
+{
472
+ struct lazy_name_thread_data *d = _data;
473
+ int k;
474
+
475
+ for (k = 0; k < d->istate->cache_nr; k++) {
476
+ struct cache_entry *ce_k = d->istate->cache[k];
477
+ ce_k->ce_flags |= CE_HASHED;
478
+ hashmap_entry_init(ce_k, d->lazy_entries[k].hash_name);
479
+ hashmap_add(&d->istate->name_hash, ce_k);
480
+ }
481
+
482
+ return NULL;
483
+}
484
+
485
+static inline void lazy_update_dir_ref_counts(
486
+ struct index_state *istate,
487
+ struct lazy_entry *lazy_entries)
488
+{
489
+ int k;
490
+
491
+ for (k = 0; k < istate->cache_nr; k++) {
492
+ if (lazy_entries[k].dir)
493
+ lazy_entries[k].dir->nr++;
494
+ }
495
+}
496
+
497
+static void threaded_lazy_init_name_hash(
498
+ struct index_state *istate)
499
+{
500
+ int nr_each;
501
+ int k_start;
502
+ int t;
503
+ struct lazy_entry *lazy_entries;
504
+ struct lazy_dir_thread_data *td_dir;
505
+ struct lazy_name_thread_data *td_name;
506
+
507
+ k_start = 0;
508
+ nr_each = DIV_ROUND_UP(istate->cache_nr, lazy_nr_dir_threads);
509
+
510
+ lazy_entries = xcalloc(istate->cache_nr, sizeof(struct lazy_entry));
511
+ td_dir = xcalloc(lazy_nr_dir_threads, sizeof(struct lazy_dir_thread_data));
512
+ td_name = xcalloc(1, sizeof(struct lazy_name_thread_data));
513
+
514
+ init_dir_mutex();
515
+
516
+ /*
517
+ * Phase 1:
518
+ * Build "istate->dir_hash" using n "dir" threads (and a read-only index).
519
+ */
520
+ for (t = 0; t < lazy_nr_dir_threads; t++) {
521
+ struct lazy_dir_thread_data *td_dir_t = td_dir + t;
522
+ td_dir_t->istate = istate;
523
+ td_dir_t->lazy_entries = lazy_entries;
524
+ td_dir_t->k_start = k_start;
525
+ k_start += nr_each;
526
+ if (k_start > istate->cache_nr)
527
+ k_start = istate->cache_nr;
528
+ td_dir_t->k_end = k_start;
529
+ if (pthread_create(&td_dir_t->pthread, NULL, lazy_dir_thread_proc, td_dir_t))
530
+ die("unable to create lazy_dir_thread");
531
+ }
532
+ for (t = 0; t < lazy_nr_dir_threads; t++) {
533
+ struct lazy_dir_thread_data *td_dir_t = td_dir + t;
534
+ if (pthread_join(td_dir_t->pthread, NULL))
535
+ die("unable to join lazy_dir_thread");
536
+ }
537
+
538
+ /*
539
+ * Phase 2:
540
+ * Iterate over all index entries and add them to the "istate->name_hash"
541
+ * using a single "name" background thread.
542
+ * (Testing showed it wasn't worth running more than 1 thread for this.)
543
+ *
544
+ * Meanwhile, finish updating the parent directory ref-counts for each
545
+ * index entry using the current thread. (This step is very fast and
546
+ * doesn't need threading.)
547
+ */
548
+ td_name->istate = istate;
549
+ td_name->lazy_entries = lazy_entries;
550
+ if (pthread_create(&td_name->pthread, NULL, lazy_name_thread_proc, td_name))
551
+ die("unable to create lazy_name_thread");
552
+
553
+ lazy_update_dir_ref_counts(istate, lazy_entries);
554
+
555
+ if (pthread_join(td_name->pthread, NULL))
556
+ die("unable to join lazy_name_thread");
557
+
558
+ cleanup_dir_mutex();
559
+
560
+ free(td_name);
561
+ free(td_dir);
562
+ free(lazy_entries);
563
+}
564
+
565
+#endif
566
+
567
+static void lazy_init_name_hash(struct index_state *istate)
568
+{
569
if (istate->name_hash_initialized)
570
return;
571
hashmap_init(&istate->name_hash, (hashmap_cmp_fn) cache_entry_cmp,
572
istate->cache_nr);
573
hashmap_init(&istate->dir_hash, (hashmap_cmp_fn) dir_entry_cmp,
574
istate->cache_nr);
125
- for (nr = 0; nr < istate->cache_nr; nr++)
126
- hash_index_entry(istate, istate->cache[nr]);
575
+
576
+ if (lookup_lazy_params(istate)) {
577
+ hashmap_disallow_rehash(&istate->dir_hash, 1);
578
+ threaded_lazy_init_name_hash(istate);
579
+ hashmap_disallow_rehash(&istate->dir_hash, 0);
580
+ } else {
581
+ int nr;
582
+ for (nr = 0; nr < istate->cache_nr; nr++)
583
+ hash_index_entry(istate, istate->cache[nr]);
584
+ }
585
+
586
istate->name_hash_initialized = 1;
587
}
588
589
+/*
590
+ * A test routine for t/helper/ sources.
591
+ *
592
+ * Returns the number of threads used or 0 when
593
+ * the non-threaded code path was used.
594
+ *
595
+ * Requesting threading WILL NOT override guards
596
+ * in lookup_lazy_params().
597
+ */
598
+int test_lazy_init_name_hash(struct index_state *istate, int try_threaded)
599
+{
600
+ lazy_nr_dir_threads = 0;
601
+ lazy_try_threaded = try_threaded;
602
+
603
+ lazy_init_name_hash(istate);
604
+
605
+ return lazy_nr_dir_threads;
606
+}
607
+
608
void add_name_hash(struct index_state *istate, struct cache_entry *ce)
609
{
610
if (istate->name_hash_initialized)