@matheusbsbs / matheus-bagliano / commits / 9fc42d6091
Fork 0

Optimize directory listing with pathspec limiter.

The way things are set up, you can now pass a "pathspec" to the "read_directory()" function. If you pass NULL, it acts exactly like it used to do (read everything). If you pass a non-NULL pointer, it will simplify it into a "these are the prefixes without any special characters", and stop any readdir() early if the path in question doesn't match any of the prefixes. NOTE! This does *not* obviate the need for the caller to do the *exact* pathspec match later. It's a first-level filter on "read_directory()", but it does not do the full pathspec thing. Maybe it should. But in the meantime, builtin-add.c really does need to do first read_directory(dir, .., pathspec); if (pathspec) prune_directory(dir, pathspec, baselen); ie the "prune_directory()" part will do the *exact* pathspec pruning, while the "read_directory()" will use the pathspec just to do some quick high-level pruning of the directories it will recurse into. Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Linus Torvalds committed Mar 30, 2007 at 20:39 UTC 9fc42d609197ef331f1c5b2ce5ef6fc9f2ee634f
5 files changed +95 -9
builtin-add.c
+1 -1
@@ -87,7 +87,7 @@ static void fill_directory(struct dir_struct *dir, const char **pathspec)
87 }
88
89 /* Read the directory and prune it */
90 - read_directory(dir, path, base, baselen);
90 + read_directory(dir, path, base, baselen, pathspec);
91 if (pathspec)
92 prune_directory(dir, pathspec, baselen);
93 }
builtin-ls-files.c
+1 -1
@@ -216,7 +216,7 @@ static void show_files(struct dir_struct *dir, const char *prefix)
216
217 if (baselen)
218 path = base = prefix;
219 - read_directory(dir, path, base, baselen);
219 + read_directory(dir, path, base, baselen, pathspec);
220 if (show_others)
221 show_other_files(dir);
222 if (show_killed)
dir.c
+91 -5
@@ -8,6 +8,11 @@
8 #include "cache.h"
9 #include "dir.h"
10
11 +struct path_simplify {
12 + int len;
13 + const char *path;
14 +};
15 +
16 int common_prefix(const char **pathspec)
17 {
18 const char *path, *slash, *next;
@@ -292,6 +297,31 @@ static int dir_exists(const char *dirname, int len)
297 return !strncmp(active_cache[pos]->name, dirname, len);
298 }
299
300 +/*
301 + * This is an inexact early pruning of any recursive directory
302 + * reading - if the path cannot possibly be in the pathspec,
303 + * return true, and we'll skip it early.
304 + */
305 +static int simplify_away(const char *path, int pathlen, const struct path_simplify *simplify)
306 +{
307 + if (simplify) {
308 + for (;;) {
309 + const char *match = simplify->path;
310 + int len = simplify->len;
311 +
312 + if (!match)
313 + break;
314 + if (len > pathlen)
315 + len = pathlen;
316 + if (!memcmp(path, match, len))
317 + return 0;
318 + simplify++;
319 + }
320 + return 1;
321 + }
322 + return 0;
323 +}
324 +
325 /*
326 * Read a directory tree. We currently ignore anything but
327 * directories, regular files and symlinks. That's because git
@@ -301,7 +331,7 @@ static int dir_exists(const char *dirname, int len)
331 * Also, we ignore the name ".git" (even if it is not a directory).
332 * That likely will not change.
333 */
304 -static int read_directory_recursive(struct dir_struct *dir, const char *path, const char *base, int baselen, int check_only)
334 +static int read_directory_recursive(struct dir_struct *dir, const char *path, const char *base, int baselen, int check_only, const struct path_simplify *simplify)
335 {
336 DIR *fdir = opendir(path);
337 int contents = 0;
@@ -324,6 +354,8 @@ static int read_directory_recursive(struct dir_struct *dir, const char *path, co
354 continue;
355 len = strlen(de->d_name);
356 memcpy(fullname + baselen, de->d_name, len+1);
357 + if (simplify_away(fullname, baselen + len, simplify))
358 + continue;
359 if (excluded(dir, fullname) != dir->show_ignored) {
360 if (!dir->show_ignored || DTYPE(de) != DT_DIR) {
361 continue;
@@ -350,13 +382,13 @@ static int read_directory_recursive(struct dir_struct *dir, const char *path, co
382 if (dir->hide_empty_directories &&
383 !read_directory_recursive(dir,
384 fullname, fullname,
353 - baselen + len, 1))
385 + baselen + len, 1, simplify))
386 continue;
387 break;
388 }
389
390 contents += read_directory_recursive(dir,
359 - fullname, fullname, baselen + len, 0);
391 + fullname, fullname, baselen + len, 0, simplify);
392 continue;
393 case DT_REG:
394 case DT_LNK:
@@ -386,8 +418,61 @@ static int cmp_name(const void *p1, const void *p2)
418 e2->name, e2->len);
419 }
420
389 -int read_directory(struct dir_struct *dir, const char *path, const char *base, int baselen)
421 +/*
422 + * Return the length of the "simple" part of a path match limiter.
423 + */
424 +static int simple_length(const char *match)
425 {
426 + const char special[256] = {
427 + [0] = 1, ['?'] = 1,
428 + ['\\'] = 1, ['*'] = 1,
429 + ['['] = 1
430 + };
431 + int len = -1;
432 +
433 + for (;;) {
434 + unsigned char c = *match++;
435 + len++;
436 + if (special[c])
437 + return len;
438 + }
439 +}
440 +
441 +static struct path_simplify *create_simplify(const char **pathspec)
442 +{
443 + int nr, alloc = 0;
444 + struct path_simplify *simplify = NULL;
445 +
446 + if (!pathspec)
447 + return NULL;
448 +
449 + for (nr = 0 ; ; nr++) {
450 + const char *match;
451 + if (nr >= alloc) {
452 + alloc = alloc_nr(alloc);
453 + simplify = xrealloc(simplify, alloc * sizeof(*simplify));
454 + }
455 + match = *pathspec++;
456 + if (!match)
457 + break;
458 + simplify[nr].path = match;
459 + simplify[nr].len = simple_length(match);
460 + }
461 + simplify[nr].path = NULL;
462 + simplify[nr].len = 0;
463 + return simplify;
464 +}
465 +
466 +static void free_simplify(struct path_simplify *simplify)
467 +{
468 + if (simplify)
469 + free(simplify);
470 +}
471 +
472 +int read_directory(struct dir_struct *dir, const char *path, const char *base, int baselen, const char **pathspec)
473 +{
474 + struct path_simplify *simplify = create_simplify(pathspec);
475 +
476 /*
477 * Make sure to do the per-directory exclude for all the
478 * directories leading up to our base.
@@ -414,7 +499,8 @@ int read_directory(struct dir_struct *dir, const char *path, const char *base, i
499 }
500 }
501
417 - read_directory_recursive(dir, path, base, baselen, 0);
502 + read_directory_recursive(dir, path, base, baselen, 0, simplify);
503 + free_simplify(simplify);
504 qsort(dir->entries, dir->nr, sizeof(struct dir_entry *), cmp_name);
505 return dir->nr;
506 }
dir.h
+1 -1
@@ -48,7 +48,7 @@ extern int common_prefix(const char **pathspec);
48 #define MATCHED_EXACTLY 3
49 extern int match_pathspec(const char **pathspec, const char *name, int namelen, int prefix, char *seen);
50
51 -extern int read_directory(struct dir_struct *, const char *path, const char *base, int baselen);
51 +extern int read_directory(struct dir_struct *, const char *path, const char *base, int baselen, const char **pathspec);
52 extern int push_exclude_per_directory(struct dir_struct *, const char *, int);
53 extern void pop_exclude_per_directory(struct dir_struct *, int);
54
wt-status.c
+1 -1
@@ -260,7 +260,7 @@ static void wt_status_print_untracked(struct wt_status *s)
260 if (file_exists(x))
261 add_excludes_from_file(&dir, x);
262
263 - read_directory(&dir, ".", "", 0);
263 + read_directory(&dir, ".", "", 0, NULL);
264 for(i = 0; i < dir.nr; i++) {
265 /* check for matching entry, which is unmerged; lifted from
266 * builtin-ls-files:show_other_files */