@matheusbsbs / matheus-bagliano / commits / c40641b77b

Fork 0

Optimize symlink/directory detection

This is the base for making symlink detection in the middle fo a pathname saner and (much) more efficient. Under various loads, we want to verify that the full path leading up to a filename is a real directory tree, and that when we successfully do an 'lstat()' on a filename, we don't get a false positive due to a symlink in the middle of the path that git should have seen as a symlink, not as a normal path component. The 'has_symlink_leading_path()' function already did this, and cached a single level of symlink information, but didn't cache the _lack_ of a symlink, so the normal behaviour was actually the wrong way around, and we ended up doing an 'lstat()' on each path component to check that it was a real directory. This caches the last detected full directory and symlink entries, and speeds up especially deep directory structures a lot by avoiding to lstat() all the directories leading up to each entry in the index. [ This can - and should - probably be extended upon so that we eventually never do a bare 'lstat()' on any path entries at *all* when checking the index, but always check the full path carefully. Right now we do not generally check the whole path for all our normal quick index revalidation. We should also make sure that we're careful about all the invalidation, ie when we remove a link and replace it by a directory we should invalidate the symlink cache if it matches (and vice versa for the directory cache). But regardless, the basic function needs to be sane to do that. The old 'has_symlink_leading_path()' was not capable enough - or indeed the code readable enough - to really do that sanely. So I'm pushing this as not just an optimization, but as a base for further work. ] Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Linus Torvalds committed May 9, 2008 at 09:21 UTC c40641b77b0274186fd1b327d5dc3246f814aaaf

5 files changed +61 -47

builtin-apply.c

+1 -1

@@ -2247,7 +2247,7 @@ static int check_to_create_blob(const char *new_name, int ok_if_exists)

2247 * In such a case, path "new_name" does not exist as

2248 * far as git is concerned.

2249 */

2250 - if (has_symlink_leading_path(new_name, NULL))

2250 + if (has_symlink_leading_path(strlen(new_name), new_name))

2251 return 0;

2252

2253 return error("%s: already exists in working directory", new_name);

cache.h

+1 -1

@@ -598,7 +598,7 @@ struct checkout {

598 };

599

600 extern int checkout_entry(struct cache_entry *ce, const struct checkout *state, char *topath);

601 -extern int has_symlink_leading_path(const char *name, char *last_symlink);

601 +extern int has_symlink_leading_path(int len, const char *name);

602

603 extern struct alternate_object_database {

604 struct alternate_object_database *next;

diff-lib.c

+5 -5

@@ -347,14 +347,14 @@ int run_diff_files_cmd(struct rev_info *revs, int argc, const char **argv)

347 * exists for ce that is a submodule -- it is a submodule that is not

348 * checked out). Return negative for an error.

349 */

350 -static int check_removed(const struct cache_entry *ce, struct stat *st, char *symcache)

350 +static int check_removed(const struct cache_entry *ce, struct stat *st)

351 {

352 if (lstat(ce->name, st) < 0) {

353 if (errno != ENOENT && errno != ENOTDIR)

354 return -1;

355 return 1;

356 }

357 - if (has_symlink_leading_path(ce->name, symcache))

357 + if (has_symlink_leading_path(ce_namelen(ce), ce->name))

358 return 1;

359 if (S_ISDIR(st->st_mode)) {

360 unsigned char sub[20];

@@ -421,7 +421,7 @@ int run_diff_files(struct rev_info *revs, unsigned int option)

421 memset(&(dpath->parent[0]), 0,

422 sizeof(struct combine_diff_parent)*5);

423

424 - changed = check_removed(ce, &st, symcache);

424 + changed = check_removed(ce, &st);

425 if (!changed)

426 dpath->mode = ce_mode_from_stat(ce, st.st_mode);

427 else {

@@ -485,7 +485,7 @@ int run_diff_files(struct rev_info *revs, unsigned int option)

485 if (ce_uptodate(ce))

486 continue;

487

488 - changed = check_removed(ce, &st, symcache);

488 + changed = check_removed(ce, &st);

489 if (changed) {

490 if (changed < 0) {

491 perror(ce->name);

@@ -546,7 +546,7 @@ static int get_stat_data(struct cache_entry *ce,

546 if (!cached) {

547 int changed;

548 struct stat st;

549 - changed = check_removed(ce, &st, cbdata->symcache);

549 + changed = check_removed(ce, &st);

550 if (changed < 0)

551 return -1;

552 else if (changed) {

symlinks.c

+49 -33

@@ -1,48 +1,64 @@

1 #include "cache.h"

3 -int has_symlink_leading_path(const char *name, char *last_symlink)

4 -{

3 +struct pathname {

4 + int len;

5 char path[PATH_MAX];

6 - const char *sp, *ep;

7 - char *dp;

8 -

9 - sp = name;

10 - dp = path;

11 -

12 - if (last_symlink && *last_symlink) {

13 - size_t last_len = strlen(last_symlink);

14 - size_t len = strlen(name);

15 - if (last_len < len &&

16 - !strncmp(name, last_symlink, last_len) &&

17 - name[last_len] == '/')

18 - return 1;

19 - *last_symlink = '\0';

6 +};

7 +

8 +/* Return matching pathname prefix length, or zero if not matching */

9 +static inline int match_pathname(int len, const char *name, struct pathname *match)

10 +{

11 + int match_len = match->len;

12 + return (len > match_len &&

13 + name[match_len] == '/' &&

14 + !memcmp(name, match->path, match_len)) ? match_len : 0;

15 +}

16 +

17 +static inline void set_pathname(int len, const char *name, struct pathname *match)

18 +{

19 + if (len < PATH_MAX) {

20 + match->len = len;

21 + memcpy(match->path, name, len);

22 + match->path[len] = 0;

23 }

24 +}

25 +

26 +int has_symlink_leading_path(int len, const char *name)

27 +{

28 + static struct pathname link, nonlink;

29 + char path[PATH_MAX];

30 + struct stat st;

31 + char *sp;

32 + int known_dir;

22 - while (1) {

23 - size_t len;

24 - struct stat st;

34 + /*

35 + * See if the last known symlink cache matches.

36 + */

37 + if (match_pathname(len, name, &link))

38 + return 1;

26 - ep = strchr(sp, '/');

27 - if (!ep)

28 - break;

29 - len = ep - sp;

30 - if (PATH_MAX <= dp + len - path + 2)

31 - return 0; /* new name is longer than that??? */

32 - memcpy(dp, sp, len);

33 - dp[len] = 0;

40 + /*

41 + * Get rid of the last known directory part

42 + */

43 + known_dir = match_pathname(len, name, &nonlink);

44 +

45 + while ((sp = strchr(name + known_dir + 1, '/')) != NULL) {

46 + int thislen = sp - name ;

47 + memcpy(path, name, thislen);

48 + path[thislen] = 0;

50 if (lstat(path, &st))

51 return 0;

52 + if (S_ISDIR(st.st_mode)) {

53 + set_pathname(thislen, path, &nonlink);

54 + known_dir = thislen;

55 + continue;

56 + }

57 if (S_ISLNK(st.st_mode)) {

38 - if (last_symlink)

39 - strcpy(last_symlink, path);

58 + set_pathname(thislen, path, &link);

59 return 1;

60 }

42 -

43 - dp[len++] = '/';

44 - dp = dp + len;

45 - sp = ep + 1;

61 + break;

62 }

63 return 0;

64 }

unpack-trees.c

+5 -7

@@ -26,11 +26,12 @@ static void add_entry(struct unpack_trees_options *o, struct cache_entry *ce,

26 * directories, in case this unlink is the removal of the

27 * last entry in the directory -- empty directories are removed.

28 */

29 -static void unlink_entry(char *name, char *last_symlink)

29 +static void unlink_entry(struct cache_entry *ce)

30 {

31 char *cp, *prev;

32 + char *name = ce->name;

33 - if (has_symlink_leading_path(name, last_symlink))

34 + if (has_symlink_leading_path(ce_namelen(ce), ce->name))

35 return;

36 if (unlink(name))

37 return;

@@ -58,7 +59,6 @@ static int check_updates(struct unpack_trees_options *o)

59 {

60 unsigned cnt = 0, total = 0;

61 struct progress *progress = NULL;

61 - char last_symlink[PATH_MAX];

62 struct index_state *index = &o->result;

63 int i;

64 int errs = 0;

@@ -75,14 +75,13 @@ static int check_updates(struct unpack_trees_options *o)

75 cnt = 0;

76 }

78 - *last_symlink = '\0';

78 for (i = 0; i < index->cache_nr; i++) {

79 struct cache_entry *ce = index->cache[i];

81 if (ce->ce_flags & CE_REMOVE) {

82 display_progress(progress, ++cnt);

83 if (o->update)

85 - unlink_entry(ce->name, last_symlink);

84 + unlink_entry(ce);

85 remove_index_entry_at(&o->result, i);

86 i--;

87 continue;

@@ -97,7 +96,6 @@ static int check_updates(struct unpack_trees_options *o)

96 ce->ce_flags &= ~CE_UPDATE;

97 if (o->update) {

98 errs |= checkout_entry(ce, &state, NULL);

100 - *last_symlink = '\0';

99 }

100 }

101 }

@@ -553,7 +551,7 @@ static int verify_absent(struct cache_entry *ce, const char *action,

551 if (o->index_only || o->reset || !o->update)

552 return 0;

553

556 - if (has_symlink_leading_path(ce->name, NULL))

554 + if (has_symlink_leading_path(ce_namelen(ce), ce->name))

555 return 0;

556

557 if (!lstat(ce->name, &st)) {