sparse-index: convert from full to sparse
If we have a full index, then we can convert it to a sparse index by replacing directories outside of the sparse cone with sparse directory entries. The convert_to_sparse() method does this, when the situation is appropriate. For now, we avoid converting the index to a sparse index if: 1. the index is split. 2. the index is already sparse. 3. sparse-checkout is disabled. 4. sparse-checkout does not use cone mode. Finally, we currently limit the conversion to when the GIT_TEST_SPARSE_INDEX environment variable is enabled. A mode using Git config will be added in a later change. The trickiest thing about this conversion is that we might not be able to mark a directory as a sparse directory just because it is outside the sparse cone. There might be unmerged files within that directory, so we need to look for those. Also, if there is some strange reason why a file is not marked with CE_SKIP_WORKTREE, then we should give up on converting that directory. There is still hope that some of its subdirectories might be able to convert to sparse, so we keep looking deeper. The conversion process is assisted by the cache-tree extension. This is calculated from the full index if it does not already exist. We then abandon the cache-tree as it no longer applies to the newly-sparse index. Thus, this cache-tree will be recalculated in every sparse-full-sparse round-trip until we integrate the cache-tree extension with the sparse index. Some Git commands use the index after writing it. For example, 'git add' will update the index, then write it to disk, then read its entries to report information. To keep the in-memory index in a full state after writing, we re-expand it to a full one after the write. This is wasteful for commands that only write the index and do not read from it again, but that is only the case until we make those commands "sparse aware." We can compare the behavior of the sparse-index in t1092-sparse-checkout-compability.sh by using GIT_TEST_SPARSE_INDEX=1 when operating on the 'sparse-index' repo. We can also compare the two sparse repos directly, such as comparing their indexes (when expanded to full in the case of the 'sparse-index' repo). We also verify that the index is actually populated with sparse directory entries. The 'checkout and reset (mixed)' test is marked for failure when comparing a sparse repo to a full repo, but we can compare the two sparse-checkout cases directly to ensure that we are not changing the behavior when using a sparse index. Signed-off-by: Derrick Stolee <dstolee@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Derrick Stolee committed
Mar 30, 2021 at 13:10 UTC
6e773527b6b03976cefbb0f9571bd40dd5995e6c
6 files changed
+228
-4
cache-tree.c
+3
@@ -6,6 +6,7 @@
6
#include "object-store.h"
7
#include "replace-object.h"
8
#include "promisor-remote.h"
9
+#include "sparse-index.h"
10
11
#ifndef DEBUG_CACHE_TREE
12
#define DEBUG_CACHE_TREE 0
@@ -442,6 +443,8 @@ int cache_tree_update(struct index_state *istate, int flags)
443
if (i)
444
return i;
445
446
+ ensure_full_index(istate);
447
+
448
if (!istate->cache_tree)
449
istate->cache_tree = cache_tree();
450
cache.h
+2
@@ -251,6 +251,8 @@ static inline unsigned int create_ce_mode(unsigned int mode)
251
{
252
if (S_ISLNK(mode))
253
return S_IFLNK;
254
+ if (S_ISSPARSEDIR(mode))
255
+ return S_IFDIR;
256
if (S_ISDIR(mode) || S_ISGITLINK(mode))
257
return S_IFGITLINK;
258
return S_IFREG | ce_permissions(mode);
read-cache.c
+24
-2
@@ -25,6 +25,7 @@
25
#include "fsmonitor.h"
26
#include "thread-utils.h"
27
#include "progress.h"
28
+#include "sparse-index.h"
29
30
/* Mask for the name length in ce_flags in the on-disk index */
31
@@ -1003,8 +1004,14 @@ inside:
1004
1005
c = *path++;
1006
if ((c == '.' && !verify_dotfile(path, mode)) ||
1006
- is_dir_sep(c) || c == '\0')
1007
+ is_dir_sep(c))
1008
return 0;
1009
+ /*
1010
+ * allow terminating directory separators for
1011
+ * sparse directory entries.
1012
+ */
1013
+ if (c == '\0')
1014
+ return S_ISDIR(mode);
1015
} else if (c == '\\' && protect_ntfs) {
1016
if (is_ntfs_dotgit(path))
1017
return 0;
@@ -3088,6 +3095,14 @@ static int do_write_locked_index(struct index_state *istate, struct lock_file *l
3095
unsigned flags)
3096
{
3097
int ret;
3098
+ int was_full = !istate->sparse_index;
3099
+
3100
+ ret = convert_to_sparse(istate);
3101
+
3102
+ if (ret) {
3103
+ warning(_("failed to convert to a sparse-index"));
3104
+ return ret;
3105
+ }
3106
3107
/*
3108
* TODO trace2: replace "the_repository" with the actual repo instance
@@ -3099,6 +3114,9 @@ static int do_write_locked_index(struct index_state *istate, struct lock_file *l
3114
trace2_region_leave_printf("index", "do_write_index", the_repository,
3115
"%s", get_lock_file_path(lock));
3116
3117
+ if (was_full)
3118
+ ensure_full_index(istate);
3119
+
3120
if (ret)
3121
return ret;
3122
if (flags & COMMIT_LOCK)
@@ -3189,9 +3207,10 @@ static int write_shared_index(struct index_state *istate,
3207
struct tempfile **temp)
3208
{
3209
struct split_index *si = istate->split_index;
3192
- int ret;
3210
+ int ret, was_full = !istate->sparse_index;
3211
3212
move_cache_to_base_index(istate);
3213
+ convert_to_sparse(istate);
3214
3215
trace2_region_enter_printf("index", "shared/do_write_index",
3216
the_repository, "%s", get_tempfile_path(*temp));
@@ -3199,6 +3218,9 @@ static int write_shared_index(struct index_state *istate,
3218
trace2_region_leave_printf("index", "shared/do_write_index",
3219
the_repository, "%s", get_tempfile_path(*temp));
3220
3221
+ if (was_full)
3222
+ ensure_full_index(istate);
3223
+
3224
if (ret)
3225
return ret;
3226
ret = adjust_shared_perm(get_tempfile_path(*temp));
sparse-index.c
+139
@@ -4,6 +4,145 @@
4
#include "tree.h"
5
#include "pathspec.h"
6
#include "trace2.h"
7
+#include "cache-tree.h"
8
+#include "config.h"
9
+#include "dir.h"
10
+#include "fsmonitor.h"
11
+
12
+static struct cache_entry *construct_sparse_dir_entry(
13
+ struct index_state *istate,
14
+ const char *sparse_dir,
15
+ struct cache_tree *tree)
16
+{
17
+ struct cache_entry *de;
18
+
19
+ de = make_cache_entry(istate, S_IFDIR, &tree->oid, sparse_dir, 0, 0);
20
+
21
+ de->ce_flags |= CE_SKIP_WORKTREE;
22
+ return de;
23
+}
24
+
25
+/*
26
+ * Returns the number of entries "inserted" into the index.
27
+ */
28
+static int convert_to_sparse_rec(struct index_state *istate,
29
+ int num_converted,
30
+ int start, int end,
31
+ const char *ct_path, size_t ct_pathlen,
32
+ struct cache_tree *ct)
33
+{
34
+ int i, can_convert = 1;
35
+ int start_converted = num_converted;
36
+ enum pattern_match_result match;
37
+ int dtype;
38
+ struct strbuf child_path = STRBUF_INIT;
39
+ struct pattern_list *pl = istate->sparse_checkout_patterns;
40
+
41
+ /*
42
+ * Is the current path outside of the sparse cone?
43
+ * Then check if the region can be replaced by a sparse
44
+ * directory entry (everything is sparse and merged).
45
+ */
46
+ match = path_matches_pattern_list(ct_path, ct_pathlen,
47
+ NULL, &dtype, pl, istate);
48
+ if (match != NOT_MATCHED)
49
+ can_convert = 0;
50
+
51
+ for (i = start; can_convert && i < end; i++) {
52
+ struct cache_entry *ce = istate->cache[i];
53
+
54
+ if (ce_stage(ce) ||
55
+ !(ce->ce_flags & CE_SKIP_WORKTREE))
56
+ can_convert = 0;
57
+ }
58
+
59
+ if (can_convert) {
60
+ struct cache_entry *se;
61
+ se = construct_sparse_dir_entry(istate, ct_path, ct);
62
+
63
+ istate->cache[num_converted++] = se;
64
+ return 1;
65
+ }
66
+
67
+ for (i = start; i < end; ) {
68
+ int count, span, pos = -1;
69
+ const char *base, *slash;
70
+ struct cache_entry *ce = istate->cache[i];
71
+
72
+ /*
73
+ * Detect if this is a normal entry outside of any subtree
74
+ * entry.
75
+ */
76
+ base = ce->name + ct_pathlen;
77
+ slash = strchr(base, '/');
78
+
79
+ if (slash)
80
+ pos = cache_tree_subtree_pos(ct, base, slash - base);
81
+
82
+ if (pos < 0) {
83
+ istate->cache[num_converted++] = ce;
84
+ i++;
85
+ continue;
86
+ }
87
+
88
+ strbuf_setlen(&child_path, 0);
89
+ strbuf_add(&child_path, ce->name, slash - ce->name + 1);
90
+
91
+ span = ct->down[pos]->cache_tree->entry_count;
92
+ count = convert_to_sparse_rec(istate,
93
+ num_converted, i, i + span,
94
+ child_path.buf, child_path.len,
95
+ ct->down[pos]->cache_tree);
96
+ num_converted += count;
97
+ i += span;
98
+ }
99
+
100
+ strbuf_release(&child_path);
101
+ return num_converted - start_converted;
102
+}
103
+
104
+int convert_to_sparse(struct index_state *istate)
105
+{
106
+ if (istate->split_index || istate->sparse_index ||
107
+ !core_apply_sparse_checkout || !core_sparse_checkout_cone)
108
+ return 0;
109
+
110
+ /*
111
+ * For now, only create a sparse index with the
112
+ * GIT_TEST_SPARSE_INDEX environment variable. We will relax
113
+ * this once we have a proper way to opt-in (and later still,
114
+ * opt-out).
115
+ */
116
+ if (!git_env_bool("GIT_TEST_SPARSE_INDEX", 0))
117
+ return 0;
118
+
119
+ if (!istate->sparse_checkout_patterns) {
120
+ istate->sparse_checkout_patterns = xcalloc(1, sizeof(struct pattern_list));
121
+ if (get_sparse_checkout_patterns(istate->sparse_checkout_patterns) < 0)
122
+ return 0;
123
+ }
124
+
125
+ if (!istate->sparse_checkout_patterns->use_cone_patterns) {
126
+ warning(_("attempting to use sparse-index without cone mode"));
127
+ return -1;
128
+ }
129
+
130
+ if (cache_tree_update(istate, 0)) {
131
+ warning(_("unable to update cache-tree, staying full"));
132
+ return -1;
133
+ }
134
+
135
+ remove_fsmonitor(istate);
136
+
137
+ trace2_region_enter("index", "convert_to_sparse", istate->repo);
138
+ istate->cache_nr = convert_to_sparse_rec(istate,
139
+ 0, 0, istate->cache_nr,
140
+ "", 0, istate->cache_tree);
141
+ istate->drop_cache_tree = 1;
142
+ istate->sparse_index = 1;
143
+ trace2_region_leave("index", "convert_to_sparse", istate->repo);
144
+ return 0;
145
+}
146
147
static void set_index_entry(struct index_state *istate, int nr, struct cache_entry *ce)
148
{
sparse-index.h
+1
@@ -3,5 +3,6 @@
3
4
struct index_state;
5
void ensure_full_index(struct index_state *istate);
6
+int convert_to_sparse(struct index_state *istate);
7
8
#endif
t/t1092-sparse-checkout-compatibility.sh
+59
-2
@@ -2,6 +2,11 @@
2
3
test_description='compare full workdir to sparse workdir'
4
5
+# The verify_cache_tree() check is not sparse-aware (yet).
6
+# So, disable the check until that integration is complete.
7
+GIT_TEST_CHECK_CACHE_TREE=0
8
+GIT_TEST_SPLIT_INDEX=0
9
+
10
. ./test-lib.sh
11
12
test_expect_success 'setup' '
@@ -121,7 +126,9 @@ run_on_all () {
126
test_all_match () {
127
run_on_all "$@" &&
128
test_cmp full-checkout-out sparse-checkout-out &&
124
- test_cmp full-checkout-err sparse-checkout-err
129
+ test_cmp full-checkout-out sparse-index-out &&
130
+ test_cmp full-checkout-err sparse-checkout-err &&
131
+ test_cmp full-checkout-err sparse-index-err
132
}
133
134
test_sparse_match () {
@@ -130,6 +137,38 @@ test_sparse_match () {
137
test_cmp sparse-checkout-err sparse-index-err
138
}
139
140
+test_expect_success 'sparse-index contents' '
141
+ init_repos &&
142
+
143
+ test-tool -C sparse-index read-cache --table >cache &&
144
+ for dir in folder1 folder2 x
145
+ do
146
+ TREE=$(git -C sparse-index rev-parse HEAD:$dir) &&
147
+ grep "040000 tree $TREE $dir/" cache \
148
+ || return 1
149
+ done &&
150
+
151
+ GIT_TEST_SPARSE_INDEX=1 git -C sparse-index sparse-checkout set folder1 &&
152
+
153
+ test-tool -C sparse-index read-cache --table >cache &&
154
+ for dir in deep folder2 x
155
+ do
156
+ TREE=$(git -C sparse-index rev-parse HEAD:$dir) &&
157
+ grep "040000 tree $TREE $dir/" cache \
158
+ || return 1
159
+ done &&
160
+
161
+ GIT_TEST_SPARSE_INDEX=1 git -C sparse-index sparse-checkout set deep/deeper1 &&
162
+
163
+ test-tool -C sparse-index read-cache --table >cache &&
164
+ for dir in deep/deeper2 folder1 folder2 x
165
+ do
166
+ TREE=$(git -C sparse-index rev-parse HEAD:$dir) &&
167
+ grep "040000 tree $TREE $dir/" cache \
168
+ || return 1
169
+ done
170
+'
171
+
172
test_expect_success 'expanded in-memory index matches full index' '
173
init_repos &&
174
test_sparse_match test-tool read-cache --expand --table
@@ -137,6 +176,7 @@ test_expect_success 'expanded in-memory index matches full index' '
176
177
test_expect_success 'status with options' '
178
init_repos &&
179
+ test_sparse_match ls &&
180
test_all_match git status --porcelain=v2 &&
181
test_all_match git status --porcelain=v2 -z -u &&
182
test_all_match git status --porcelain=v2 -uno &&
@@ -273,6 +313,17 @@ test_expect_failure 'checkout and reset (mixed)' '
313
test_all_match git reset update-folder2
314
'
315
316
+# Ensure that sparse-index behaves identically to
317
+# sparse-checkout with a full index.
318
+test_expect_success 'checkout and reset (mixed) [sparse]' '
319
+ init_repos &&
320
+
321
+ test_sparse_match git checkout -b reset-test update-deep &&
322
+ test_sparse_match git reset deepest &&
323
+ test_sparse_match git reset update-folder1 &&
324
+ test_sparse_match git reset update-folder2
325
+'
326
+
327
test_expect_success 'merge' '
328
init_repos &&
329
@@ -309,14 +360,20 @@ test_expect_success 'clean' '
360
test_all_match git status --porcelain=v2 &&
361
test_all_match git clean -f &&
362
test_all_match git status --porcelain=v2 &&
363
+ test_sparse_match ls &&
364
+ test_sparse_match ls folder1 &&
365
366
test_all_match git clean -xf &&
367
test_all_match git status --porcelain=v2 &&
368
+ test_sparse_match ls &&
369
+ test_sparse_match ls folder1 &&
370
371
test_all_match git clean -xdf &&
372
test_all_match git status --porcelain=v2 &&
373
+ test_sparse_match ls &&
374
+ test_sparse_match ls folder1 &&
375
319
- test_path_is_dir sparse-checkout/folder1
376
+ test_sparse_match test_path_is_dir folder1
377
'
378
379
test_done