sparse-index: implement ensure_full_index()
We will mark an in-memory index_state as having sparse directory entries with the sparse_index bit. These currently cannot exist, but we will add a mechanism for collapsing a full index to a sparse one in a later change. That will happen at write time, so we must first allow parsing the format before writing it. Commands or methods that require a full index in order to operate can call ensure_full_index() to expand that index in-memory. This requires parsing trees using that index's repository. Sparse directory entries have a specific 'ce_mode' value. The macro S_ISSPARSEDIR(ce->ce_mode) can check if a cache_entry 'ce' has this type. This ce_mode is not possible with the existing index formats, so we don't also verify all properties of a sparse-directory entry, which are: 1. ce->ce_mode == 0040000 2. ce->flags & CE_SKIP_WORKTREE is true 3. ce->name[ce->namelen - 1] == '/' (ends in dir separator) 4. ce->oid references a tree object. These are all semi-enforced in ensure_full_index() to some extent. Any deviation will cause a warning at minimum or a failure in the worst case. Signed-off-by: Derrick Stolee <dstolee@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Derrick Stolee committed
Mar 30, 2021 at 13:10 UTC
4300f8442a276aa48d327b8371af9c2917bd3d5a
3 files changed
+118
-2
cache.h
+12
-1
@@ -204,6 +204,8 @@ struct cache_entry {
204
#error "CE_EXTENDED_FLAGS out of range"
205
#endif
206
207
+#define S_ISSPARSEDIR(m) ((m) == S_IFDIR)
208
+
209
/* Forward structure decls */
210
struct pathspec;
211
struct child_process;
@@ -319,7 +321,14 @@ struct index_state {
321
drop_cache_tree : 1,
322
updated_workdir : 1,
323
updated_skipworktree : 1,
322
- fsmonitor_has_run_once : 1;
324
+ fsmonitor_has_run_once : 1,
325
+
326
+ /*
327
+ * sparse_index == 1 when sparse-directory
328
+ * entries exist. Requires sparse-checkout
329
+ * in cone mode.
330
+ */
331
+ sparse_index : 1;
332
struct hashmap name_hash;
333
struct hashmap dir_hash;
334
struct object_id oid;
@@ -722,6 +731,8 @@ int read_index_from(struct index_state *, const char *path,
731
const char *gitdir);
732
int is_index_unborn(struct index_state *);
733
734
+void ensure_full_index(struct index_state *istate);
735
+
736
/* For use with `write_locked_index()`. */
737
#define COMMIT_LOCK (1 << 0)
738
#define SKIP_IF_UNCHANGED (1 << 1)
read-cache.c
+9
@@ -101,6 +101,9 @@ static const char *alternate_index_output;
101
102
static void set_index_entry(struct index_state *istate, int nr, struct cache_entry *ce)
103
{
104
+ if (S_ISSPARSEDIR(ce->ce_mode))
105
+ istate->sparse_index = 1;
106
+
107
istate->cache[nr] = ce;
108
add_name_hash(istate, ce);
109
}
@@ -2273,6 +2276,12 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)
2276
trace2_data_intmax("index", the_repository, "read/cache_nr",
2277
istate->cache_nr);
2278
2279
+ if (!istate->repo)
2280
+ istate->repo = the_repository;
2281
+ prepare_repo_settings(istate->repo);
2282
+ if (istate->repo->settings.command_requires_full_index)
2283
+ ensure_full_index(istate);
2284
+
2285
return istate->cache_nr;
2286
2287
unmap:
sparse-index.c
+97
-1
@@ -1,8 +1,104 @@
1
#include "cache.h"
2
#include "repository.h"
3
#include "sparse-index.h"
4
+#include "tree.h"
5
+#include "pathspec.h"
6
+#include "trace2.h"
7
+
8
+static void set_index_entry(struct index_state *istate, int nr, struct cache_entry *ce)
9
+{
10
+ ALLOC_GROW(istate->cache, nr + 1, istate->cache_alloc);
11
+
12
+ istate->cache[nr] = ce;
13
+ add_name_hash(istate, ce);
14
+}
15
+
16
+static int add_path_to_index(const struct object_id *oid,
17
+ struct strbuf *base, const char *path,
18
+ unsigned int mode, void *context)
19
+{
20
+ struct index_state *istate = (struct index_state *)context;
21
+ struct cache_entry *ce;
22
+ size_t len = base->len;
23
+
24
+ if (S_ISDIR(mode))
25
+ return READ_TREE_RECURSIVE;
26
+
27
+ strbuf_addstr(base, path);
28
+
29
+ ce = make_cache_entry(istate, mode, oid, base->buf, 0, 0);
30
+ ce->ce_flags |= CE_SKIP_WORKTREE;
31
+ set_index_entry(istate, istate->cache_nr++, ce);
32
+
33
+ strbuf_setlen(base, len);
34
+ return 0;
35
+}
36
37
void ensure_full_index(struct index_state *istate)
38
{
7
- /* intentionally left blank */
39
+ int i;
40
+ struct index_state *full;
41
+ struct strbuf base = STRBUF_INIT;
42
+
43
+ if (!istate || !istate->sparse_index)
44
+ return;
45
+
46
+ if (!istate->repo)
47
+ istate->repo = the_repository;
48
+
49
+ trace2_region_enter("index", "ensure_full_index", istate->repo);
50
+
51
+ /* initialize basics of new index */
52
+ full = xcalloc(1, sizeof(struct index_state));
53
+ memcpy(full, istate, sizeof(struct index_state));
54
+
55
+ /* then change the necessary things */
56
+ full->sparse_index = 0;
57
+ full->cache_alloc = (3 * istate->cache_alloc) / 2;
58
+ full->cache_nr = 0;
59
+ ALLOC_ARRAY(full->cache, full->cache_alloc);
60
+
61
+ for (i = 0; i < istate->cache_nr; i++) {
62
+ struct cache_entry *ce = istate->cache[i];
63
+ struct tree *tree;
64
+ struct pathspec ps;
65
+
66
+ if (!S_ISSPARSEDIR(ce->ce_mode)) {
67
+ set_index_entry(full, full->cache_nr++, ce);
68
+ continue;
69
+ }
70
+ if (!(ce->ce_flags & CE_SKIP_WORKTREE))
71
+ warning(_("index entry is a directory, but not sparse (%08x)"),
72
+ ce->ce_flags);
73
+
74
+ /* recursively walk into cd->name */
75
+ tree = lookup_tree(istate->repo, &ce->oid);
76
+
77
+ memset(&ps, 0, sizeof(ps));
78
+ ps.recursive = 1;
79
+ ps.has_wildcard = 1;
80
+ ps.max_depth = -1;
81
+
82
+ strbuf_setlen(&base, 0);
83
+ strbuf_add(&base, ce->name, strlen(ce->name));
84
+
85
+ read_tree_at(istate->repo, tree, &base, &ps,
86
+ add_path_to_index, full);
87
+
88
+ /* free directory entries. full entries are re-used */
89
+ discard_cache_entry(ce);
90
+ }
91
+
92
+ /* Copy back into original index. */
93
+ memcpy(&istate->name_hash, &full->name_hash, sizeof(full->name_hash));
94
+ istate->sparse_index = 0;
95
+ free(istate->cache);
96
+ istate->cache = full->cache;
97
+ istate->cache_nr = full->cache_nr;
98
+ istate->cache_alloc = full->cache_alloc;
99
+
100
+ strbuf_release(&base);
101
+ free(full);
102
+
103
+ trace2_region_leave("index", "ensure_full_index", istate->repo);
104
}