config: create core.multiPackIndex setting
The core.multiPackIndex config setting controls the multi-pack- index (MIDX) feature. If false, the setting will disable all reads from the multi-pack-index file. Read this config setting in the new prepare_multi_pack_index_one() which is called during prepare_packed_git(). This check is run once per repository. Add comparison commands in t5319-multi-pack-index.sh to check typical Git behavior remains the same as the config setting is turned on and off. This currently includes 'git rev-list' and 'git log' commands to trigger several object database reads. Currently, these would only catch an error in the prepare_multi_pack_index_one(), but with later commits will catch errors in object lookups, abbreviations, and approximate object counts. Signed-off-by: Derrick Stolee <dstolee@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Derrick Stolee committed
Jul 12, 2018 at 15:39 UTC
c4d25228ebb22a60f1fcb267e19c503bab708cdc
6 files changed
+85
-14
Documentation/config.txt
+5
@@ -908,6 +908,11 @@ core.commitGraph::
908
Enable git commit graph feature. Allows reading from the
909
commit-graph file.
910
911
+core.multiPackIndex::
912
+ Use the multi-pack-index file to track multiple packfiles using a
913
+ single index. See link:technical/multi-pack-index.html[the
914
+ multi-pack-index design document].
915
+
916
core.sparseCheckout::
917
Enable "sparse checkout" feature. See section "Sparse checkout" in
918
linkgit:git-read-tree[1] for more information.
midx.c
+25
@@ -1,4 +1,5 @@
1
#include "cache.h"
2
+#include "config.h"
3
#include "csum-file.h"
4
#include "dir.h"
5
#include "lockfile.h"
@@ -177,6 +178,30 @@ cleanup_fail:
178
return NULL;
179
}
180
181
+int prepare_multi_pack_index_one(struct repository *r, const char *object_dir)
182
+{
183
+ struct multi_pack_index *m = r->objects->multi_pack_index;
184
+ struct multi_pack_index *m_search;
185
+ int config_value;
186
+
187
+ if (repo_config_get_bool(r, "core.multipackindex", &config_value) ||
188
+ !config_value)
189
+ return 0;
190
+
191
+ for (m_search = m; m_search; m_search = m_search->next)
192
+ if (!strcmp(object_dir, m_search->object_dir))
193
+ return 1;
194
+
195
+ r->objects->multi_pack_index = load_multi_pack_index(object_dir);
196
+
197
+ if (r->objects->multi_pack_index) {
198
+ r->objects->multi_pack_index->next = m;
199
+ return 1;
200
+ }
201
+
202
+ return 0;
203
+}
204
+
205
static size_t write_midx_header(struct hashfile *f,
206
unsigned char num_chunks,
207
uint32_t num_packs)
midx.h
+5
@@ -1,7 +1,11 @@
1
#ifndef __MIDX_H__
2
#define __MIDX_H__
3
4
+#include "repository.h"
5
+
6
struct multi_pack_index {
7
+ struct multi_pack_index *next;
8
+
9
int fd;
10
11
const unsigned char *data;
@@ -25,6 +29,7 @@ struct multi_pack_index {
29
};
30
31
struct multi_pack_index *load_multi_pack_index(const char *object_dir);
32
+int prepare_multi_pack_index_one(struct repository *r, const char *object_dir);
33
34
int write_midx_file(const char *object_dir);
35
object-store.h
+7
@@ -105,6 +105,13 @@ struct raw_object_store {
105
*/
106
struct oidmap *replace_map;
107
108
+ /*
109
+ * private data
110
+ *
111
+ * should only be accessed directly by packfile.c and midx.c
112
+ */
113
+ struct multi_pack_index *multi_pack_index;
114
+
115
/*
116
* private data
117
*
packfile.c
+5
-1
@@ -15,6 +15,7 @@
15
#include "tree-walk.h"
16
#include "tree.h"
17
#include "object-store.h"
18
+#include "midx.h"
19
20
char *odb_pack_name(struct strbuf *buf,
21
const unsigned char *sha1,
@@ -935,10 +936,13 @@ static void prepare_packed_git(struct repository *r)
936
937
if (r->objects->packed_git_initialized)
938
return;
939
+ prepare_multi_pack_index_one(r, r->objects->objectdir);
940
prepare_packed_git_one(r, r->objects->objectdir, 1);
941
prepare_alt_odb(r);
940
- for (alt = r->objects->alt_odb_list; alt; alt = alt->next)
942
+ for (alt = r->objects->alt_odb_list; alt; alt = alt->next) {
943
+ prepare_multi_pack_index_one(r, alt->path);
944
prepare_packed_git_one(r, alt->path, 0);
945
+ }
946
rearrange_packed_git(r);
947
prepare_packed_git_mru(r);
948
r->objects->packed_git_initialized = 1;
t/t5319-multi-pack-index.sh
+38
-13
@@ -3,6 +3,8 @@
3
test_description='multi-pack-indexes'
4
. ./test-lib.sh
5
6
+objdir=.git/objects
7
+
8
midx_read_expect () {
9
NUM_PACKS=$1
10
NUM_OBJECTS=$2
@@ -76,18 +78,35 @@ test_expect_success 'create objects' '
78
'
79
80
test_expect_success 'write midx with one v1 pack' '
79
- pack=$(git pack-objects --index-version=1 pack/test <obj-list) &&
80
- test_when_finished rm pack/test-$pack.pack pack/test-$pack.idx pack/multi-pack-index &&
81
- git multi-pack-index --object-dir=. write &&
82
- midx_read_expect 1 18 4 .
81
+ pack=$(git pack-objects --index-version=1 $objdir/pack/test <obj-list) &&
82
+ test_when_finished rm $objdir/pack/test-$pack.pack \
83
+ $objdir/pack/test-$pack.idx $objdir/pack/multi-pack-index &&
84
+ git multi-pack-index --object-dir=$objdir write &&
85
+ midx_read_expect 1 18 4 $objdir
86
'
87
88
+midx_git_two_modes () {
89
+ git -c core.multiPackIndex=false $1 >expect &&
90
+ git -c core.multiPackIndex=true $1 >actual &&
91
+ test_cmp expect actual
92
+}
93
+
94
+compare_results_with_midx () {
95
+ MSG=$1
96
+ test_expect_success "check normal git operations: $MSG" '
97
+ midx_git_two_modes "rev-list --objects --all" &&
98
+ midx_git_two_modes "log --raw"
99
+ '
100
+}
101
+
102
test_expect_success 'write midx with one v2 pack' '
86
- git pack-objects --index-version=2,0x40 pack/test <obj-list &&
87
- git multi-pack-index --object-dir=. write &&
88
- midx_read_expect 1 18 4 .
103
+ git pack-objects --index-version=2,0x40 $objdir/pack/test <obj-list &&
104
+ git multi-pack-index --object-dir=$objdir write &&
105
+ midx_read_expect 1 18 4 $objdir
106
'
107
108
+compare_results_with_midx "one v2 pack"
109
+
110
test_expect_success 'add more objects' '
111
for i in $(test_seq 6 10)
112
do
@@ -97,25 +116,31 @@ test_expect_success 'add more objects' '
116
'
117
118
test_expect_success 'write midx with two packs' '
100
- git pack-objects --index-version=1 pack/test-2 <obj-list &&
101
- git multi-pack-index --object-dir=. write &&
102
- midx_read_expect 2 34 4 .
119
+ git pack-objects --index-version=1 $objdir/pack/test-2 <obj-list &&
120
+ git multi-pack-index --object-dir=$objdir write &&
121
+ midx_read_expect 2 34 4 $objdir
122
'
123
124
+compare_results_with_midx "two packs"
125
+
126
test_expect_success 'add more packs' '
127
for j in $(test_seq 11 20)
128
do
129
generate_objects $j &&
130
commit_and_list_objects &&
110
- git pack-objects --index-version=2 pack/test-pack <obj-list
131
+ git pack-objects --index-version=2 $objdir/pack/test-pack <obj-list
132
done
133
'
134
135
+compare_results_with_midx "mixed mode (two packs + extra)"
136
+
137
test_expect_success 'write midx with twelve packs' '
115
- git multi-pack-index --object-dir=. write &&
116
- midx_read_expect 12 74 4 .
138
+ git multi-pack-index --object-dir=$objdir write &&
139
+ midx_read_expect 12 74 4 $objdir
140
'
141
142
+compare_results_with_midx "twelve packs"
143
+
144
# usage: corrupt_data <file> <pos> [<data>]
145
corrupt_data () {
146
file=$1