@matheusbsbs / matheus-bagliano / commits / 9535ce7337

pack-objects: add list-objects filtering

Teach pack-objects to use the filtering provided by the traverse_commit_list_filtered() interface to omit unwanted objects from the resulting packfile. Filtering requires the use of the "--stdout" option. Add t5317 test. In the future, we will introduce a "partial clone" mechanism wherein an object in a repo, obtained from a remote, may reference a missing object that can be dynamically fetched from that remote once needed. This "partial clone" mechanism will have a way, sometimes slow, of determining if a missing link is one of the links expected to be produced by this mechanism. This patch introduces handling of missing objects to help debugging and development of the "partial clone" mechanism, and once the mechanism is implemented, for a power user to perform operations that are missing-object aware without incurring the cost of checking if a missing link is expected. Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com> Reviewed-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Jeff Hostetler committed Nov 21, 2017 at 20:58 UTC 9535ce7337b9add29fab3cb0021ccaf0da8c7138
3 files changed +456 -2
Documentation/git-pack-objects.txt
+18 -1
@@ -12,7 +12,8 @@ SYNOPSIS
12 'git pack-objects' [-q | --progress | --all-progress] [--all-progress-implied]
13 [--no-reuse-delta] [--delta-base-offset] [--non-empty]
14 [--local] [--incremental] [--window=<n>] [--depth=<n>]
15 - [--revs [--unpacked | --all]] [--stdout | base-name]
15 + [--revs [--unpacked | --all]]
16 + [--stdout [--filter=<filter-spec>] | base-name]
17 [--shallow] [--keep-true-parents] < object-list
18
19
@@ -236,6 +237,22 @@ So does `git bundle` (see linkgit:git-bundle[1]) when it creates a bundle.
237 With this option, parents that are hidden by grafts are packed
238 nevertheless.
239
240 +--filter=<filter-spec>::
241 + Requires `--stdout`. Omits certain objects (usually blobs) from
242 + the resulting packfile. See linkgit:git-rev-list[1] for valid
243 + `<filter-spec>` forms.
244 +
245 +--missing=<missing-action>::
246 + A debug option to help with future "partial clone" development.
247 + This option specifies how missing objects are handled.
248 ++
249 +The form '--missing=error' requests that pack-objects stop with an error if
250 +a missing object is encountered. This is the default action.
251 ++
252 +The form '--missing=allow-any' will allow object traversal to continue
253 +if a missing object is encountered. Missing objects will silently be
254 +omitted from the results.
255 +
256 SEE ALSO
257 --------
258 linkgit:git-rev-list[1]
builtin/pack-objects.c
+63 -1
@@ -15,6 +15,8 @@
15 #include "diff.h"
16 #include "revision.h"
17 #include "list-objects.h"
18 +#include "list-objects-filter.h"
19 +#include "list-objects-filter-options.h"
20 #include "pack-objects.h"
21 #include "progress.h"
22 #include "refs.h"
@@ -79,6 +81,15 @@ static unsigned long cache_max_small_delta_size = 1000;
81
82 static unsigned long window_memory_limit = 0;
83
84 +static struct list_objects_filter_options filter_options;
85 +
86 +enum missing_action {
87 + MA_ERROR = 0, /* fail if any missing objects are encountered */
88 + MA_ALLOW_ANY, /* silently allow ALL missing objects */
89 +};
90 +static enum missing_action arg_missing_action;
91 +static show_object_fn fn_show_object;
92 +
93 /*
94 * stats
95 */
@@ -2552,6 +2563,42 @@ static void show_object(struct object *obj, const char *name, void *data)
2563 obj->flags |= OBJECT_ADDED;
2564 }
2565
2566 +static void show_object__ma_allow_any(struct object *obj, const char *name, void *data)
2567 +{
2568 + assert(arg_missing_action == MA_ALLOW_ANY);
2569 +
2570 + /*
2571 + * Quietly ignore ALL missing objects. This avoids problems with
2572 + * staging them now and getting an odd error later.
2573 + */
2574 + if (!has_object_file(&obj->oid))
2575 + return;
2576 +
2577 + show_object(obj, name, data);
2578 +}
2579 +
2580 +static int option_parse_missing_action(const struct option *opt,
2581 + const char *arg, int unset)
2582 +{
2583 + assert(arg);
2584 + assert(!unset);
2585 +
2586 + if (!strcmp(arg, "error")) {
2587 + arg_missing_action = MA_ERROR;
2588 + fn_show_object = show_object;
2589 + return 0;
2590 + }
2591 +
2592 + if (!strcmp(arg, "allow-any")) {
2593 + arg_missing_action = MA_ALLOW_ANY;
2594 + fn_show_object = show_object__ma_allow_any;
2595 + return 0;
2596 + }
2597 +
2598 + die(_("invalid value for --missing"));
2599 + return 0;
2600 +}
2601 +
2602 static void show_edge(struct commit *commit)
2603 {
2604 add_preferred_base(commit->object.oid.hash);
@@ -2816,7 +2863,12 @@ static void get_object_list(int ac, const char **av)
2863 if (prepare_revision_walk(&revs))
2864 die("revision walk setup failed");
2865 mark_edges_uninteresting(&revs, show_edge);
2819 - traverse_commit_list(&revs, show_commit, show_object, NULL);
2866 +
2867 + if (!fn_show_object)
2868 + fn_show_object = show_object;
2869 + traverse_commit_list_filtered(&filter_options, &revs,
2870 + show_commit, fn_show_object, NULL,
2871 + NULL);
2872
2873 if (unpack_unreachable_expiration) {
2874 revs.ignore_missing_links = 1;
@@ -2952,6 +3004,10 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
3004 N_("use a bitmap index if available to speed up counting objects")),
3005 OPT_BOOL(0, "write-bitmap-index", &write_bitmap_index,
3006 N_("write a bitmap index together with the pack index")),
3007 + OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),
3008 + { OPTION_CALLBACK, 0, "missing", NULL, N_("action"),
3009 + N_("handling for missing objects"), PARSE_OPT_NONEG,
3010 + option_parse_missing_action },
3011 OPT_END(),
3012 };
3013
@@ -3028,6 +3084,12 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
3084 if (!rev_list_all || !rev_list_reflog || !rev_list_index)
3085 unpack_unreachable_expiration = 0;
3086
3087 + if (filter_options.choice) {
3088 + if (!pack_to_stdout)
3089 + die("cannot use --filter without --stdout.");
3090 + use_bitmap_index = 0;
3091 + }
3092 +
3093 /*
3094 * "soft" reasons not to use bitmaps - for on-disk repack by default we want
3095 *
t/t5317-pack-objects-filter-objects.sh new
+375
@@ -0,0 +1,375 @@
1 +#!/bin/sh
2 +
3 +test_description='git pack-objects using object filtering'
4 +
5 +. ./test-lib.sh
6 +
7 +# Test blob:none filter.
8 +
9 +test_expect_success 'setup r1' '
10 + echo "{print \$1}" >print_1.awk &&
11 + echo "{print \$2}" >print_2.awk &&
12 +
13 + git init r1 &&
14 + for n in 1 2 3 4 5
15 + do
16 + echo "This is file: $n" > r1/file.$n
17 + git -C r1 add file.$n
18 + git -C r1 commit -m "$n"
19 + done
20 +'
21 +
22 +test_expect_success 'verify blob count in normal packfile' '
23 + git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \
24 + | awk -f print_2.awk \
25 + | sort >expected &&
26 + git -C r1 pack-objects --rev --stdout >all.pack <<-EOF &&
27 + HEAD
28 + EOF
29 + git -C r1 index-pack ../all.pack &&
30 + git -C r1 verify-pack -v ../all.pack \
31 + | grep blob \
32 + | awk -f print_1.awk \
33 + | sort >observed &&
34 + test_cmp observed expected
35 +'
36 +
37 +test_expect_success 'verify blob:none packfile has no blobs' '
38 + git -C r1 pack-objects --rev --stdout --filter=blob:none >filter.pack <<-EOF &&
39 + HEAD
40 + EOF
41 + git -C r1 index-pack ../filter.pack &&
42 + git -C r1 verify-pack -v ../filter.pack \
43 + | grep blob \
44 + | awk -f print_1.awk \
45 + | sort >observed &&
46 + nr=$(wc -l <observed) &&
47 + test 0 -eq $nr
48 +'
49 +
50 +test_expect_success 'verify normal and blob:none packfiles have same commits/trees' '
51 + git -C r1 verify-pack -v ../all.pack \
52 + | grep -E "commit|tree" \
53 + | awk -f print_1.awk \
54 + | sort >expected &&
55 + git -C r1 verify-pack -v ../filter.pack \
56 + | grep -E "commit|tree" \
57 + | awk -f print_1.awk \
58 + | sort >observed &&
59 + test_cmp observed expected
60 +'
61 +
62 +# Test blob:limit=<n>[kmg] filter.
63 +# We boundary test around the size parameter. The filter is strictly less than
64 +# the value, so size 500 and 1000 should have the same results, but 1001 should
65 +# filter more.
66 +
67 +test_expect_success 'setup r2' '
68 + git init r2 &&
69 + for n in 1000 10000
70 + do
71 + printf "%"$n"s" X > r2/large.$n
72 + git -C r2 add large.$n
73 + git -C r2 commit -m "$n"
74 + done
75 +'
76 +
77 +test_expect_success 'verify blob count in normal packfile' '
78 + git -C r2 ls-files -s large.1000 large.10000 \
79 + | awk -f print_2.awk \
80 + | sort >expected &&
81 + git -C r2 pack-objects --rev --stdout >all.pack <<-EOF &&
82 + HEAD
83 + EOF
84 + git -C r2 index-pack ../all.pack &&
85 + git -C r2 verify-pack -v ../all.pack \
86 + | grep blob \
87 + | awk -f print_1.awk \
88 + | sort >observed &&
89 + test_cmp observed expected
90 +'
91 +
92 +test_expect_success 'verify blob:limit=500 omits all blobs' '
93 + git -C r2 pack-objects --rev --stdout --filter=blob:limit=500 >filter.pack <<-EOF &&
94 + HEAD
95 + EOF
96 + git -C r2 index-pack ../filter.pack &&
97 + git -C r2 verify-pack -v ../filter.pack \
98 + | grep blob \
99 + | awk -f print_1.awk \
100 + | sort >observed &&
101 + nr=$(wc -l <observed) &&
102 + test 0 -eq $nr
103 +'
104 +
105 +test_expect_success 'verify blob:limit=1000' '
106 + git -C r2 pack-objects --rev --stdout --filter=blob:limit=1000 >filter.pack <<-EOF &&
107 + HEAD
108 + EOF
109 + git -C r2 index-pack ../filter.pack &&
110 + git -C r2 verify-pack -v ../filter.pack \
111 + | grep blob \
112 + | awk -f print_1.awk \
113 + | sort >observed &&
114 + nr=$(wc -l <observed) &&
115 + test 0 -eq $nr
116 +'
117 +
118 +test_expect_success 'verify blob:limit=1001' '
119 + git -C r2 ls-files -s large.1000 \
120 + | awk -f print_2.awk \
121 + | sort >expected &&
122 + git -C r2 pack-objects --rev --stdout --filter=blob:limit=1001 >filter.pack <<-EOF &&
123 + HEAD
124 + EOF
125 + git -C r2 index-pack ../filter.pack &&
126 + git -C r2 verify-pack -v ../filter.pack \
127 + | grep blob \
128 + | awk -f print_1.awk \
129 + | sort >observed &&
130 + test_cmp observed expected
131 +'
132 +
133 +test_expect_success 'verify blob:limit=10001' '
134 + git -C r2 ls-files -s large.1000 large.10000 \
135 + | awk -f print_2.awk \
136 + | sort >expected &&
137 + git -C r2 pack-objects --rev --stdout --filter=blob:limit=10001 >filter.pack <<-EOF &&
138 + HEAD
139 + EOF
140 + git -C r2 index-pack ../filter.pack &&
141 + git -C r2 verify-pack -v ../filter.pack \
142 + | grep blob \
143 + | awk -f print_1.awk \
144 + | sort >observed &&
145 + test_cmp observed expected
146 +'
147 +
148 +test_expect_success 'verify blob:limit=1k' '
149 + git -C r2 ls-files -s large.1000 \
150 + | awk -f print_2.awk \
151 + | sort >expected &&
152 + git -C r2 pack-objects --rev --stdout --filter=blob:limit=1k >filter.pack <<-EOF &&
153 + HEAD
154 + EOF
155 + git -C r2 index-pack ../filter.pack &&
156 + git -C r2 verify-pack -v ../filter.pack \
157 + | grep blob \
158 + | awk -f print_1.awk \
159 + | sort >observed &&
160 + test_cmp observed expected
161 +'
162 +
163 +test_expect_success 'verify blob:limit=1m' '
164 + git -C r2 ls-files -s large.1000 large.10000 \
165 + | awk -f print_2.awk \
166 + | sort >expected &&
167 + git -C r2 pack-objects --rev --stdout --filter=blob:limit=1m >filter.pack <<-EOF &&
168 + HEAD
169 + EOF
170 + git -C r2 index-pack ../filter.pack &&
171 + git -C r2 verify-pack -v ../filter.pack \
172 + | grep blob \
173 + | awk -f print_1.awk \
174 + | sort >observed &&
175 + test_cmp observed expected
176 +'
177 +
178 +test_expect_success 'verify normal and blob:limit packfiles have same commits/trees' '
179 + git -C r2 verify-pack -v ../all.pack \
180 + | grep -E "commit|tree" \
181 + | awk -f print_1.awk \
182 + | sort >expected &&
183 + git -C r2 verify-pack -v ../filter.pack \
184 + | grep -E "commit|tree" \
185 + | awk -f print_1.awk \
186 + | sort >observed &&
187 + test_cmp observed expected
188 +'
189 +
190 +# Test sparse:path=<path> filter.
191 +# Use a local file containing a sparse-checkout specification to filter
192 +# out blobs not required for the corresponding sparse-checkout. We do not
193 +# require sparse-checkout to actually be enabled.
194 +
195 +test_expect_success 'setup r3' '
196 + git init r3 &&
197 + mkdir r3/dir1 &&
198 + for n in sparse1 sparse2
199 + do
200 + echo "This is file: $n" > r3/$n
201 + git -C r3 add $n
202 + echo "This is file: dir1/$n" > r3/dir1/$n
203 + git -C r3 add dir1/$n
204 + done &&
205 + git -C r3 commit -m "sparse" &&
206 + echo dir1/ >pattern1 &&
207 + echo sparse1 >pattern2
208 +'
209 +
210 +test_expect_success 'verify blob count in normal packfile' '
211 + git -C r3 ls-files -s sparse1 sparse2 dir1/sparse1 dir1/sparse2 \
212 + | awk -f print_2.awk \
213 + | sort >expected &&
214 + git -C r3 pack-objects --rev --stdout >all.pack <<-EOF &&
215 + HEAD
216 + EOF
217 + git -C r3 index-pack ../all.pack &&
218 + git -C r3 verify-pack -v ../all.pack \
219 + | grep blob \
220 + | awk -f print_1.awk \
221 + | sort >observed &&
222 + test_cmp observed expected
223 +'
224 +
225 +test_expect_success 'verify sparse:path=pattern1' '
226 + git -C r3 ls-files -s dir1/sparse1 dir1/sparse2 \
227 + | awk -f print_2.awk \
228 + | sort >expected &&
229 + git -C r3 pack-objects --rev --stdout --filter=sparse:path=../pattern1 >filter.pack <<-EOF &&
230 + HEAD
231 + EOF
232 + git -C r3 index-pack ../filter.pack &&
233 + git -C r3 verify-pack -v ../filter.pack \
234 + | grep blob \
235 + | awk -f print_1.awk \
236 + | sort >observed &&
237 + test_cmp observed expected
238 +'
239 +
240 +test_expect_success 'verify normal and sparse:path=pattern1 packfiles have same commits/trees' '
241 + git -C r3 verify-pack -v ../all.pack \
242 + | grep -E "commit|tree" \
243 + | awk -f print_1.awk \
244 + | sort >expected &&
245 + git -C r3 verify-pack -v ../filter.pack \
246 + | grep -E "commit|tree" \
247 + | awk -f print_1.awk \
248 + | sort >observed &&
249 + test_cmp observed expected
250 +'
251 +
252 +test_expect_success 'verify sparse:path=pattern2' '
253 + git -C r3 ls-files -s sparse1 dir1/sparse1 \
254 + | awk -f print_2.awk \
255 + | sort >expected &&
256 + git -C r3 pack-objects --rev --stdout --filter=sparse:path=../pattern2 >filter.pack <<-EOF &&
257 + HEAD
258 + EOF
259 + git -C r3 index-pack ../filter.pack &&
260 + git -C r3 verify-pack -v ../filter.pack \
261 + | grep blob \
262 + | awk -f print_1.awk \
263 + | sort >observed &&
264 + test_cmp observed expected
265 +'
266 +
267 +test_expect_success 'verify normal and sparse:path=pattern2 packfiles have same commits/trees' '
268 + git -C r3 verify-pack -v ../all.pack \
269 + | grep -E "commit|tree" \
270 + | awk -f print_1.awk \
271 + | sort >expected &&
272 + git -C r3 verify-pack -v ../filter.pack \
273 + | grep -E "commit|tree" \
274 + | awk -f print_1.awk \
275 + | sort >observed &&
276 + test_cmp observed expected
277 +'
278 +
279 +# Test sparse:oid=<oid-ish> filter.
280 +# Like sparse:path, but we get the sparse-checkout specification from
281 +# a blob rather than a file on disk.
282 +
283 +test_expect_success 'setup r4' '
284 + git init r4 &&
285 + mkdir r4/dir1 &&
286 + for n in sparse1 sparse2
287 + do
288 + echo "This is file: $n" > r4/$n
289 + git -C r4 add $n
290 + echo "This is file: dir1/$n" > r4/dir1/$n
291 + git -C r4 add dir1/$n
292 + done &&
293 + echo dir1/ >r4/pattern &&
294 + git -C r4 add pattern &&
295 + git -C r4 commit -m "pattern"
296 +'
297 +
298 +test_expect_success 'verify blob count in normal packfile' '
299 + git -C r4 ls-files -s pattern sparse1 sparse2 dir1/sparse1 dir1/sparse2 \
300 + | awk -f print_2.awk \
301 + | sort >expected &&
302 + git -C r4 pack-objects --rev --stdout >all.pack <<-EOF &&
303 + HEAD
304 + EOF
305 + git -C r4 index-pack ../all.pack &&
306 + git -C r4 verify-pack -v ../all.pack \
307 + | grep blob \
308 + | awk -f print_1.awk \
309 + | sort >observed &&
310 + test_cmp observed expected
311 +'
312 +
313 +test_expect_success 'verify sparse:oid=OID' '
314 + git -C r4 ls-files -s dir1/sparse1 dir1/sparse2 \
315 + | awk -f print_2.awk \
316 + | sort >expected &&
317 + oid=$(git -C r4 ls-files -s pattern | awk -f print_2.awk) &&
318 + git -C r4 pack-objects --rev --stdout --filter=sparse:oid=$oid >filter.pack <<-EOF &&
319 + HEAD
320 + EOF
321 + git -C r4 index-pack ../filter.pack &&
322 + git -C r4 verify-pack -v ../filter.pack \
323 + | grep blob \
324 + | awk -f print_1.awk \
325 + | sort >observed &&
326 + test_cmp observed expected
327 +'
328 +
329 +test_expect_success 'verify sparse:oid=oid-ish' '
330 + git -C r4 ls-files -s dir1/sparse1 dir1/sparse2 \
331 + | awk -f print_2.awk \
332 + | sort >expected &&
333 + git -C r4 pack-objects --rev --stdout --filter=sparse:oid=master:pattern >filter.pack <<-EOF &&
334 + HEAD
335 + EOF
336 + git -C r4 index-pack ../filter.pack &&
337 + git -C r4 verify-pack -v ../filter.pack \
338 + | grep blob \
339 + | awk -f print_1.awk \
340 + | sort >observed &&
341 + test_cmp observed expected
342 +'
343 +
344 +# Delete some loose objects and use pack-objects, but WITHOUT any filtering.
345 +# This models previously omitted objects that we did not receive.
346 +
347 +test_expect_success 'setup r1 - delete loose blobs' '
348 + git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \
349 + | awk -f print_2.awk \
350 + | sort >expected &&
351 + for id in `cat expected | sed "s|..|&/|"`
352 + do
353 + rm r1/.git/objects/$id
354 + done
355 +'
356 +
357 +test_expect_success 'verify pack-objects fails w/ missing objects' '
358 + test_must_fail git -C r1 pack-objects --rev --stdout >miss.pack <<-EOF
359 + HEAD
360 + EOF
361 +'
362 +
363 +test_expect_success 'verify pack-objects fails w/ --missing=error' '
364 + test_must_fail git -C r1 pack-objects --rev --stdout --missing=error >miss.pack <<-EOF
365 + HEAD
366 + EOF
367 +'
368 +
369 +test_expect_success 'verify pack-objects w/ --missing=allow-any' '
370 + git -C r1 pack-objects --rev --stdout --missing=allow-any >miss.pack <<-EOF
371 + HEAD
372 + EOF
373 +'
374 +
375 +test_done