rev-list: add list-objects filtering support
Teach rev-list to use the filtering provided by the traverse_commit_list_filtered() interface to omit unwanted objects from the result. Object filtering is only allowed when one of the "--objects*" options are used. When the "--filter-print-omitted" option is used, the omitted objects are printed at the end. These are marked with a "~". This option can be combined with "--quiet" to get a list of just the omitted objects. Add t6112 test. In the future, we will introduce a "partial clone" mechanism wherein an object in a repo, obtained from a remote, may reference a missing object that can be dynamically fetched from that remote once needed. This "partial clone" mechanism will have a way, sometimes slow, of determining if a missing link is one of the links expected to be produced by this mechanism. This patch introduces handling of missing objects to help debugging and development of the "partial clone" mechanism, and once the mechanism is implemented, for a power user to perform operations that are missing-object aware without incurring the cost of checking if a missing link is expected. Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com> Reviewed-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jeff Hostetler committed
Nov 21, 2017 at 20:58 UTC
caf3827e2f6a8a0315301d52b5bf46e269b3a388
4 files changed
+370
-3
Documentation/git-rev-list.txt
+3
-1
@@ -47,7 +47,9 @@ SYNOPSIS
47
[ --fixed-strings | -F ]
48
[ --date=<format>]
49
[ [ --objects | --objects-edge | --objects-edge-aggressive ]
50
- [ --unpacked ] ]
50
+ [ --unpacked ]
51
+ [ --filter=<filter-spec> [ --filter-print-omitted ] ] ]
52
+ [ --missing=<missing-action> ]
53
[ --pretty | --header ]
54
[ --bisect ]
55
[ --bisect-vars ]
Documentation/rev-list-options.txt
+36
@@ -706,6 +706,42 @@ ifdef::git-rev-list[]
706
--unpacked::
707
Only useful with `--objects`; print the object IDs that are not
708
in packs.
709
+
710
+--filter=<filter-spec>::
711
+ Only useful with one of the `--objects*`; omits objects (usually
712
+ blobs) from the list of printed objects. The '<filter-spec>'
713
+ may be one of the following:
714
++
715
+The form '--filter=blob:none' omits all blobs.
716
++
717
+The form '--filter=blob:limit=<n>[kmg]' omits blobs larger than n bytes
718
+or units. The value may be zero.
719
++
720
+The form '--filter=sparse:oid=<oid-ish>' uses a sparse-checkout
721
+specification contained in the object (or the object that the expression
722
+evaluates to) to omit blobs that would not be not required for a
723
+sparse checkout on the requested refs.
724
++
725
+The form '--filter=sparse:path=<path>' similarly uses a sparse-checkout
726
+specification contained in <path>.
727
+
728
+--filter-print-omitted::
729
+ Only useful with `--filter=`; prints a list of the objects omitted
730
+ by the filter. Object IDs are prefixed with a ``~'' character.
731
+
732
+--missing=<missing-action>::
733
+ A debug option to help with future "partial clone" development.
734
+ This option specifies how missing objects are handled.
735
++
736
+The form '--missing=error' requests that rev-list stop with an error if
737
+a missing object is encountered. This is the default action.
738
++
739
+The form '--missing=allow-any' will allow object traversal to continue
740
+if a missing object is encountered. Missing objects will silently be
741
+omitted from the results.
742
++
743
+The form '--missing=print' is like 'allow-any', but will also print a
744
+list of the missing objects. Object IDs are prefixed with a ``?'' character.
745
endif::git-rev-list[]
746
747
--no-walk[=(sorted|unsorted)]::
builtin/rev-list.c
+106
-2
@@ -4,6 +4,8 @@
4
#include "diff.h"
5
#include "revision.h"
6
#include "list-objects.h"
7
+#include "list-objects-filter.h"
8
+#include "list-objects-filter-options.h"
9
#include "pack.h"
10
#include "pack-bitmap.h"
11
#include "builtin.h"
@@ -12,6 +14,7 @@
14
#include "bisect.h"
15
#include "progress.h"
16
#include "reflog-walk.h"
17
+#include "oidset.h"
18
19
static const char rev_list_usage[] =
20
"git rev-list [OPTION] <commit-id>... [ -- paths... ]\n"
@@ -55,6 +58,20 @@ static const char rev_list_usage[] =
58
static struct progress *progress;
59
static unsigned progress_counter;
60
61
+static struct list_objects_filter_options filter_options;
62
+static struct oidset omitted_objects;
63
+static int arg_print_omitted; /* print objects omitted by filter */
64
+
65
+static struct oidset missing_objects;
66
+enum missing_action {
67
+ MA_ERROR = 0, /* fail if any missing objects are encountered */
68
+ MA_ALLOW_ANY, /* silently allow ALL missing objects */
69
+ MA_PRINT, /* print ALL missing objects in special section */
70
+};
71
+static enum missing_action arg_missing_action;
72
+
73
+#define DEFAULT_OIDSET_SIZE (16*1024)
74
+
75
static void finish_commit(struct commit *commit, void *data);
76
static void show_commit(struct commit *commit, void *data)
77
{
@@ -178,11 +195,31 @@ static void finish_commit(struct commit *commit, void *data)
195
free_commit_buffer(commit);
196
}
197
198
+static inline void finish_object__ma(struct object *obj)
199
+{
200
+ switch (arg_missing_action) {
201
+ case MA_ERROR:
202
+ die("missing blob object '%s'", oid_to_hex(&obj->oid));
203
+ return;
204
+
205
+ case MA_ALLOW_ANY:
206
+ return;
207
+
208
+ case MA_PRINT:
209
+ oidset_insert(&missing_objects, &obj->oid);
210
+ return;
211
+
212
+ default:
213
+ BUG("unhandled missing_action");
214
+ return;
215
+ }
216
+}
217
+
218
static void finish_object(struct object *obj, const char *name, void *cb_data)
219
{
220
struct rev_list_info *info = cb_data;
221
if (obj->type == OBJ_BLOB && !has_object_file(&obj->oid))
185
- die("missing blob object '%s'", oid_to_hex(&obj->oid));
222
+ finish_object__ma(obj);
223
if (info->revs->verify_objects && !obj->parsed && obj->type != OBJ_COMMIT)
224
parse_object(&obj->oid);
225
}
@@ -269,6 +306,26 @@ static int show_object_fast(
306
return 1;
307
}
308
309
+static inline int parse_missing_action_value(const char *value)
310
+{
311
+ if (!strcmp(value, "error")) {
312
+ arg_missing_action = MA_ERROR;
313
+ return 1;
314
+ }
315
+
316
+ if (!strcmp(value, "allow-any")) {
317
+ arg_missing_action = MA_ALLOW_ANY;
318
+ return 1;
319
+ }
320
+
321
+ if (!strcmp(value, "print")) {
322
+ arg_missing_action = MA_PRINT;
323
+ return 1;
324
+ }
325
+
326
+ return 0;
327
+}
328
+
329
int cmd_rev_list(int argc, const char **argv, const char *prefix)
330
{
331
struct rev_info revs;
@@ -335,6 +392,26 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix)
392
show_progress = arg;
393
continue;
394
}
395
+
396
+ if (skip_prefix(arg, ("--" CL_ARG__FILTER "="), &arg)) {
397
+ parse_list_objects_filter(&filter_options, arg);
398
+ if (filter_options.choice && !revs.blob_objects)
399
+ die(_("object filtering requires --objects"));
400
+ if (filter_options.choice == LOFC_SPARSE_OID &&
401
+ !filter_options.sparse_oid_value)
402
+ die(_("invalid sparse value '%s'"),
403
+ filter_options.filter_spec);
404
+ continue;
405
+ }
406
+ if (!strcmp(arg, "--filter-print-omitted")) {
407
+ arg_print_omitted = 1;
408
+ continue;
409
+ }
410
+
411
+ if (skip_prefix(arg, "--missing=", &arg) &&
412
+ parse_missing_action_value(arg))
413
+ continue;
414
+
415
usage(rev_list_usage);
416
417
}
@@ -360,6 +437,9 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix)
437
if (revs.show_notes)
438
die(_("rev-list does not support display of notes"));
439
440
+ if (filter_options.choice && use_bitmap_index)
441
+ die(_("cannot combine --use-bitmap-index with object filtering"));
442
+
443
save_commit_buffer = (revs.verbose_header ||
444
revs.grep_filter.pattern_list ||
445
revs.grep_filter.header_list);
@@ -404,7 +484,31 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix)
484
return show_bisect_vars(&info, reaches, all);
485
}
486
407
- traverse_commit_list(&revs, show_commit, show_object, &info);
487
+ if (arg_print_omitted)
488
+ oidset_init(&omitted_objects, DEFAULT_OIDSET_SIZE);
489
+ if (arg_missing_action == MA_PRINT)
490
+ oidset_init(&missing_objects, DEFAULT_OIDSET_SIZE);
491
+
492
+ traverse_commit_list_filtered(
493
+ &filter_options, &revs, show_commit, show_object, &info,
494
+ (arg_print_omitted ? &omitted_objects : NULL));
495
+
496
+ if (arg_print_omitted) {
497
+ struct oidset_iter iter;
498
+ struct object_id *oid;
499
+ oidset_iter_init(&omitted_objects, &iter);
500
+ while ((oid = oidset_iter_next(&iter)))
501
+ printf("~%s\n", oid_to_hex(oid));
502
+ oidset_clear(&omitted_objects);
503
+ }
504
+ if (arg_missing_action == MA_PRINT) {
505
+ struct oidset_iter iter;
506
+ struct object_id *oid;
507
+ oidset_iter_init(&missing_objects, &iter);
508
+ while ((oid = oidset_iter_next(&iter)))
509
+ printf("?%s\n", oid_to_hex(oid));
510
+ oidset_clear(&missing_objects);
511
+ }
512
513
stop_progress(&progress);
514
t/t6112-rev-list-filters-objects.sh
new
+225
@@ -0,0 +1,225 @@
1
+#!/bin/sh
2
+
3
+test_description='git rev-list using object filtering'
4
+
5
+. ./test-lib.sh
6
+
7
+# Test the blob:none filter.
8
+
9
+test_expect_success 'setup r1' '
10
+ echo "{print \$1}" >print_1.awk &&
11
+ echo "{print \$2}" >print_2.awk &&
12
+
13
+ git init r1 &&
14
+ for n in 1 2 3 4 5
15
+ do
16
+ echo "This is file: $n" > r1/file.$n
17
+ git -C r1 add file.$n
18
+ git -C r1 commit -m "$n"
19
+ done
20
+'
21
+
22
+test_expect_success 'verify blob:none omits all 5 blobs' '
23
+ git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \
24
+ | awk -f print_2.awk \
25
+ | sort >expected &&
26
+ git -C r1 rev-list HEAD --quiet --objects --filter-print-omitted --filter=blob:none \
27
+ | awk -f print_1.awk \
28
+ | sed "s/~//" \
29
+ | sort >observed &&
30
+ test_cmp observed expected
31
+'
32
+
33
+test_expect_success 'verify emitted+omitted == all' '
34
+ git -C r1 rev-list HEAD --objects \
35
+ | awk -f print_1.awk \
36
+ | sort >expected &&
37
+ git -C r1 rev-list HEAD --objects --filter-print-omitted --filter=blob:none \
38
+ | awk -f print_1.awk \
39
+ | sed "s/~//" \
40
+ | sort >observed &&
41
+ test_cmp observed expected
42
+'
43
+
44
+
45
+# Test blob:limit=<n>[kmg] filter.
46
+# We boundary test around the size parameter. The filter is strictly less than
47
+# the value, so size 500 and 1000 should have the same results, but 1001 should
48
+# filter more.
49
+
50
+test_expect_success 'setup r2' '
51
+ git init r2 &&
52
+ for n in 1000 10000
53
+ do
54
+ printf "%"$n"s" X > r2/large.$n
55
+ git -C r2 add large.$n
56
+ git -C r2 commit -m "$n"
57
+ done
58
+'
59
+
60
+test_expect_success 'verify blob:limit=500 omits all blobs' '
61
+ git -C r2 ls-files -s large.1000 large.10000 \
62
+ | awk -f print_2.awk \
63
+ | sort >expected &&
64
+ git -C r2 rev-list HEAD --quiet --objects --filter-print-omitted --filter=blob:limit=500 \
65
+ | awk -f print_1.awk \
66
+ | sed "s/~//" \
67
+ | sort >observed &&
68
+ test_cmp observed expected
69
+'
70
+
71
+test_expect_success 'verify emitted+omitted == all' '
72
+ git -C r2 rev-list HEAD --objects \
73
+ | awk -f print_1.awk \
74
+ | sort >expected &&
75
+ git -C r2 rev-list HEAD --objects --filter-print-omitted --filter=blob:limit=500 \
76
+ | awk -f print_1.awk \
77
+ | sed "s/~//" \
78
+ | sort >observed &&
79
+ test_cmp observed expected
80
+'
81
+
82
+test_expect_success 'verify blob:limit=1000' '
83
+ git -C r2 ls-files -s large.1000 large.10000 \
84
+ | awk -f print_2.awk \
85
+ | sort >expected &&
86
+ git -C r2 rev-list HEAD --quiet --objects --filter-print-omitted --filter=blob:limit=1000 \
87
+ | awk -f print_1.awk \
88
+ | sed "s/~//" \
89
+ | sort >observed &&
90
+ test_cmp observed expected
91
+'
92
+
93
+test_expect_success 'verify blob:limit=1001' '
94
+ git -C r2 ls-files -s large.10000 \
95
+ | awk -f print_2.awk \
96
+ | sort >expected &&
97
+ git -C r2 rev-list HEAD --quiet --objects --filter-print-omitted --filter=blob:limit=1001 \
98
+ | awk -f print_1.awk \
99
+ | sed "s/~//" \
100
+ | sort >observed &&
101
+ test_cmp observed expected
102
+'
103
+
104
+test_expect_success 'verify blob:limit=1k' '
105
+ git -C r2 ls-files -s large.10000 \
106
+ | awk -f print_2.awk \
107
+ | sort >expected &&
108
+ git -C r2 rev-list HEAD --quiet --objects --filter-print-omitted --filter=blob:limit=1k \
109
+ | awk -f print_1.awk \
110
+ | sed "s/~//" \
111
+ | sort >observed &&
112
+ test_cmp observed expected
113
+'
114
+
115
+test_expect_success 'verify blob:limit=1m' '
116
+ cat </dev/null >expected &&
117
+ git -C r2 rev-list HEAD --quiet --objects --filter-print-omitted --filter=blob:limit=1m \
118
+ | awk -f print_1.awk \
119
+ | sed "s/~//" \
120
+ | sort >observed &&
121
+ test_cmp observed expected
122
+'
123
+
124
+# Test sparse:path=<path> filter.
125
+# Use a local file containing a sparse-checkout specification to filter
126
+# out blobs not required for the corresponding sparse-checkout. We do not
127
+# require sparse-checkout to actually be enabled.
128
+
129
+test_expect_success 'setup r3' '
130
+ git init r3 &&
131
+ mkdir r3/dir1 &&
132
+ for n in sparse1 sparse2
133
+ do
134
+ echo "This is file: $n" > r3/$n
135
+ git -C r3 add $n
136
+ echo "This is file: dir1/$n" > r3/dir1/$n
137
+ git -C r3 add dir1/$n
138
+ done &&
139
+ git -C r3 commit -m "sparse" &&
140
+ echo dir1/ >pattern1 &&
141
+ echo sparse1 >pattern2
142
+'
143
+
144
+test_expect_success 'verify sparse:path=pattern1 omits top-level files' '
145
+ git -C r3 ls-files -s sparse1 sparse2 \
146
+ | awk -f print_2.awk \
147
+ | sort >expected &&
148
+ git -C r3 rev-list HEAD --quiet --objects --filter-print-omitted --filter=sparse:path=../pattern1 \
149
+ | awk -f print_1.awk \
150
+ | sed "s/~//" \
151
+ | sort >observed &&
152
+ test_cmp observed expected
153
+'
154
+
155
+test_expect_success 'verify sparse:path=pattern2 omits both sparse2 files' '
156
+ git -C r3 ls-files -s sparse2 dir1/sparse2 \
157
+ | awk -f print_2.awk \
158
+ | sort >expected &&
159
+ git -C r3 rev-list HEAD --quiet --objects --filter-print-omitted --filter=sparse:path=../pattern2 \
160
+ | awk -f print_1.awk \
161
+ | sed "s/~//" \
162
+ | sort >observed &&
163
+ test_cmp observed expected
164
+'
165
+
166
+# Test sparse:oid=<oid-ish> filter.
167
+# Like sparse:path, but we get the sparse-checkout specification from
168
+# a blob rather than a file on disk.
169
+
170
+test_expect_success 'setup r3 part 2' '
171
+ echo dir1/ >r3/pattern &&
172
+ git -C r3 add pattern &&
173
+ git -C r3 commit -m "pattern"
174
+'
175
+
176
+test_expect_success 'verify sparse:oid=OID omits top-level files' '
177
+ git -C r3 ls-files -s pattern sparse1 sparse2 \
178
+ | awk -f print_2.awk \
179
+ | sort >expected &&
180
+ oid=$(git -C r3 ls-files -s pattern | awk -f print_2.awk) &&
181
+ git -C r3 rev-list HEAD --quiet --objects --filter-print-omitted --filter=sparse:oid=$oid \
182
+ | awk -f print_1.awk \
183
+ | sed "s/~//" \
184
+ | sort >observed &&
185
+ test_cmp observed expected
186
+'
187
+
188
+test_expect_success 'verify sparse:oid=oid-ish omits top-level files' '
189
+ git -C r3 ls-files -s pattern sparse1 sparse2 \
190
+ | awk -f print_2.awk \
191
+ | sort >expected &&
192
+ git -C r3 rev-list HEAD --quiet --objects --filter-print-omitted --filter=sparse:oid=master:pattern \
193
+ | awk -f print_1.awk \
194
+ | sed "s/~//" \
195
+ | sort >observed &&
196
+ test_cmp observed expected
197
+'
198
+
199
+# Delete some loose objects and use rev-list, but WITHOUT any filtering.
200
+# This models previously omitted objects that we did not receive.
201
+
202
+test_expect_success 'rev-list W/ --missing=print' '
203
+ git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \
204
+ | awk -f print_2.awk \
205
+ | sort >expected &&
206
+ for id in `cat expected | sed "s|..|&/|"`
207
+ do
208
+ rm r1/.git/objects/$id
209
+ done &&
210
+ git -C r1 rev-list --quiet HEAD --missing=print --objects \
211
+ | awk -f print_1.awk \
212
+ | sed "s/?//" \
213
+ | sort >observed &&
214
+ test_cmp observed expected
215
+'
216
+
217
+test_expect_success 'rev-list W/O --missing fails' '
218
+ test_must_fail git -C r1 rev-list --quiet --objects HEAD
219
+'
220
+
221
+test_expect_success 'rev-list W/ missing=allow-any' '
222
+ git -C r1 rev-list --quiet --missing=allow-any --objects HEAD
223
+'
224
+
225
+test_done