rev-list: support termination at promisor objects
Teach rev-list to support termination of an object traversal at any object from a promisor remote (whether one that the local repo also has, or one that the local repo knows about because it has another promisor object that references it). This will be used subsequently in gc and in the connectivity check used by fetch. For efficiency, if an object is referenced by a promisor object, and is in the local repo only as a non-promisor object, object traversal will not stop there. This is to avoid building the list of promisor object references. (In list-objects.c, the case where obj is NULL in process_blob() and process_tree() do not need to be changed because those happen only when there is a conflict between the expected type and the existing object. If the object doesn't exist, an object will be synthesized, which is fine.) Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jonathan Tan committed
Dec 8, 2017 at 15:27 UTC
df11e1964825b825e179ccdbc1b9e3a6fc09e67a
7 files changed
+239
-11
Documentation/rev-list-options.txt
+11
@@ -745,10 +745,21 @@ The form '--missing=allow-any' will allow object traversal to continue
745
if a missing object is encountered. Missing objects will silently be
746
omitted from the results.
747
+
748
+The form '--missing=allow-promisor' is like 'allow-any', but will only
749
+allow object traversal to continue for EXPECTED promisor missing objects.
750
+Unexpected missing objects will raise an error.
751
++
752
The form '--missing=print' is like 'allow-any', but will also print a
753
list of the missing objects. Object IDs are prefixed with a ``?'' character.
754
endif::git-rev-list[]
755
756
+--exclude-promisor-objects::
757
+ (For internal use only.) Prefilter object traversal at
758
+ promisor boundary. This is used with partial clone. This is
759
+ stronger than `--missing=allow-promisor` because it limits the
760
+ traversal, rather than just silencing errors about missing
761
+ objects.
762
+
763
--no-walk[=(sorted|unsorted)]::
764
Only show the given commits, but do not traverse their ancestors.
765
This has no effect if a range is specified. If the argument
builtin/rev-list.c
+63
-6
@@ -15,6 +15,7 @@
15
#include "progress.h"
16
#include "reflog-walk.h"
17
#include "oidset.h"
18
+#include "packfile.h"
19
20
static const char rev_list_usage[] =
21
"git rev-list [OPTION] <commit-id>... [ -- paths... ]\n"
@@ -67,6 +68,7 @@ enum missing_action {
68
MA_ERROR = 0, /* fail if any missing objects are encountered */
69
MA_ALLOW_ANY, /* silently allow ALL missing objects */
70
MA_PRINT, /* print ALL missing objects in special section */
71
+ MA_ALLOW_PROMISOR, /* silently allow all missing PROMISOR objects */
72
};
73
static enum missing_action arg_missing_action;
74
@@ -197,6 +199,12 @@ static void finish_commit(struct commit *commit, void *data)
199
200
static inline void finish_object__ma(struct object *obj)
201
{
202
+ /*
203
+ * Whether or not we try to dynamically fetch missing objects
204
+ * from the server, we currently DO NOT have the object. We
205
+ * can either print, allow (ignore), or conditionally allow
206
+ * (ignore) them.
207
+ */
208
switch (arg_missing_action) {
209
case MA_ERROR:
210
die("missing blob object '%s'", oid_to_hex(&obj->oid));
@@ -209,25 +217,36 @@ static inline void finish_object__ma(struct object *obj)
217
oidset_insert(&missing_objects, &obj->oid);
218
return;
219
220
+ case MA_ALLOW_PROMISOR:
221
+ if (is_promisor_object(&obj->oid))
222
+ return;
223
+ die("unexpected missing blob object '%s'",
224
+ oid_to_hex(&obj->oid));
225
+ return;
226
+
227
default:
228
BUG("unhandled missing_action");
229
return;
230
}
231
}
232
218
-static void finish_object(struct object *obj, const char *name, void *cb_data)
233
+static int finish_object(struct object *obj, const char *name, void *cb_data)
234
{
235
struct rev_list_info *info = cb_data;
221
- if (obj->type == OBJ_BLOB && !has_object_file(&obj->oid))
236
+ if (obj->type == OBJ_BLOB && !has_object_file(&obj->oid)) {
237
finish_object__ma(obj);
238
+ return 1;
239
+ }
240
if (info->revs->verify_objects && !obj->parsed && obj->type != OBJ_COMMIT)
241
parse_object(&obj->oid);
242
+ return 0;
243
}
244
245
static void show_object(struct object *obj, const char *name, void *cb_data)
246
{
247
struct rev_list_info *info = cb_data;
230
- finish_object(obj, name, cb_data);
248
+ if (finish_object(obj, name, cb_data))
249
+ return;
250
display_progress(progress, ++progress_counter);
251
if (info->flags & REV_LIST_QUIET)
252
return;
@@ -315,11 +334,19 @@ static inline int parse_missing_action_value(const char *value)
334
335
if (!strcmp(value, "allow-any")) {
336
arg_missing_action = MA_ALLOW_ANY;
337
+ fetch_if_missing = 0;
338
return 1;
339
}
340
341
if (!strcmp(value, "print")) {
342
arg_missing_action = MA_PRINT;
343
+ fetch_if_missing = 0;
344
+ return 1;
345
+ }
346
+
347
+ if (!strcmp(value, "allow-promisor")) {
348
+ arg_missing_action = MA_ALLOW_PROMISOR;
349
+ fetch_if_missing = 0;
350
return 1;
351
}
352
@@ -344,6 +371,35 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix)
371
init_revisions(&revs, prefix);
372
revs.abbrev = DEFAULT_ABBREV;
373
revs.commit_format = CMIT_FMT_UNSPECIFIED;
374
+
375
+ /*
376
+ * Scan the argument list before invoking setup_revisions(), so that we
377
+ * know if fetch_if_missing needs to be set to 0.
378
+ *
379
+ * "--exclude-promisor-objects" acts as a pre-filter on missing objects
380
+ * by not crossing the boundary from realized objects to promisor
381
+ * objects.
382
+ *
383
+ * Let "--missing" to conditionally set fetch_if_missing.
384
+ */
385
+ for (i = 1; i < argc; i++) {
386
+ const char *arg = argv[i];
387
+ if (!strcmp(arg, "--exclude-promisor-objects")) {
388
+ fetch_if_missing = 0;
389
+ revs.exclude_promisor_objects = 1;
390
+ break;
391
+ }
392
+ }
393
+ for (i = 1; i < argc; i++) {
394
+ const char *arg = argv[i];
395
+ if (skip_prefix(arg, "--missing=", &arg)) {
396
+ if (revs.exclude_promisor_objects)
397
+ die(_("cannot combine --exclude-promisor-objects and --missing"));
398
+ if (parse_missing_action_value(arg))
399
+ break;
400
+ }
401
+ }
402
+
403
argc = setup_revisions(argc, argv, &revs, NULL);
404
405
memset(&info, 0, sizeof(info));
@@ -412,9 +468,10 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix)
468
continue;
469
}
470
415
- if (skip_prefix(arg, "--missing=", &arg) &&
416
- parse_missing_action_value(arg))
417
- continue;
471
+ if (!strcmp(arg, "--exclude-promisor-objects"))
472
+ continue; /* already handled above */
473
+ if (skip_prefix(arg, "--missing=", &arg))
474
+ continue; /* already handled above */
475
476
usage(rev_list_usage);
477
list-objects.c
+28
-1
@@ -9,6 +9,7 @@
9
#include "list-objects.h"
10
#include "list-objects-filter.h"
11
#include "list-objects-filter-options.h"
12
+#include "packfile.h"
13
14
static void process_blob(struct rev_info *revs,
15
struct blob *blob,
@@ -30,6 +31,20 @@ static void process_blob(struct rev_info *revs,
31
if (obj->flags & (UNINTERESTING | SEEN))
32
return;
33
34
+ /*
35
+ * Pre-filter known-missing objects when explicitly requested.
36
+ * Otherwise, a missing object error message may be reported
37
+ * later (depending on other filtering criteria).
38
+ *
39
+ * Note that this "--exclude-promisor-objects" pre-filtering
40
+ * may cause the actual filter to report an incomplete list
41
+ * of missing objects.
42
+ */
43
+ if (revs->exclude_promisor_objects &&
44
+ !has_object_file(&obj->oid) &&
45
+ is_promisor_object(&obj->oid))
46
+ return;
47
+
48
pathlen = path->len;
49
strbuf_addstr(path, name);
50
if (filter_fn)
@@ -91,6 +106,8 @@ static void process_tree(struct rev_info *revs,
106
all_entries_interesting: entry_not_interesting;
107
int baselen = base->len;
108
enum list_objects_filter_result r = LOFR_MARK_SEEN | LOFR_DO_SHOW;
109
+ int gently = revs->ignore_missing_links ||
110
+ revs->exclude_promisor_objects;
111
112
if (!revs->tree_objects)
113
return;
@@ -98,9 +115,19 @@ static void process_tree(struct rev_info *revs,
115
die("bad tree object");
116
if (obj->flags & (UNINTERESTING | SEEN))
117
return;
101
- if (parse_tree_gently(tree, revs->ignore_missing_links) < 0) {
118
+ if (parse_tree_gently(tree, gently) < 0) {
119
if (revs->ignore_missing_links)
120
return;
121
+
122
+ /*
123
+ * Pre-filter known-missing tree objects when explicitly
124
+ * requested. This may cause the actual filter to report
125
+ * an incomplete list of missing objects.
126
+ */
127
+ if (revs->exclude_promisor_objects &&
128
+ is_promisor_object(&obj->oid))
129
+ return;
130
+
131
die("bad tree object %s", oid_to_hex(&obj->oid));
132
}
133
object.c
+1
-1
@@ -252,7 +252,7 @@ struct object *parse_object(const struct object_id *oid)
252
if (obj && obj->parsed)
253
return obj;
254
255
- if ((obj && obj->type == OBJ_BLOB) ||
255
+ if ((obj && obj->type == OBJ_BLOB && has_object_file(oid)) ||
256
(!obj && has_object_file(oid) &&
257
sha1_object_info(oid->hash, NULL) == OBJ_BLOB)) {
258
if (check_sha1_signature(repl, NULL, 0, NULL) < 0) {
revision.c
+31
-2
@@ -198,6 +198,8 @@ static struct object *get_reference(struct rev_info *revs, const char *name,
198
if (!object) {
199
if (revs->ignore_missing)
200
return object;
201
+ if (revs->exclude_promisor_objects && is_promisor_object(oid))
202
+ return NULL;
203
die("bad object %s", name);
204
}
205
object->flags |= flags;
@@ -790,9 +792,17 @@ static int add_parents_to_list(struct rev_info *revs, struct commit *commit,
792
793
for (parent = commit->parents; parent; parent = parent->next) {
794
struct commit *p = parent->item;
793
-
794
- if (parse_commit_gently(p, revs->ignore_missing_links) < 0)
795
+ int gently = revs->ignore_missing_links ||
796
+ revs->exclude_promisor_objects;
797
+ if (parse_commit_gently(p, gently) < 0) {
798
+ if (revs->exclude_promisor_objects &&
799
+ is_promisor_object(&p->object.oid)) {
800
+ if (revs->first_parent_only)
801
+ break;
802
+ continue;
803
+ }
804
return -1;
805
+ }
806
if (revs->show_source && !p->util)
807
p->util = commit->util;
808
p->object.flags |= left_flag;
@@ -2088,6 +2098,10 @@ static int handle_revision_opt(struct rev_info *revs, int argc, const char **arg
2098
revs->limited = 1;
2099
} else if (!strcmp(arg, "--ignore-missing")) {
2100
revs->ignore_missing = 1;
2101
+ } else if (!strcmp(arg, "--exclude-promisor-objects")) {
2102
+ if (fetch_if_missing)
2103
+ die("BUG: exclude_promisor_objects can only be used when fetch_if_missing is 0");
2104
+ revs->exclude_promisor_objects = 1;
2105
} else {
2106
int opts = diff_opt_parse(&revs->diffopt, argv, argc, revs->prefix);
2107
if (!opts)
@@ -2830,6 +2844,16 @@ void reset_revision_walk(void)
2844
clear_object_flags(SEEN | ADDED | SHOWN);
2845
}
2846
2847
+static int mark_uninteresting(const struct object_id *oid,
2848
+ struct packed_git *pack,
2849
+ uint32_t pos,
2850
+ void *unused)
2851
+{
2852
+ struct object *o = parse_object(oid);
2853
+ o->flags |= UNINTERESTING | SEEN;
2854
+ return 0;
2855
+}
2856
+
2857
int prepare_revision_walk(struct rev_info *revs)
2858
{
2859
int i;
@@ -2858,6 +2882,11 @@ int prepare_revision_walk(struct rev_info *revs)
2882
(revs->limited && limiting_can_increase_treesame(revs)))
2883
revs->treesame.name = "treesame";
2884
2885
+ if (revs->exclude_promisor_objects) {
2886
+ for_each_packed_object(mark_uninteresting, NULL,
2887
+ FOR_EACH_OBJECT_PROMISOR_ONLY);
2888
+ }
2889
+
2890
if (revs->no_walk != REVISION_WALK_NO_WALK_UNSORTED)
2891
commit_list_sort_by_date(&revs->commits);
2892
if (revs->no_walk)
revision.h
+4
-1
@@ -121,7 +121,10 @@ struct rev_info {
121
bisect:1,
122
ancestry_path:1,
123
first_parent_only:1,
124
- line_level_traverse:1;
124
+ line_level_traverse:1,
125
+
126
+ /* for internal use only */
127
+ exclude_promisor_objects:1;
128
129
/* Diff flags */
130
unsigned int diff:1,
t/t0410-partial-clone.sh
+101
@@ -160,6 +160,107 @@ test_expect_success 'fetching of missing objects' '
160
git verify-pack --verbose "$IDX" | grep "$HASH"
161
'
162
163
+test_expect_success 'rev-list stops traversal at missing and promised commit' '
164
+ rm -rf repo &&
165
+ test_create_repo repo &&
166
+ test_commit -C repo foo &&
167
+ test_commit -C repo bar &&
168
+
169
+ FOO=$(git -C repo rev-parse foo) &&
170
+ promise_and_delete "$FOO" &&
171
+
172
+ git -C repo config core.repositoryformatversion 1 &&
173
+ git -C repo config extensions.partialclone "arbitrary string" &&
174
+ git -C repo rev-list --exclude-promisor-objects --objects bar >out &&
175
+ grep $(git -C repo rev-parse bar) out &&
176
+ ! grep $FOO out
177
+'
178
+
179
+test_expect_success 'rev-list stops traversal at missing and promised tree' '
180
+ rm -rf repo &&
181
+ test_create_repo repo &&
182
+ test_commit -C repo foo &&
183
+ mkdir repo/a_dir &&
184
+ echo something >repo/a_dir/something &&
185
+ git -C repo add a_dir/something &&
186
+ git -C repo commit -m bar &&
187
+
188
+ # foo^{tree} (tree referenced from commit)
189
+ TREE=$(git -C repo rev-parse foo^{tree}) &&
190
+
191
+ # a tree referenced by HEAD^{tree} (tree referenced from tree)
192
+ TREE2=$(git -C repo ls-tree HEAD^{tree} | grep " tree " | head -1 | cut -b13-52) &&
193
+
194
+ promise_and_delete "$TREE" &&
195
+ promise_and_delete "$TREE2" &&
196
+
197
+ git -C repo config core.repositoryformatversion 1 &&
198
+ git -C repo config extensions.partialclone "arbitrary string" &&
199
+ git -C repo rev-list --exclude-promisor-objects --objects HEAD >out &&
200
+ grep $(git -C repo rev-parse foo) out &&
201
+ ! grep $TREE out &&
202
+ grep $(git -C repo rev-parse HEAD) out &&
203
+ ! grep $TREE2 out
204
+'
205
+
206
+test_expect_success 'rev-list stops traversal at missing and promised blob' '
207
+ rm -rf repo &&
208
+ test_create_repo repo &&
209
+ echo something >repo/something &&
210
+ git -C repo add something &&
211
+ git -C repo commit -m foo &&
212
+
213
+ BLOB=$(git -C repo hash-object -w something) &&
214
+ promise_and_delete "$BLOB" &&
215
+
216
+ git -C repo config core.repositoryformatversion 1 &&
217
+ git -C repo config extensions.partialclone "arbitrary string" &&
218
+ git -C repo rev-list --exclude-promisor-objects --objects HEAD >out &&
219
+ grep $(git -C repo rev-parse HEAD) out &&
220
+ ! grep $BLOB out
221
+'
222
+
223
+test_expect_success 'rev-list stops traversal at promisor commit, tree, and blob' '
224
+ rm -rf repo &&
225
+ test_create_repo repo &&
226
+ test_commit -C repo foo &&
227
+ test_commit -C repo bar &&
228
+ test_commit -C repo baz &&
229
+
230
+ COMMIT=$(git -C repo rev-parse foo) &&
231
+ TREE=$(git -C repo rev-parse bar^{tree}) &&
232
+ BLOB=$(git hash-object repo/baz.t) &&
233
+ printf "%s\n%s\n%s\n" $COMMIT $TREE $BLOB | pack_as_from_promisor &&
234
+
235
+ git -C repo config core.repositoryformatversion 1 &&
236
+ git -C repo config extensions.partialclone "arbitrary string" &&
237
+ git -C repo rev-list --exclude-promisor-objects --objects HEAD >out &&
238
+ ! grep $COMMIT out &&
239
+ ! grep $TREE out &&
240
+ ! grep $BLOB out &&
241
+ grep $(git -C repo rev-parse bar) out # sanity check that some walking was done
242
+'
243
+
244
+test_expect_success 'rev-list accepts missing and promised objects on command line' '
245
+ rm -rf repo &&
246
+ test_create_repo repo &&
247
+ test_commit -C repo foo &&
248
+ test_commit -C repo bar &&
249
+ test_commit -C repo baz &&
250
+
251
+ COMMIT=$(git -C repo rev-parse foo) &&
252
+ TREE=$(git -C repo rev-parse bar^{tree}) &&
253
+ BLOB=$(git hash-object repo/baz.t) &&
254
+
255
+ promise_and_delete $COMMIT &&
256
+ promise_and_delete $TREE &&
257
+ promise_and_delete $BLOB &&
258
+
259
+ git -C repo config core.repositoryformatversion 1 &&
260
+ git -C repo config extensions.partialclone "arbitrary string" &&
261
+ git -C repo rev-list --exclude-promisor-objects --objects "$COMMIT" "$TREE" "$BLOB"
262
+'
263
+
264
LIB_HTTPD_PORT=12345 # default port, 410, cannot be used as non-root
265
. "$TEST_DIRECTORY"/lib-httpd.sh
266
start_httpd