fetch, upload-pack: --deepen=N extends shallow boundary by N commits
In git-fetch, --depth argument is always relative with the latest remote refs. This makes it a bit difficult to cover this use case, where the user wants to make the shallow history, say 3 levels deeper. It would work if remote refs have not moved yet, but nobody can guarantee that, especially when that use case is performed a couple months after the last clone or "git fetch --depth". Also, modifying shallow boundary using --depth does not work well with clones created by --since or --not. This patch fixes that. A new argument --deepen=<N> will add <N> more (*) parent commits to the current history regardless of where remote refs are. Have/Want negotiation is still respected. So if remote refs move, the server will send two chunks: one between "have" and "want" and another to extend shallow history. In theory, the client could send no "want"s in order to get the second chunk only. But the protocol does not allow that. Either you send no want lines, which means ls-remote; or you have to send at least one want line that carries deep-relative to the server.. The main work was done by Dongcan Jiang. I fixed it up here and there. And of course all the bugs belong to me. (*) We could even support --deepen=<N> where <N> is negative. In that case we can cut some history from the shallow clone. This operation (and --depth=<shorter depth>) does not require interaction with remote side (and more complicated to implement as a result). Helped-by: Duy Nguyen <pclouds@gmail.com> Helped-by: Eric Sunshine <sunshine@sunshineco.com> Helped-by: Junio C Hamano <gitster@pobox.com> Signed-off-by: Dongcan Jiang <dongcan.jiang@gmail.com> Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Nguyễn Thái Ngọc Duy committed
Jun 12, 2016 at 17:54 UTC
cccf74e2da85808478c784e403a69bbfe2b9f518
15 files changed
+132
-6
Documentation/fetch-options.txt
+5
@@ -14,6 +14,11 @@
14
linkgit:git-clone[1]), deepen or shorten the history to the specified
15
number of commits. Tags for the deepened commits are not fetched.
16
17
+--deepen=<depth>::
18
+ Similar to --depth, except it specifies the number of commits
19
+ from the current shallow boundary instead of from the tip of
20
+ each remote branch history.
21
+
22
--shallow-since=<date>::
23
Deepen or shorten the history of a shallow repository to
24
include all reachable commits after <date>.
Documentation/git-fetch-pack.txt
+5
@@ -96,6 +96,11 @@ be in a separate packet, and the list must end with a flush packet.
96
exclude commits reachable from a specified remote branch or tag.
97
This option can be specified multiple times.
98
99
+--deepen-relative::
100
+ Argument --depth specifies the number of commits from the
101
+ current shallow boundary instead of from the tip of each
102
+ remote branch history.
103
+
104
--no-progress::
105
Do not show the progress.
106
Documentation/gitremote-helpers.txt
+4
@@ -422,6 +422,10 @@ set by Git if the remote helper has the 'option' capability.
422
Deepens the history of a shallow repository excluding ref.
423
Multiple options add up.
424
425
+'option deepen-relative {'true'|'false'}::
426
+ Deepens the history of a shallow repository relative to
427
+ current boundary. Only valid when used with "option depth".
428
+
429
'option followtags' {'true'|'false'}::
430
If enabled the helper should automatically fetch annotated
431
tag objects if the object the tag points at was transferred
Documentation/technical/protocol-capabilities.txt
+7
@@ -197,6 +197,13 @@ specific revision, instead of depth. Internally it's equivalent of
197
doing "rev-list --not <rev>" on the server side. "deepen-not"
198
cannot be used with "deepen", but can be used with "deepen-since".
199
200
+deepen-relative
201
+---------------
202
+
203
+If this capability is requested by the client, the semantics of
204
+"deepen" command is changed. The "depth" argument is the depth from
205
+the current shallow boundary, instead of the depth from remote refs.
206
+
207
no-progress
208
-----------
209
builtin/fetch-pack.c
+4
@@ -113,6 +113,10 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
113
string_list_append(&deepen_not, arg);
114
continue;
115
}
116
+ if (!strcmp(arg, "--deepen-relative")) {
117
+ args.deepen_relative = 1;
118
+ continue;
119
+ }
120
if (!strcmp("--no-progress", arg)) {
121
args.no_progress = 1;
122
continue;
builtin/fetch.c
+13
-1
@@ -34,7 +34,7 @@ static int fetch_prune_config = -1; /* unspecified */
34
static int prune = -1; /* unspecified */
35
#define PRUNE_BY_DEFAULT 0 /* do we prune by default? */
36
37
-static int all, append, dry_run, force, keep, multiple, update_head_ok, verbosity;
37
+static int all, append, dry_run, force, keep, multiple, update_head_ok, verbosity, deepen_relative;
38
static int progress = -1, recurse_submodules = RECURSE_SUBMODULES_DEFAULT;
39
static int tags = TAGS_DEFAULT, unshallow, update_shallow, deepen;
40
static int max_children = 1;
@@ -121,6 +121,8 @@ static struct option builtin_fetch_options[] = {
121
N_("deepen history of shallow repository based on time")),
122
OPT_STRING_LIST(0, "shallow-exclude", &deepen_not, N_("revision"),
123
N_("deepen history of shallow clone by excluding rev")),
124
+ OPT_INTEGER(0, "deepen", &deepen_relative,
125
+ N_("deepen history of shallow clone")),
126
{ OPTION_SET_INT, 0, "unshallow", &unshallow, NULL,
127
N_("convert to a complete repository"),
128
PARSE_OPT_NONEG | PARSE_OPT_NOARG, NULL, 1 },
@@ -881,6 +883,8 @@ static struct transport *prepare_transport(struct remote *remote, int deepen)
883
if (deepen && deepen_not.nr)
884
set_option(transport, TRANS_OPT_DEEPEN_NOT,
885
(const char *)&deepen_not);
886
+ if (deepen_relative)
887
+ set_option(transport, TRANS_OPT_DEEPEN_RELATIVE, "yes");
888
if (update_shallow)
889
set_option(transport, TRANS_OPT_UPDATE_SHALLOW, "yes");
890
return transport;
@@ -906,6 +910,7 @@ static void backfill_tags(struct transport *transport, struct ref *ref_map)
910
911
transport_set_option(transport, TRANS_OPT_FOLLOWTAGS, NULL);
912
transport_set_option(transport, TRANS_OPT_DEPTH, "0");
913
+ transport_set_option(transport, TRANS_OPT_DEEPEN_RELATIVE, NULL);
914
fetch_refs(transport, ref_map);
915
916
if (gsecondary) {
@@ -1177,6 +1182,13 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
1182
argc = parse_options(argc, argv, prefix,
1183
builtin_fetch_options, builtin_fetch_usage, 0);
1184
1185
+ if (deepen_relative) {
1186
+ if (deepen_relative < 0)
1187
+ die(_("Negative depth in --deepen is not supported"));
1188
+ if (depth)
1189
+ die(_("--deepen and --depth are mutually exclusive"));
1190
+ depth = xstrfmt("%d", deepen_relative);
1191
+ }
1192
if (unshallow) {
1193
if (depth)
1194
die(_("--depth and --unshallow cannot be used together"));
fetch-pack.c
+3
@@ -324,6 +324,7 @@ static int find_common(struct fetch_pack_args *args,
324
if (no_done) strbuf_addstr(&c, " no-done");
325
if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
326
if (use_sideband == 1) strbuf_addstr(&c, " side-band");
327
+ if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
328
if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
329
if (args->no_progress) strbuf_addstr(&c, " no-progress");
330
if (args->include_tag) strbuf_addstr(&c, " include-tag");
@@ -883,6 +884,8 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
884
deepen_not_ok = 1;
885
else if (args->deepen_not)
886
die(_("Server does not support --shallow-exclude"));
887
+ if (!server_supports("deepen-relative") && args->deepen_relative)
888
+ die(_("Server does not support --deepen"));
889
890
if (everything_local(args, &ref, sought, nr_sought)) {
891
packet_flush(fd[1]);
fetch-pack.h
+1
@@ -12,6 +12,7 @@ struct fetch_pack_args {
12
int depth;
13
const char *deepen_since;
14
const struct string_list *deepen_not;
15
+ unsigned deepen_relative:1;
16
unsigned quiet:1;
17
unsigned keep_pack:1;
18
unsigned lock_pack:1;
remote-curl.c
+13
-1
@@ -30,7 +30,8 @@ struct options {
30
dry_run : 1,
31
thin : 1,
32
/* One of the SEND_PACK_PUSH_CERT_* constants. */
33
- push_cert : 2;
33
+ push_cert : 2,
34
+ deepen_relative : 1;
35
};
36
static struct options options;
37
static struct string_list cas_options = STRING_LIST_INIT_DUP;
@@ -70,6 +71,15 @@ static int set_option(const char *name, const char *value)
71
string_list_append(&options.deepen_not, value);
72
return 0;
73
}
74
+ else if (!strcmp(name, "deepen-relative")) {
75
+ if (!strcmp(value, "true"))
76
+ options.deepen_relative = 1;
77
+ else if (!strcmp(value, "false"))
78
+ options.deepen_relative = 0;
79
+ else
80
+ return -1;
81
+ return 0;
82
+ }
83
else if (!strcmp(name, "followtags")) {
84
if (!strcmp(value, "true"))
85
options.followtags = 1;
@@ -761,6 +771,8 @@ static int fetch_git(struct discovery *heads,
771
for (i = 0; i < options.deepen_not.nr; i++)
772
argv_array_pushf(&args, "--shallow-exclude=%s",
773
options.deepen_not.items[i].string);
774
+ if (options.deepen_relative && options.depth)
775
+ argv_array_push(&args, "--deepen-relative");
776
argv_array_push(&args, url.buf);
777
778
for (i = 0; i < nr_heads; i++) {
t/t5500-fetch-pack.sh
+23
@@ -682,4 +682,27 @@ test_expect_success 'fetch exclude tag one' '
682
test_cmp expected actual
683
'
684
685
+test_expect_success 'fetching deepen' '
686
+ test_create_repo shallow-deepen &&
687
+ (
688
+ cd shallow-deepen &&
689
+ test_commit one &&
690
+ test_commit two &&
691
+ test_commit three &&
692
+ git clone --depth 1 "file://$(pwd)/." deepen &&
693
+ test_commit four &&
694
+ git -C deepen log --pretty=tformat:%s master >actual &&
695
+ echo three >expected &&
696
+ test_cmp expected actual &&
697
+ git -C deepen fetch --deepen=1 &&
698
+ git -C deepen log --pretty=tformat:%s origin/master >actual &&
699
+ cat >expected <<-\EOF &&
700
+ four
701
+ three
702
+ two
703
+ EOF
704
+ test_cmp expected actual
705
+ )
706
+'
707
+
708
test_done
t/t5539-fetch-http-shallow.sh
+26
@@ -120,5 +120,31 @@ test_expect_success 'fetch exclude tag one' '
120
test_cmp expected actual
121
'
122
123
+test_expect_success 'fetching deepen' '
124
+ test_create_repo shallow-deepen &&
125
+ (
126
+ cd shallow-deepen &&
127
+ test_commit one &&
128
+ test_commit two &&
129
+ test_commit three &&
130
+ mv .git "$HTTPD_DOCUMENT_ROOT_PATH/shallow-deepen.git" &&
131
+ git clone --depth 1 $HTTPD_URL/smart/shallow-deepen.git deepen &&
132
+ mv "$HTTPD_DOCUMENT_ROOT_PATH/shallow-deepen.git" .git &&
133
+ test_commit four &&
134
+ git -C deepen log --pretty=tformat:%s master >actual &&
135
+ echo three >expected &&
136
+ test_cmp expected actual &&
137
+ mv .git "$HTTPD_DOCUMENT_ROOT_PATH/shallow-deepen.git" &&
138
+ git -C deepen fetch --deepen=1 &&
139
+ git -C deepen log --pretty=tformat:%s origin/master >actual &&
140
+ cat >expected <<-\EOF &&
141
+ four
142
+ three
143
+ two
144
+ EOF
145
+ test_cmp expected actual
146
+ )
147
+'
148
+
149
stop_httpd
150
test_done
transport-helper.c
+1
@@ -258,6 +258,7 @@ static const char *boolean_options[] = {
258
TRANS_OPT_THIN,
259
TRANS_OPT_KEEP,
260
TRANS_OPT_FOLLOWTAGS,
261
+ TRANS_OPT_DEEPEN_RELATIVE
262
};
263
264
static int strbuf_set_helper_option(struct helper_data *data,
transport.c
+4
@@ -157,6 +157,9 @@ static int set_git_option(struct git_transport_options *opts,
157
} else if (!strcmp(name, TRANS_OPT_DEEPEN_NOT)) {
158
opts->deepen_not = (const struct string_list *)value;
159
return 0;
160
+ } else if (!strcmp(name, TRANS_OPT_DEEPEN_RELATIVE)) {
161
+ opts->deepen_relative = !!value;
162
+ return 0;
163
}
164
return 1;
165
}
@@ -213,6 +216,7 @@ static int fetch_refs_via_pack(struct transport *transport,
216
args.depth = data->options.depth;
217
args.deepen_since = data->options.deepen_since;
218
args.deepen_not = data->options.deepen_not;
219
+ args.deepen_relative = data->options.deepen_relative;
220
args.check_self_contained_and_connected =
221
data->options.check_self_contained_and_connected;
222
args.cloning = transport->cloning;
transport.h
+4
@@ -14,6 +14,7 @@ struct git_transport_options {
14
unsigned check_self_contained_and_connected : 1;
15
unsigned self_contained_and_connected : 1;
16
unsigned update_shallow : 1;
17
+ unsigned deepen_relative : 1;
18
int depth;
19
const char *deepen_since;
20
const struct string_list *deepen_not;
@@ -181,6 +182,9 @@ int transport_restrict_protocols(void);
182
/* Limit the depth of the fetch based on revs if not null */
183
#define TRANS_OPT_DEEPEN_NOT "deepen-not"
184
185
+/* Limit the deepen of the fetch if not null */
186
+#define TRANS_OPT_DEEPEN_RELATIVE "deepen-relative"
187
+
188
/* Aggressively fetch annotated tags if possible */
189
#define TRANS_OPT_FOLLOWTAGS "followtags"
190
upload-pack.c
+19
-4
@@ -32,6 +32,7 @@ static const char upload_pack_usage[] = "git upload-pack [--strict] [--timeout=<
32
33
static unsigned long oldest_have;
34
35
+static int deepen_relative;
36
static int multi_ack;
37
static int no_done;
38
static int use_thin_pack, use_ofs_delta, use_include_tag;
@@ -674,7 +675,8 @@ static void send_unshallow(const struct object_array *shallows)
675
}
676
}
677
677
-static void deepen(int depth, const struct object_array *shallows)
678
+static void deepen(int depth, int deepen_relative,
679
+ struct object_array *shallows)
680
{
681
if (depth == INFINITE_DEPTH && !is_repository_shallow()) {
682
int i;
@@ -683,6 +685,17 @@ static void deepen(int depth, const struct object_array *shallows)
685
struct object *object = shallows->objects[i].item;
686
object->flags |= NOT_SHALLOW;
687
}
688
+ } else if (deepen_relative) {
689
+ struct object_array reachable_shallows = OBJECT_ARRAY_INIT;
690
+ struct commit_list *result;
691
+
692
+ get_reachable_list(shallows, &reachable_shallows);
693
+ result = get_shallow_commits(&reachable_shallows,
694
+ depth + 1,
695
+ SHALLOW, NOT_SHALLOW);
696
+ send_shallow(result);
697
+ free_commit_list(result);
698
+ object_array_clear(&reachable_shallows);
699
} else {
700
struct commit_list *result;
701
@@ -779,6 +792,8 @@ static void receive_needs(void)
792
793
features = arg + 40;
794
795
+ if (parse_feature_request(features, "deepen-relative"))
796
+ deepen_relative = 1;
797
if (parse_feature_request(features, "multi_ack_detailed"))
798
multi_ack = 2;
799
else if (parse_feature_request(features, "multi_ack"))
@@ -828,7 +843,7 @@ static void receive_needs(void)
843
if (depth > 0 && deepen_rev_list)
844
die("git upload-pack: deepen and deepen-since (or deepen-not) cannot be used together");
845
if (depth > 0)
831
- deepen(depth, &shallows);
846
+ deepen(depth, deepen_relative, &shallows);
847
else if (deepen_rev_list) {
848
struct argv_array av = ARGV_ARRAY_INIT;
849
int i;
@@ -899,8 +914,8 @@ static int send_ref(const char *refname, const struct object_id *oid,
914
int flag, void *cb_data)
915
{
916
static const char *capabilities = "multi_ack thin-pack side-band"
902
- " side-band-64k ofs-delta shallow deepen-since deepen-not no-progress"
903
- " include-tag multi_ack_detailed";
917
+ " side-band-64k ofs-delta shallow deepen-since deepen-not"
918
+ " deepen-relative no-progress include-tag multi_ack_detailed";
919
const char *refname_nons = strip_namespace(refname);
920
struct object_id peeled;
921