upload-pack: add object filtering for partial clone
Teach upload-pack to negotiate object filtering over the protocol and to send filter parameters to pack-objects. This is intended for partial clone and fetch. The idea to make upload-pack configurable using uploadpack.allowFilter comes from Jonathan Tan's work in [1]. [1] https://public-inbox.org/git/f211093280b422c32cc1b7034130072f35c5ed51.1506714999.git.jonathantanmy@google.com/ Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jeff Hostetler committed
Dec 8, 2017 at 15:58 UTC
10ac85c785afed4163ed931149a87b30c7c5eaf9
4 files changed
+45
-1
Documentation/config.txt
+4
@@ -3268,6 +3268,10 @@ uploadpack.packObjectsHook::
3268
was run. I.e., `upload-pack` will feed input intended for
3269
`pack-objects` to the hook, and expects a completed packfile on
3270
stdout.
3271
+
3272
+uploadpack.allowFilter::
3273
+ If this option is set, `upload-pack` will advertise partial
3274
+ clone and partial fetch object filtering.
3275
+
3276
Note that this configuration variable is ignored if it is seen in the
3277
repository-level config (this is a safety measure against fetching from
Documentation/technical/pack-protocol.txt
+8
@@ -212,6 +212,7 @@ out of what the server said it could do with the first 'want' line.
212
upload-request = want-list
213
*shallow-line
214
*1depth-request
215
+ [filter-request]
216
flush-pkt
217
218
want-list = first-want
@@ -227,6 +228,8 @@ out of what the server said it could do with the first 'want' line.
228
additional-want = PKT-LINE("want" SP obj-id)
229
230
depth = 1*DIGIT
231
+
232
+ filter-request = PKT-LINE("filter" SP filter-spec)
233
----
234
235
Clients MUST send all the obj-ids it wants from the reference
@@ -249,6 +252,11 @@ complete those commits. Commits whose parents are not received as a
252
result are defined as shallow and marked as such in the server. This
253
information is sent back to the client in the next step.
254
255
+The client can optionally request that pack-objects omit various
256
+objects from the packfile using one of several filtering techniques.
257
+These are intended for use with partial clone and partial fetch
258
+operations. See `rev-list` for possible "filter-spec" values.
259
+
260
Once all the 'want's and 'shallow's (and optional 'deepen') are
261
transferred, clients MUST send a flush-pkt, to tell the server side
262
that it is done sending the list.
Documentation/technical/protocol-capabilities.txt
+8
@@ -309,3 +309,11 @@ to accept a signed push certificate, and asks the <nonce> to be
309
included in the push certificate. A send-pack client MUST NOT
310
send a push-cert packet unless the receive-pack server advertises
311
this capability.
312
+
313
+filter
314
+------
315
+
316
+If the upload-pack server advertises the 'filter' capability,
317
+fetch-pack may send "filter" commands to request a partial clone
318
+or partial fetch and request that the server omit various objects
319
+from the packfile.
upload-pack.c
+25
-1
@@ -10,6 +10,8 @@
10
#include "diff.h"
11
#include "revision.h"
12
#include "list-objects.h"
13
+#include "list-objects-filter.h"
14
+#include "list-objects-filter-options.h"
15
#include "run-command.h"
16
#include "connect.h"
17
#include "sigchain.h"
@@ -18,6 +20,7 @@
20
#include "parse-options.h"
21
#include "argv-array.h"
22
#include "prio-queue.h"
23
+#include "quote.h"
24
25
static const char * const upload_pack_usage[] = {
26
N_("git upload-pack [<options>] <dir>"),
@@ -64,6 +67,10 @@ static int advertise_refs;
67
static int stateless_rpc;
68
static const char *pack_objects_hook;
69
70
+static int filter_capability_requested;
71
+static int filter_advertise;
72
+static struct list_objects_filter_options filter_options;
73
+
74
static void reset_timeout(void)
75
{
76
alarm(timeout);
@@ -131,6 +138,12 @@ static void create_pack_file(void)
138
argv_array_push(&pack_objects.args, "--delta-base-offset");
139
if (use_include_tag)
140
argv_array_push(&pack_objects.args, "--include-tag");
141
+ if (filter_options.filter_spec) {
142
+ struct strbuf buf = STRBUF_INIT;
143
+ sq_quote_buf(&buf, filter_options.filter_spec);
144
+ argv_array_pushf(&pack_objects.args, "--filter=%s", buf.buf);
145
+ strbuf_release(&buf);
146
+ }
147
148
pack_objects.in = -1;
149
pack_objects.out = -1;
@@ -794,6 +807,12 @@ static void receive_needs(void)
807
deepen_rev_list = 1;
808
continue;
809
}
810
+ if (skip_prefix(line, "filter ", &arg)) {
811
+ if (!filter_capability_requested)
812
+ die("git upload-pack: filtering capability not negotiated");
813
+ parse_list_objects_filter(&filter_options, arg);
814
+ continue;
815
+ }
816
if (!skip_prefix(line, "want ", &arg) ||
817
get_oid_hex(arg, &oid_buf))
818
die("git upload-pack: protocol error, "
@@ -821,6 +840,8 @@ static void receive_needs(void)
840
no_progress = 1;
841
if (parse_feature_request(features, "include-tag"))
842
use_include_tag = 1;
843
+ if (parse_feature_request(features, "filter"))
844
+ filter_capability_requested = 1;
845
846
o = parse_object(&oid_buf);
847
if (!o) {
@@ -940,7 +961,7 @@ static int send_ref(const char *refname, const struct object_id *oid,
961
struct strbuf symref_info = STRBUF_INIT;
962
963
format_symref_info(&symref_info, cb_data);
943
- packet_write_fmt(1, "%s %s%c%s%s%s%s%s agent=%s\n",
964
+ packet_write_fmt(1, "%s %s%c%s%s%s%s%s%s agent=%s\n",
965
oid_to_hex(oid), refname_nons,
966
0, capabilities,
967
(allow_unadvertised_object_request & ALLOW_TIP_SHA1) ?
@@ -949,6 +970,7 @@ static int send_ref(const char *refname, const struct object_id *oid,
970
" allow-reachable-sha1-in-want" : "",
971
stateless_rpc ? " no-done" : "",
972
symref_info.buf,
973
+ filter_advertise ? " filter" : "",
974
git_user_agent_sanitized());
975
strbuf_release(&symref_info);
976
} else {
@@ -1027,6 +1049,8 @@ static int upload_pack_config(const char *var, const char *value, void *unused)
1049
} else if (current_config_scope() != CONFIG_SCOPE_REPO) {
1050
if (!strcmp("uploadpack.packobjectshook", var))
1051
return git_config_string(&pack_objects_hook, var, value);
1052
+ } else if (!strcmp("uploadpack.allowfilter", var)) {
1053
+ filter_advertise = git_config_bool(var, value);
1054
}
1055
return parse_hide_refs_config(var, value, "uploadpack");
1056
}