introduce fetch-object: fetch one promisor object
Introduce fetch-object, providing the ability to fetch one object from a promisor remote. This uses fetch-pack. To do this, the transport mechanism has been updated with 2 flags, "from-promisor" to indicate that the resulting pack comes from a promisor remote (and thus should be annotated as such by index-pack), and "no-dependents" to indicate that only the objects themselves need to be fetched (but fetching additional objects is nevertheless safe). Whenever "no-dependents" is used, fetch-pack will refrain from using any object flags, because it is most likely invoked as part of a dynamic object fetch by another Git command (which may itself use object flags). An alternative to this is to leave fetch-pack alone, and instead update the allocation of flags so that fetch-pack's flags never overlap with any others, but this will end up shrinking the number of flags available to nearly every other Git command (that is, every Git command that accesses objects), so the approach in this commit was used instead. This will be tested in a subsequent commit. Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jonathan Tan committed
Dec 5, 2017 at 16:58 UTC
88e2f9ed8efaf069bea65ab6920bcdcd7c8a4da1
11 files changed
+126
-25
Documentation/gitremote-helpers.txt
+7
@@ -466,6 +466,13 @@ set by Git if the remote helper has the 'option' capability.
466
Transmit <string> as a push option. As the push option
467
must not contain LF or NUL characters, the string is not encoded.
468
469
+'option from-promisor' {'true'|'false'}::
470
+ Indicate that these objects are being fetched from a promisor.
471
+
472
+'option no-dependents' {'true'|'false'}::
473
+ Indicate that only the objects wanted need to be fetched, not
474
+ their dependents.
475
+
476
SEE ALSO
477
--------
478
linkgit:git-remote[1]
Makefile
+1
@@ -792,6 +792,7 @@ LIB_OBJS += ewah/ewah_bitmap.o
792
LIB_OBJS += ewah/ewah_io.o
793
LIB_OBJS += ewah/ewah_rlw.o
794
LIB_OBJS += exec_cmd.o
795
+LIB_OBJS += fetch-object.o
796
LIB_OBJS += fetch-pack.o
797
LIB_OBJS += fsck.o
798
LIB_OBJS += gettext.o
builtin/fetch-pack.c
+8
@@ -143,6 +143,14 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
143
args.update_shallow = 1;
144
continue;
145
}
146
+ if (!strcmp("--from-promisor", arg)) {
147
+ args.from_promisor = 1;
148
+ continue;
149
+ }
150
+ if (!strcmp("--no-dependents", arg)) {
151
+ args.no_dependents = 1;
152
+ continue;
153
+ }
154
usage(fetch_pack_usage);
155
}
156
if (deepen_not.nr)
builtin/index-pack.c
+13
-3
@@ -1429,14 +1429,16 @@ static void write_special_file(const char *suffix, const char *msg,
1429
if (close(fd) != 0)
1430
die_errno(_("cannot close written %s file '%s'"),
1431
suffix, filename);
1432
- *report = suffix;
1432
+ if (report)
1433
+ *report = suffix;
1434
}
1435
strbuf_release(&name_buf);
1436
}
1437
1438
static void final(const char *final_pack_name, const char *curr_pack_name,
1439
const char *final_index_name, const char *curr_index_name,
1439
- const char *keep_msg, unsigned char *sha1)
1440
+ const char *keep_msg, const char *promisor_msg,
1441
+ unsigned char *sha1)
1442
{
1443
const char *report = "pack";
1444
struct strbuf pack_name = STRBUF_INIT;
@@ -1455,6 +1457,9 @@ static void final(const char *final_pack_name, const char *curr_pack_name,
1457
if (keep_msg)
1458
write_special_file("keep", keep_msg, final_pack_name, sha1,
1459
&report);
1460
+ if (promisor_msg)
1461
+ write_special_file("promisor", promisor_msg, final_pack_name,
1462
+ sha1, NULL);
1463
1464
if (final_pack_name != curr_pack_name) {
1465
if (!final_pack_name)
@@ -1644,6 +1649,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
1649
const char *curr_index;
1650
const char *index_name = NULL, *pack_name = NULL;
1651
const char *keep_msg = NULL;
1652
+ const char *promisor_msg = NULL;
1653
struct strbuf index_name_buf = STRBUF_INIT;
1654
struct pack_idx_entry **idx_objects;
1655
struct pack_idx_option opts;
@@ -1693,6 +1699,10 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
1699
keep_msg = "";
1700
} else if (starts_with(arg, "--keep=")) {
1701
keep_msg = arg + 7;
1702
+ } else if (!strcmp(arg, "--promisor")) {
1703
+ promisor_msg = "";
1704
+ } else if (starts_with(arg, "--promisor=")) {
1705
+ promisor_msg = arg + strlen("--promisor=");
1706
} else if (starts_with(arg, "--threads=")) {
1707
char *end;
1708
nr_threads = strtoul(arg+10, &end, 0);
@@ -1803,7 +1813,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
1813
if (!verify)
1814
final(pack_name, curr_pack,
1815
index_name, curr_index,
1806
- keep_msg,
1816
+ keep_msg, promisor_msg,
1817
pack_sha1);
1818
else
1819
close(input_fd);
fetch-object.c
new
+24
@@ -0,0 +1,24 @@
1
+#include "cache.h"
2
+#include "packfile.h"
3
+#include "pkt-line.h"
4
+#include "strbuf.h"
5
+#include "transport.h"
6
+#include "fetch-object.h"
7
+
8
+void fetch_object(const char *remote_name, const unsigned char *sha1)
9
+{
10
+ struct remote *remote;
11
+ struct transport *transport;
12
+ struct ref *ref;
13
+
14
+ remote = remote_get(remote_name);
15
+ if (!remote->url[0])
16
+ die(_("Remote with no URL"));
17
+ transport = transport_get(remote, remote->url[0]);
18
+
19
+ ref = alloc_ref(sha1_to_hex(sha1));
20
+ hashcpy(ref->old_oid.hash, sha1);
21
+ transport_set_option(transport, TRANS_OPT_FROM_PROMISOR, "1");
22
+ transport_set_option(transport, TRANS_OPT_NO_DEPENDENTS, "1");
23
+ transport_fetch_refs(transport, ref);
24
+}
fetch-object.h
new
+6
@@ -0,0 +1,6 @@
1
+#ifndef FETCH_OBJECT_H
2
+#define FETCH_OBJECT_H
3
+
4
+extern void fetch_object(const char *remote_name, const unsigned char *sha1);
5
+
6
+#endif
fetch-pack.c
+27
-21
@@ -450,6 +450,8 @@ static int find_common(struct fetch_pack_args *args,
450
451
flushes = 0;
452
retval = -1;
453
+ if (args->no_dependents)
454
+ goto done;
455
while ((oid = get_rev())) {
456
packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
457
print_verbose(args, "have %s", oid_to_hex(oid));
@@ -734,29 +736,31 @@ static int everything_local(struct fetch_pack_args *args,
736
}
737
}
738
737
- if (!args->deepen) {
738
- for_each_ref(mark_complete_oid, NULL);
739
- for_each_cached_alternate(mark_alternate_complete);
740
- commit_list_sort_by_date(&complete);
741
- if (cutoff)
742
- mark_recent_complete_commits(args, cutoff);
743
- }
739
+ if (!args->no_dependents) {
740
+ if (!args->deepen) {
741
+ for_each_ref(mark_complete_oid, NULL);
742
+ for_each_cached_alternate(mark_alternate_complete);
743
+ commit_list_sort_by_date(&complete);
744
+ if (cutoff)
745
+ mark_recent_complete_commits(args, cutoff);
746
+ }
747
745
- /*
746
- * Mark all complete remote refs as common refs.
747
- * Don't mark them common yet; the server has to be told so first.
748
- */
749
- for (ref = *refs; ref; ref = ref->next) {
750
- struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
751
- NULL, 0);
748
+ /*
749
+ * Mark all complete remote refs as common refs.
750
+ * Don't mark them common yet; the server has to be told so first.
751
+ */
752
+ for (ref = *refs; ref; ref = ref->next) {
753
+ struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
754
+ NULL, 0);
755
753
- if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
754
- continue;
756
+ if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
757
+ continue;
758
756
- if (!(o->flags & SEEN)) {
757
- rev_list_push((struct commit *)o, COMMON_REF | SEEN);
759
+ if (!(o->flags & SEEN)) {
760
+ rev_list_push((struct commit *)o, COMMON_REF | SEEN);
761
759
- mark_common((struct commit *)o, 1, 1);
762
+ mark_common((struct commit *)o, 1, 1);
763
+ }
764
}
765
}
766
@@ -832,7 +836,7 @@ static int get_pack(struct fetch_pack_args *args,
836
argv_array_push(&cmd.args, alternate_shallow_file);
837
}
838
835
- if (do_keep) {
839
+ if (do_keep || args->from_promisor) {
840
if (pack_lockfile)
841
cmd.out = -1;
842
cmd_name = "index-pack";
@@ -842,7 +846,7 @@ static int get_pack(struct fetch_pack_args *args,
846
argv_array_push(&cmd.args, "-v");
847
if (args->use_thin_pack)
848
argv_array_push(&cmd.args, "--fix-thin");
845
- if (args->lock_pack || unpack_limit) {
849
+ if (do_keep && (args->lock_pack || unpack_limit)) {
850
char hostname[HOST_NAME_MAX + 1];
851
if (xgethostname(hostname, sizeof(hostname)))
852
xsnprintf(hostname, sizeof(hostname), "localhost");
@@ -852,6 +856,8 @@ static int get_pack(struct fetch_pack_args *args,
856
}
857
if (args->check_self_contained_and_connected)
858
argv_array_push(&cmd.args, "--check-self-contained-and-connected");
859
+ if (args->from_promisor)
860
+ argv_array_push(&cmd.args, "--promisor");
861
}
862
else {
863
cmd_name = "unpack-objects";
fetch-pack.h
+8
@@ -29,6 +29,14 @@ struct fetch_pack_args {
29
unsigned cloning:1;
30
unsigned update_shallow:1;
31
unsigned deepen:1;
32
+ unsigned from_promisor:1;
33
+
34
+ /*
35
+ * If 1, fetch_pack() will also not modify any object flags.
36
+ * This allows fetch_pack() to safely be called by any function,
37
+ * regardless of which object flags it uses (if any).
38
+ */
39
+ unsigned no_dependents:1;
40
};
41
42
/*
remote-curl.c
+13
-1
@@ -33,7 +33,9 @@ struct options {
33
thin : 1,
34
/* One of the SEND_PACK_PUSH_CERT_* constants. */
35
push_cert : 2,
36
- deepen_relative : 1;
36
+ deepen_relative : 1,
37
+ from_promisor : 1,
38
+ no_dependents : 1;
39
};
40
static struct options options;
41
static struct string_list cas_options = STRING_LIST_INIT_DUP;
@@ -157,6 +159,12 @@ static int set_option(const char *name, const char *value)
159
return -1;
160
return 0;
161
#endif /* LIBCURL_VERSION_NUM >= 0x070a08 */
162
+ } else if (!strcmp(name, "from-promisor")) {
163
+ options.from_promisor = 1;
164
+ return 0;
165
+ } else if (!strcmp(name, "no-dependents")) {
166
+ options.no_dependents = 1;
167
+ return 0;
168
} else {
169
return 1 /* unsupported */;
170
}
@@ -822,6 +830,10 @@ static int fetch_git(struct discovery *heads,
830
options.deepen_not.items[i].string);
831
if (options.deepen_relative && options.depth)
832
argv_array_push(&args, "--deepen-relative");
833
+ if (options.from_promisor)
834
+ argv_array_push(&args, "--from-promisor");
835
+ if (options.no_dependents)
836
+ argv_array_push(&args, "--no-dependents");
837
argv_array_push(&args, url.buf);
838
839
for (i = 0; i < nr_heads; i++) {
transport.c
+8
@@ -160,6 +160,12 @@ static int set_git_option(struct git_transport_options *opts,
160
} else if (!strcmp(name, TRANS_OPT_DEEPEN_RELATIVE)) {
161
opts->deepen_relative = !!value;
162
return 0;
163
+ } else if (!strcmp(name, TRANS_OPT_FROM_PROMISOR)) {
164
+ opts->from_promisor = !!value;
165
+ return 0;
166
+ } else if (!strcmp(name, TRANS_OPT_NO_DEPENDENTS)) {
167
+ opts->no_dependents = !!value;
168
+ return 0;
169
}
170
return 1;
171
}
@@ -228,6 +234,8 @@ static int fetch_refs_via_pack(struct transport *transport,
234
data->options.check_self_contained_and_connected;
235
args.cloning = transport->cloning;
236
args.update_shallow = data->options.update_shallow;
237
+ args.from_promisor = data->options.from_promisor;
238
+ args.no_dependents = data->options.no_dependents;
239
240
if (!data->got_remote_heads) {
241
connect_setup(transport, 0);
transport.h
+11
@@ -15,6 +15,8 @@ struct git_transport_options {
15
unsigned self_contained_and_connected : 1;
16
unsigned update_shallow : 1;
17
unsigned deepen_relative : 1;
18
+ unsigned from_promisor : 1;
19
+ unsigned no_dependents : 1;
20
int depth;
21
const char *deepen_since;
22
const struct string_list *deepen_not;
@@ -210,6 +212,15 @@ void transport_check_allowed(const char *type);
212
/* Send push certificates */
213
#define TRANS_OPT_PUSH_CERT "pushcert"
214
215
+/* Indicate that these objects are being fetched by a promisor */
216
+#define TRANS_OPT_FROM_PROMISOR "from-promisor"
217
+
218
+/*
219
+ * Indicate that only the objects wanted need to be fetched, not their
220
+ * dependents
221
+ */
222
+#define TRANS_OPT_NO_DEPENDENTS "no-dependents"
223
+
224
/**
225
* Returns 0 if the option was used, non-zero otherwise. Prints a
226
* message to stderr if the option is not used.