fetch-pack: support more than one pack lockfile
Whenever a fetch results in a packfile being downloaded, a .keep file is generated, so that the packfile can be preserved (from, say, a running "git repack") until refs are written referring to the contents of the packfile. In a subsequent patch, a successful fetch using protocol v2 may result in more than one .keep file being generated. Therefore, teach fetch_pack() and the transport mechanism to support multiple .keep files. Implementation notes: - builtin/fetch-pack.c normally does not generate .keep files, and thus is unaffected by this or future changes. However, it has an undocumented "--lock-pack" feature, used by remote-curl.c when implementing the "fetch" remote helper command. In keeping with the remote helper protocol, only one "lock" line will ever be written; the rest will result in warnings to stderr. However, in practice, warnings will never be written because the remote-curl.c "fetch" is only used for protocol v0/v1 (which will not generate multiple .keep files). (Protocol v2 uses the "stateless-connect" command, not the "fetch" command.) - connected.c has an optimization in that connectivity checks on a ref need not be done if the target object is in a pack known to be self-contained and connected. If there are multiple packfiles, this optimization can no longer be done. Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jonathan Tan committed
Jun 10, 2020 at 13:57 UTC
9da69a6539e98da1b7ed8832cb54b494961463cb
7 files changed
+46
-35
builtin/fetch-pack.c
+11
-6
@@ -48,8 +48,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
48
struct ref **sought = NULL;
49
int nr_sought = 0, alloc_sought = 0;
50
int fd[2];
51
- char *pack_lockfile = NULL;
52
- char **pack_lockfile_ptr = NULL;
51
+ struct string_list pack_lockfiles = STRING_LIST_INIT_DUP;
52
+ struct string_list *pack_lockfiles_ptr = NULL;
53
struct child_process *conn;
54
struct fetch_pack_args args;
55
struct oid_array shallow = OID_ARRAY_INIT;
@@ -134,7 +134,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
134
}
135
if (!strcmp("--lock-pack", arg)) {
136
args.lock_pack = 1;
137
- pack_lockfile_ptr = &pack_lockfile;
137
+ pack_lockfiles_ptr = &pack_lockfiles;
138
continue;
139
}
140
if (!strcmp("--check-self-contained-and-connected", arg)) {
@@ -235,10 +235,15 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
235
}
236
237
ref = fetch_pack(&args, fd, ref, sought, nr_sought,
238
- &shallow, pack_lockfile_ptr, version);
239
- if (pack_lockfile) {
240
- printf("lock %s\n", pack_lockfile);
238
+ &shallow, pack_lockfiles_ptr, version);
239
+ if (pack_lockfiles.nr) {
240
+ int i;
241
+
242
+ printf("lock %s\n", pack_lockfiles.items[0].string);
243
fflush(stdout);
244
+ for (i = 1; i < pack_lockfiles.nr; i++)
245
+ warning(_("Lockfile created but not reported: %s"),
246
+ pack_lockfiles.items[i].string);
247
}
248
if (args.check_self_contained_and_connected &&
249
args.self_contained_and_connected) {
connected.c
+5
-3
@@ -43,10 +43,12 @@ int check_connected(oid_iterate_fn fn, void *cb_data,
43
44
if (transport && transport->smart_options &&
45
transport->smart_options->self_contained_and_connected &&
46
- transport->pack_lockfile &&
47
- strip_suffix(transport->pack_lockfile, ".keep", &base_len)) {
46
+ transport->pack_lockfiles.nr == 1 &&
47
+ strip_suffix(transport->pack_lockfiles.items[0].string,
48
+ ".keep", &base_len)) {
49
struct strbuf idx_file = STRBUF_INIT;
49
- strbuf_add(&idx_file, transport->pack_lockfile, base_len);
50
+ strbuf_add(&idx_file, transport->pack_lockfiles.items[0].string,
51
+ base_len);
52
strbuf_addstr(&idx_file, ".idx");
53
new_pack = add_packed_git(idx_file.buf, idx_file.len, 1);
54
strbuf_release(&idx_file);
fetch-pack.c
+15
-14
@@ -794,7 +794,7 @@ static void write_promisor_file(const char *keep_name,
794
}
795
796
static int get_pack(struct fetch_pack_args *args,
797
- int xd[2], char **pack_lockfile,
797
+ int xd[2], struct string_list *pack_lockfiles,
798
struct ref **sought, int nr_sought)
799
{
800
struct async demux;
@@ -838,7 +838,7 @@ static int get_pack(struct fetch_pack_args *args,
838
}
839
840
if (do_keep || args->from_promisor) {
841
- if (pack_lockfile)
841
+ if (pack_lockfiles)
842
cmd.out = -1;
843
cmd_name = "index-pack";
844
argv_array_push(&cmd.args, cmd_name);
@@ -863,7 +863,7 @@ static int get_pack(struct fetch_pack_args *args,
863
* information below. If not, we need index-pack to do it for
864
* us.
865
*/
866
- if (!(do_keep && pack_lockfile) && args->from_promisor)
866
+ if (!(do_keep && pack_lockfiles) && args->from_promisor)
867
argv_array_push(&cmd.args, "--promisor");
868
}
869
else {
@@ -899,8 +899,9 @@ static int get_pack(struct fetch_pack_args *args,
899
cmd.git_cmd = 1;
900
if (start_command(&cmd))
901
die(_("fetch-pack: unable to fork off %s"), cmd_name);
902
- if (do_keep && pack_lockfile) {
903
- *pack_lockfile = index_pack_lockfile(cmd.out);
902
+ if (do_keep && pack_lockfiles) {
903
+ string_list_append_nodup(pack_lockfiles,
904
+ index_pack_lockfile(cmd.out));
905
close(cmd.out);
906
}
907
@@ -922,8 +923,8 @@ static int get_pack(struct fetch_pack_args *args,
923
* Now that index-pack has succeeded, write the promisor file using the
924
* obtained .keep filename if necessary
925
*/
925
- if (do_keep && pack_lockfile && args->from_promisor)
926
- write_promisor_file(*pack_lockfile, sought, nr_sought);
926
+ if (do_keep && pack_lockfiles && pack_lockfiles->nr && args->from_promisor)
927
+ write_promisor_file(pack_lockfiles->items[0].string, sought, nr_sought);
928
929
return 0;
930
}
@@ -940,7 +941,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
941
const struct ref *orig_ref,
942
struct ref **sought, int nr_sought,
943
struct shallow_info *si,
943
- char **pack_lockfile)
944
+ struct string_list *pack_lockfiles)
945
{
946
struct repository *r = the_repository;
947
struct ref *ref = copy_ref_list(orig_ref);
@@ -1067,7 +1068,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
1068
alternate_shallow_file = setup_temporary_shallow(si->shallow);
1069
else
1070
alternate_shallow_file = NULL;
1070
- if (get_pack(args, fd, pack_lockfile, sought, nr_sought))
1071
+ if (get_pack(args, fd, pack_lockfiles, sought, nr_sought))
1072
die(_("git fetch-pack: fetch failed."));
1073
1074
all_done:
@@ -1457,7 +1458,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1458
struct ref **sought, int nr_sought,
1459
struct oid_array *shallows,
1460
struct shallow_info *si,
1460
- char **pack_lockfile)
1461
+ struct string_list *pack_lockfiles)
1462
{
1463
struct repository *r = the_repository;
1464
struct ref *ref = copy_ref_list(orig_ref);
@@ -1559,7 +1560,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1560
1561
/* get the pack */
1562
process_section_header(&reader, "packfile", 0);
1562
- if (get_pack(args, fd, pack_lockfile, sought, nr_sought))
1563
+ if (get_pack(args, fd, pack_lockfiles, sought, nr_sought))
1564
die(_("git fetch-pack: fetch failed."));
1565
1566
state = FETCH_DONE;
@@ -1759,7 +1760,7 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
1760
const struct ref *ref,
1761
struct ref **sought, int nr_sought,
1762
struct oid_array *shallow,
1762
- char **pack_lockfile,
1763
+ struct string_list *pack_lockfiles,
1764
enum protocol_version version)
1765
{
1766
struct ref *ref_cpy;
@@ -1794,11 +1795,11 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
1795
memset(&si, 0, sizeof(si));
1796
ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
1797
&shallows_scratch, &si,
1797
- pack_lockfile);
1798
+ pack_lockfiles);
1799
} else {
1800
prepare_shallow_info(&si, shallow);
1801
ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1801
- &si, pack_lockfile);
1802
+ &si, pack_lockfiles);
1803
}
1804
reprepare_packed_git(the_repository);
1805
fetch-pack.h
+1
-1
@@ -83,7 +83,7 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
83
struct ref **sought,
84
int nr_sought,
85
struct oid_array *shallow,
86
- char **pack_lockfile,
86
+ struct string_list *pack_lockfiles,
87
enum protocol_version version);
88
89
/*
transport-helper.c
+3
-2
@@ -410,10 +410,11 @@ static int fetch_with_fetch(struct transport *transport,
410
exit(128);
411
412
if (skip_prefix(buf.buf, "lock ", &name)) {
413
- if (transport->pack_lockfile)
413
+ if (transport->pack_lockfiles.nr)
414
warning(_("%s also locked %s"), data->name, name);
415
else
416
- transport->pack_lockfile = xstrdup(name);
416
+ string_list_append(&transport->pack_lockfiles,
417
+ name);
418
}
419
else if (data->check_connectivity &&
420
data->transport_options.check_self_contained_and_connected &&
transport.c
+8
-6
@@ -374,7 +374,7 @@ static int fetch_refs_via_pack(struct transport *transport,
374
refs = fetch_pack(&args, data->fd,
375
refs_tmp ? refs_tmp : transport->remote_refs,
376
to_fetch, nr_heads, &data->shallow,
377
- &transport->pack_lockfile, data->version);
377
+ &transport->pack_lockfiles, data->version);
378
break;
379
case protocol_v1:
380
case protocol_v0:
@@ -382,7 +382,7 @@ static int fetch_refs_via_pack(struct transport *transport,
382
refs = fetch_pack(&args, data->fd,
383
refs_tmp ? refs_tmp : transport->remote_refs,
384
to_fetch, nr_heads, &data->shallow,
385
- &transport->pack_lockfile, data->version);
385
+ &transport->pack_lockfiles, data->version);
386
break;
387
case protocol_unknown_version:
388
BUG("unknown protocol version");
@@ -929,6 +929,7 @@ struct transport *transport_get(struct remote *remote, const char *url)
929
struct transport *ret = xcalloc(1, sizeof(*ret));
930
931
ret->progress = isatty(2);
932
+ string_list_init(&ret->pack_lockfiles, 1);
933
934
if (!remote)
935
BUG("No remote provided to transport_get()");
@@ -1324,10 +1325,11 @@ int transport_fetch_refs(struct transport *transport, struct ref *refs)
1325
1326
void transport_unlock_pack(struct transport *transport)
1327
{
1327
- if (transport->pack_lockfile) {
1328
- unlink_or_warn(transport->pack_lockfile);
1329
- FREE_AND_NULL(transport->pack_lockfile);
1330
- }
1328
+ int i;
1329
+
1330
+ for (i = 0; i < transport->pack_lockfiles.nr; i++)
1331
+ unlink_or_warn(transport->pack_lockfiles.items[i].string);
1332
+ string_list_clear(&transport->pack_lockfiles, 0);
1333
}
1334
1335
int transport_connect(struct transport *transport, const char *name,
transport.h
+3
-3
@@ -5,8 +5,7 @@
5
#include "run-command.h"
6
#include "remote.h"
7
#include "list-objects-filter-options.h"
8
-
9
-struct string_list;
8
+#include "string-list.h"
9
10
struct git_transport_options {
11
unsigned thin : 1;
@@ -98,7 +97,8 @@ struct transport {
97
*/
98
const struct string_list *server_options;
99
101
- char *pack_lockfile;
100
+ struct string_list pack_lockfiles;
101
+
102
signed verbose : 3;
103
/**
104
* Transports should not set this directly, and should use this