clone: support remote shallow repository
Cloning from a shallow repository does not follow the "8 steps for new .git/shallow" because if it does we need to get through step 6 for all refs. That means commit walking down to the bottom. Instead the rule to create .git/shallow is simpler and, more importantly, cheap: if a shallow commit is found in the pack, it's probably used (i.e. reachable from some refs), so we add it. Others are dropped. One may notice this method seems flawed by the word "probably". A shallow commit may not be reachable from any refs at all if it's attached to an object island (a group of objects that are not reachable by any refs). If that object island is not complete, a new fetch request may send more objects to connect it to some ref. At that time, because we incorrectly installed the shallow commit in this island, the user will not see anything after that commit (fsck is still ok). This is not desired. Given that object islands are rare (C Git never sends such islands for security reasons) and do not really harm the repository integrity, a tradeoff is made to surprise the user occasionally but work faster everyday. A new option --strict could be added later that follows exactly the 8 steps. "git prune" can also learn to remove dangling objects _and_ the shallow commits that are attached to them from .git/shallow. Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Nguyễn Thái Ngọc Duy committed
Dec 5, 2013 at 20:02 UTC
beea4152d94cf7c77eeb6b226805b315d22b3a2f
6 files changed
+71
-7
builtin/clone.c
+1
@@ -889,6 +889,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
889
890
remote = remote_get(option_origin);
891
transport = transport_get(remote, remote->url[0]);
892
+ transport->cloning = 1;
893
894
if (!transport->get_refs_list || (!is_local && !transport->fetch))
895
die(_("Don't know how to clone %s"), transport->url);
builtin/fetch-pack.c
+1
-1
@@ -153,7 +153,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
153
get_remote_heads(fd[0], NULL, 0, &ref, 0, NULL, NULL);
154
155
ref = fetch_pack(&args, fd, conn, ref, dest,
156
- sought, nr_sought, pack_lockfile_ptr);
156
+ sought, nr_sought, NULL, pack_lockfile_ptr);
157
if (pack_lockfile) {
158
printf("lock %s\n", pack_lockfile);
159
fflush(stdout);
fetch-pack.c
+51
-3
@@ -13,6 +13,7 @@
13
#include "transport.h"
14
#include "version.h"
15
#include "prio-queue.h"
16
+#include "sha1-array.h"
17
18
static int transfer_unpack_limit = -1;
19
static int fetch_unpack_limit = -1;
@@ -774,6 +775,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
775
int fd[2],
776
const struct ref *orig_ref,
777
struct ref **sought, int nr_sought,
778
+ struct shallow_info *si,
779
char **pack_lockfile)
780
{
781
struct ref *ref = copy_ref_list(orig_ref);
@@ -852,6 +854,8 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
854
if (args->depth > 0)
855
setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
856
NULL);
857
+ else if (args->cloning && si->shallow && si->shallow->nr)
858
+ alternate_shallow_file = setup_temporary_shallow(si->shallow);
859
else
860
alternate_shallow_file = NULL;
861
if (get_pack(args, fd, pack_lockfile))
@@ -925,8 +929,11 @@ static int remove_duplicates_in_refs(struct ref **ref, int nr)
929
return dst;
930
}
931
928
-static void update_shallow(struct fetch_pack_args *args)
932
+static void update_shallow(struct fetch_pack_args *args,
933
+ struct shallow_info *si)
934
{
935
+ int i;
936
+
937
if (args->depth > 0 && alternate_shallow_file) {
938
if (*alternate_shallow_file == '\0') { /* --unshallow */
939
unlink_or_warn(git_path("shallow"));
@@ -935,6 +942,42 @@ static void update_shallow(struct fetch_pack_args *args)
942
commit_lock_file(&shallow_lock);
943
return;
944
}
945
+
946
+ if (!si->shallow || !si->shallow->nr)
947
+ return;
948
+
949
+ if (alternate_shallow_file) {
950
+ /*
951
+ * The temporary shallow file is only useful for
952
+ * index-pack and unpack-objects because it may
953
+ * contain more roots than we want. Delete it.
954
+ */
955
+ if (*alternate_shallow_file)
956
+ unlink(alternate_shallow_file);
957
+ free((char *)alternate_shallow_file);
958
+ }
959
+
960
+ if (args->cloning) {
961
+ /*
962
+ * remote is shallow, but this is a clone, there are
963
+ * no objects in repo to worry about. Accept any
964
+ * shallow points that exist in the pack (iow in repo
965
+ * after get_pack() and reprepare_packed_git())
966
+ */
967
+ struct sha1_array extra = SHA1_ARRAY_INIT;
968
+ unsigned char (*sha1)[20] = si->shallow->sha1;
969
+ for (i = 0; i < si->shallow->nr; i++)
970
+ if (has_sha1_file(sha1[i]))
971
+ sha1_array_append(&extra, sha1[i]);
972
+ if (extra.nr) {
973
+ setup_alternate_shallow(&shallow_lock,
974
+ &alternate_shallow_file,
975
+ &extra);
976
+ commit_lock_file(&shallow_lock);
977
+ }
978
+ sha1_array_clear(&extra);
979
+ return;
980
+ }
981
}
982
983
struct ref *fetch_pack(struct fetch_pack_args *args,
@@ -942,9 +985,11 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
985
const struct ref *ref,
986
const char *dest,
987
struct ref **sought, int nr_sought,
988
+ struct sha1_array *shallow,
989
char **pack_lockfile)
990
{
991
struct ref *ref_cpy;
992
+ struct shallow_info si;
993
994
fetch_pack_setup();
995
if (nr_sought)
@@ -954,8 +999,11 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
999
packet_flush(fd[1]);
1000
die("no matching remote head");
1001
}
957
- ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, pack_lockfile);
958
- update_shallow(args);
1002
+ prepare_shallow_info(&si, shallow);
1003
+ ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1004
+ &si, pack_lockfile);
1005
reprepare_packed_git();
1006
+ update_shallow(args, &si);
1007
+ clear_shallow_info(&si);
1008
return ref_cpy;
1009
}
fetch-pack.h
+4
@@ -4,6 +4,8 @@
4
#include "string-list.h"
5
#include "run-command.h"
6
7
+struct sha1_array;
8
+
9
struct fetch_pack_args {
10
const char *uploadpack;
11
int unpacklimit;
@@ -20,6 +22,7 @@ struct fetch_pack_args {
22
unsigned stateless_rpc:1;
23
unsigned check_self_contained_and_connected:1;
24
unsigned self_contained_and_connected:1;
25
+ unsigned cloning:1;
26
};
27
28
/*
@@ -33,6 +36,7 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
36
const char *dest,
37
struct ref **sought,
38
int nr_sought,
39
+ struct sha1_array *shallow,
40
char **pack_lockfile);
41
42
#endif
transport.c
+8
-3
@@ -456,6 +456,7 @@ struct git_transport_data {
456
int fd[2];
457
unsigned got_remote_heads : 1;
458
struct sha1_array extra_have;
459
+ struct sha1_array shallow;
460
};
461
462
static int set_git_option(struct git_transport_options *opts,
@@ -512,7 +513,9 @@ static struct ref *get_refs_via_connect(struct transport *transport, int for_pus
513
514
connect_setup(transport, for_push, 0);
515
get_remote_heads(data->fd[0], NULL, 0, &refs,
515
- for_push ? REF_NORMAL : 0, &data->extra_have, NULL);
516
+ for_push ? REF_NORMAL : 0,
517
+ &data->extra_have,
518
+ transport->cloning ? &data->shallow : NULL);
519
data->got_remote_heads = 1;
520
521
return refs;
@@ -539,17 +542,19 @@ static int fetch_refs_via_pack(struct transport *transport,
542
args.depth = data->options.depth;
543
args.check_self_contained_and_connected =
544
data->options.check_self_contained_and_connected;
545
+ args.cloning = transport->cloning;
546
547
if (!data->got_remote_heads) {
548
connect_setup(transport, 0, 0);
549
get_remote_heads(data->fd[0], NULL, 0, &refs_tmp, 0,
546
- NULL, NULL);
550
+ NULL,
551
+ transport->cloning ? &data->shallow : NULL);
552
data->got_remote_heads = 1;
553
}
554
555
refs = fetch_pack(&args, data->fd, data->conn,
556
refs_tmp ? refs_tmp : transport->remote_refs,
552
- dest, to_fetch, nr_heads,
557
+ dest, to_fetch, nr_heads, &data->shallow,
558
&transport->pack_lockfile);
559
close(data->fd[0]);
560
close(data->fd[1]);
transport.h
+6
@@ -35,6 +35,12 @@ struct transport {
35
*/
36
unsigned cannot_reuse : 1;
37
38
+ /*
39
+ * A hint from caller that it will be performing a clone, not
40
+ * normal fetch. IOW the repository is guaranteed empty.
41
+ */
42
+ unsigned cloning : 1;
43
+
44
/**
45
* Returns 0 if successful, positive if the option is not
46
* recognized or is inapplicable, and negative if the option