fsck: introduce partialclone extension
Currently, Git does not support repos with very large numbers of objects or repos that wish to minimize manipulation of certain blobs (for example, because they are very large) very well, even if the user operates mostly on part of the repo, because Git is designed on the assumption that every referenced object is available somewhere in the repo storage. In such an arrangement, the full set of objects is usually available in remote storage, ready to be lazily downloaded. Teach fsck about the new state of affairs. In this commit, teach fsck that missing promisor objects referenced from the reflog are not an error case; in future commits, fsck will be taught about other cases. Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jonathan Tan committed
Dec 5, 2017 at 16:58 UTC
498f1f61f123fd66eccc05c1d19356b25b4225b2
5 files changed
+171
-5
builtin/fsck.c
+1
-1
@@ -398,7 +398,7 @@ static void fsck_handle_reflog_oid(const char *refname, struct object_id *oid,
398
xstrfmt("%s@{%"PRItime"}", refname, timestamp));
399
obj->flags |= USED;
400
mark_object_reachable(obj);
401
- } else {
401
+ } else if (!is_promisor_object(oid)) {
402
error("%s: invalid reflog entry %s", refname, oid_to_hex(oid));
403
errors_found |= ERROR_REACHABLE;
404
}
cache.h
+2
-1
@@ -1587,7 +1587,8 @@ extern struct packed_git {
1587
unsigned pack_local:1,
1588
pack_keep:1,
1589
freshened:1,
1590
- do_not_close:1;
1590
+ do_not_close:1,
1591
+ pack_promisor:1;
1592
unsigned char sha1[20];
1593
struct revindex_entry *revindex;
1594
/* something like ".git/objects/pack/xxxxx.pack" */
packfile.c
+74
-3
@@ -8,6 +8,11 @@
8
#include "list.h"
9
#include "streaming.h"
10
#include "sha1-lookup.h"
11
+#include "commit.h"
12
+#include "object.h"
13
+#include "tag.h"
14
+#include "tree-walk.h"
15
+#include "tree.h"
16
17
char *odb_pack_name(struct strbuf *buf,
18
const unsigned char *sha1,
@@ -643,10 +648,10 @@ struct packed_git *add_packed_git(const char *path, size_t path_len, int local)
648
return NULL;
649
650
/*
646
- * ".pack" is long enough to hold any suffix we're adding (and
651
+ * ".promisor" is long enough to hold any suffix we're adding (and
652
* the use xsnprintf double-checks that)
653
*/
649
- alloc = st_add3(path_len, strlen(".pack"), 1);
654
+ alloc = st_add3(path_len, strlen(".promisor"), 1);
655
p = alloc_packed_git(alloc);
656
memcpy(p->pack_name, path, path_len);
657
@@ -654,6 +659,10 @@ struct packed_git *add_packed_git(const char *path, size_t path_len, int local)
659
if (!access(p->pack_name, F_OK))
660
p->pack_keep = 1;
661
662
+ xsnprintf(p->pack_name + path_len, alloc - path_len, ".promisor");
663
+ if (!access(p->pack_name, F_OK))
664
+ p->pack_promisor = 1;
665
+
666
xsnprintf(p->pack_name + path_len, alloc - path_len, ".pack");
667
if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
668
free(p);
@@ -781,7 +790,8 @@ static void prepare_packed_git_one(char *objdir, int local)
790
if (ends_with(de->d_name, ".idx") ||
791
ends_with(de->d_name, ".pack") ||
792
ends_with(de->d_name, ".bitmap") ||
784
- ends_with(de->d_name, ".keep"))
793
+ ends_with(de->d_name, ".keep") ||
794
+ ends_with(de->d_name, ".promisor"))
795
string_list_append(&garbage, path.buf);
796
else
797
report_garbage(PACKDIR_FILE_GARBAGE, path.buf);
@@ -1889,6 +1899,9 @@ int for_each_packed_object(each_packed_object_fn cb, void *data, unsigned flags)
1899
for (p = packed_git; p; p = p->next) {
1900
if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local)
1901
continue;
1902
+ if ((flags & FOR_EACH_OBJECT_PROMISOR_ONLY) &&
1903
+ !p->pack_promisor)
1904
+ continue;
1905
if (open_pack_index(p)) {
1906
pack_errors = 1;
1907
continue;
@@ -1899,3 +1912,61 @@ int for_each_packed_object(each_packed_object_fn cb, void *data, unsigned flags)
1912
}
1913
return r ? r : pack_errors;
1914
}
1915
+
1916
+static int add_promisor_object(const struct object_id *oid,
1917
+ struct packed_git *pack,
1918
+ uint32_t pos,
1919
+ void *set_)
1920
+{
1921
+ struct oidset *set = set_;
1922
+ struct object *obj = parse_object(oid);
1923
+ if (!obj)
1924
+ return 1;
1925
+
1926
+ oidset_insert(set, oid);
1927
+
1928
+ /*
1929
+ * If this is a tree, commit, or tag, the objects it refers
1930
+ * to are also promisor objects. (Blobs refer to no objects.)
1931
+ */
1932
+ if (obj->type == OBJ_TREE) {
1933
+ struct tree *tree = (struct tree *)obj;
1934
+ struct tree_desc desc;
1935
+ struct name_entry entry;
1936
+ if (init_tree_desc_gently(&desc, tree->buffer, tree->size))
1937
+ /*
1938
+ * Error messages are given when packs are
1939
+ * verified, so do not print any here.
1940
+ */
1941
+ return 0;
1942
+ while (tree_entry_gently(&desc, &entry))
1943
+ oidset_insert(set, entry.oid);
1944
+ } else if (obj->type == OBJ_COMMIT) {
1945
+ struct commit *commit = (struct commit *) obj;
1946
+ struct commit_list *parents = commit->parents;
1947
+
1948
+ oidset_insert(set, &commit->tree->object.oid);
1949
+ for (; parents; parents = parents->next)
1950
+ oidset_insert(set, &parents->item->object.oid);
1951
+ } else if (obj->type == OBJ_TAG) {
1952
+ struct tag *tag = (struct tag *) obj;
1953
+ oidset_insert(set, &tag->tagged->oid);
1954
+ }
1955
+ return 0;
1956
+}
1957
+
1958
+int is_promisor_object(const struct object_id *oid)
1959
+{
1960
+ static struct oidset promisor_objects;
1961
+ static int promisor_objects_prepared;
1962
+
1963
+ if (!promisor_objects_prepared) {
1964
+ if (repository_format_partial_clone) {
1965
+ for_each_packed_object(add_promisor_object,
1966
+ &promisor_objects,
1967
+ FOR_EACH_OBJECT_PROMISOR_ONLY);
1968
+ }
1969
+ promisor_objects_prepared = 1;
1970
+ }
1971
+ return oidset_contains(&promisor_objects, oid);
1972
+}
packfile.h
+13
@@ -1,6 +1,8 @@
1
#ifndef PACKFILE_H
2
#define PACKFILE_H
3
4
+#include "oidset.h"
5
+
6
/*
7
* Generate the filename to be used for a pack file with checksum "sha1" and
8
* extension "ext". The result is written into the strbuf "buf", overwriting
@@ -124,6 +126,11 @@ extern int has_sha1_pack(const unsigned char *sha1);
126
127
extern int has_pack_index(const unsigned char *sha1);
128
129
+/*
130
+ * Only iterate over packs obtained from the promisor remote.
131
+ */
132
+#define FOR_EACH_OBJECT_PROMISOR_ONLY 2
133
+
134
/*
135
* Iterate over packed objects in both the local
136
* repository and any alternates repositories (unless the
@@ -135,4 +142,10 @@ typedef int each_packed_object_fn(const struct object_id *oid,
142
void *data);
143
extern int for_each_packed_object(each_packed_object_fn, void *, unsigned flags);
144
145
+/*
146
+ * Return 1 if an object in a promisor packfile is or refers to the given
147
+ * object, 0 otherwise.
148
+ */
149
+extern int is_promisor_object(const struct object_id *oid);
150
+
151
#endif
t/t0410-partial-clone.sh
new
+81
@@ -0,0 +1,81 @@
1
+#!/bin/sh
2
+
3
+test_description='partial clone'
4
+
5
+. ./test-lib.sh
6
+
7
+delete_object () {
8
+ rm $1/.git/objects/$(echo $2 | sed -e 's|^..|&/|')
9
+}
10
+
11
+pack_as_from_promisor () {
12
+ HASH=$(git -C repo pack-objects .git/objects/pack/pack) &&
13
+ >repo/.git/objects/pack/pack-$HASH.promisor
14
+}
15
+
16
+test_expect_success 'missing reflog object, but promised by a commit, passes fsck' '
17
+ test_create_repo repo &&
18
+ test_commit -C repo my_commit &&
19
+
20
+ A=$(git -C repo commit-tree -m a HEAD^{tree}) &&
21
+ C=$(git -C repo commit-tree -m c -p $A HEAD^{tree}) &&
22
+
23
+ # Reference $A only from reflog, and delete it
24
+ git -C repo branch my_branch "$A" &&
25
+ git -C repo branch -f my_branch my_commit &&
26
+ delete_object repo "$A" &&
27
+
28
+ # State that we got $C, which refers to $A, from promisor
29
+ printf "$C\n" | pack_as_from_promisor &&
30
+
31
+ # Normally, it fails
32
+ test_must_fail git -C repo fsck &&
33
+
34
+ # But with the extension, it succeeds
35
+ git -C repo config core.repositoryformatversion 1 &&
36
+ git -C repo config extensions.partialclone "arbitrary string" &&
37
+ git -C repo fsck
38
+'
39
+
40
+test_expect_success 'missing reflog object, but promised by a tag, passes fsck' '
41
+ rm -rf repo &&
42
+ test_create_repo repo &&
43
+ test_commit -C repo my_commit &&
44
+
45
+ A=$(git -C repo commit-tree -m a HEAD^{tree}) &&
46
+ git -C repo tag -a -m d my_tag_name $A &&
47
+ T=$(git -C repo rev-parse my_tag_name) &&
48
+ git -C repo tag -d my_tag_name &&
49
+
50
+ # Reference $A only from reflog, and delete it
51
+ git -C repo branch my_branch "$A" &&
52
+ git -C repo branch -f my_branch my_commit &&
53
+ delete_object repo "$A" &&
54
+
55
+ # State that we got $T, which refers to $A, from promisor
56
+ printf "$T\n" | pack_as_from_promisor &&
57
+
58
+ git -C repo config core.repositoryformatversion 1 &&
59
+ git -C repo config extensions.partialclone "arbitrary string" &&
60
+ git -C repo fsck
61
+'
62
+
63
+test_expect_success 'missing reflog object alone fails fsck, even with extension set' '
64
+ rm -rf repo &&
65
+ test_create_repo repo &&
66
+ test_commit -C repo my_commit &&
67
+
68
+ A=$(git -C repo commit-tree -m a HEAD^{tree}) &&
69
+ B=$(git -C repo commit-tree -m b HEAD^{tree}) &&
70
+
71
+ # Reference $A only from reflog, and delete it
72
+ git -C repo branch my_branch "$A" &&
73
+ git -C repo branch -f my_branch my_commit &&
74
+ delete_object repo "$A" &&
75
+
76
+ git -C repo config core.repositoryformatversion 1 &&
77
+ git -C repo config extensions.partialclone "arbitrary string" &&
78
+ test_must_fail git -C repo fsck
79
+'
80
+
81
+test_done