bundle: add new version for use with SHA-256
Currently we detect the hash algorithm in use by the length of the object ID. This is inelegant and prevents us from using a different hash algorithm that is also 256 bits in length. Since we cannot extend the v2 format in a backward-compatible way, let's add a v3 format, which is identical, except for the addition of capabilities, which are prefixed by an at sign. We add "object-format" as the only capability and reject unknown capabilities, since we do not have a network connection and therefore cannot negotiate with the other side. For compatibility, default to the v2 format for SHA-1 and require v3 for SHA-256. In t5510, always use format v3 so we can be sure we produce consistent results across hash algorithms. Since head -n N lists the top N lines instead of the Nth line, let's run our output through sed to normalize it and compare it against a fixed value, which will make sure we get exactly what we're expecting. Signed-off-by: brian m. carlson <sandals@crustytoothpaste.net> Reviewed-by: Eric Sunshine <sunshine@sunshineco.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
brian m. carlson committed
Jul 29, 2020 at 23:14 UTC
c5aecfc8669b9c56fd1ba31356eedc37b7289f9c
7 files changed
+147
-31
Documentation/git-bundle.txt
+8
-1
@@ -9,7 +9,8 @@ git-bundle - Move objects and refs by archive
9
SYNOPSIS
10
--------
11
[verse]
12
-'git bundle' create [-q | --quiet | --progress | --all-progress] [--all-progress-implied] <file> <git-rev-list-args>
12
+'git bundle' create [-q | --quiet | --progress | --all-progress] [--all-progress-implied]
13
+ [--version=<version>] <file> <git-rev-list-args>
14
'git bundle' verify [-q | --quiet] <file>
15
'git bundle' list-heads <file> [<refname>...]
16
'git bundle' unbundle <file> [<refname>...]
@@ -102,6 +103,12 @@ unbundle <file>::
103
is activated. Unlike --all-progress this flag doesn't actually
104
force any progress display by itself.
105
106
+--version=<version>::
107
+ Specify the bundle version. Version 2 is the older format and can only be
108
+ used with SHA-1 repositories; the newer version 3 contains capabilities that
109
+ permit extensions. The default is the oldest supported format, based on the
110
+ hash algorithm in use.
111
+
112
-q::
113
--quiet::
114
This flag makes the command not to report its progress
Documentation/technical/bundle-format.txt
+29
-1
@@ -7,6 +7,8 @@ The Git bundle format is a format that represents both refs and Git objects.
7
We will use ABNF notation to define the Git bundle format. See
8
protocol-common.txt for the details.
9
10
+A v2 bundle looks like this:
11
+
12
----
13
bundle = signature *prerequisite *reference LF pack
14
signature = "# v2 git bundle" LF
@@ -18,9 +20,28 @@ reference = obj-id SP refname LF
20
pack = ... ; packfile
21
----
22
23
+A v3 bundle looks like this:
24
+
25
+----
26
+bundle = signature *capability *prerequisite *reference LF pack
27
+signature = "# v3 git bundle" LF
28
+
29
+capability = "@" key ["=" value] LF
30
+prerequisite = "-" obj-id SP comment LF
31
+comment = *CHAR
32
+reference = obj-id SP refname LF
33
+key = 1*(ALPHA / DIGIT / "-")
34
+value = *(%01-09 / %0b-FF)
35
+
36
+pack = ... ; packfile
37
+----
38
+
39
== Semantics
40
23
-A Git bundle consists of three parts.
41
+A Git bundle consists of several parts.
42
+
43
+* "Capabilities", which are only in the v3 format, indicate functionality that
44
+ the bundle requires to be read properly.
45
46
* "Prerequisites" lists the objects that are NOT included in the bundle and the
47
reader of the bundle MUST already have, in order to use the data in the
@@ -46,3 +67,10 @@ put any string here. The reader of the bundle MUST ignore the comment.
67
Note that the prerequisites does not represent a shallow-clone boundary. The
68
semantics of the prerequisites and the shallow-clone boundaries are different,
69
and the Git bundle v2 format cannot represent a shallow clone repository.
70
+
71
+== Capabilities
72
+
73
+Because there is no opportunity for negotiation, unknown capabilities cause 'git
74
+bundle' to abort. The only known capability is `object-format`, which specifies
75
+the hash algorithm in use, and can take the same values as the
76
+`extensions.objectFormat` configuration value.
builtin/bundle.c
+4
-1
@@ -60,6 +60,7 @@ static int cmd_bundle_create(int argc, const char **argv, const char *prefix) {
60
int all_progress_implied = 0;
61
int progress = isatty(STDERR_FILENO);
62
struct argv_array pack_opts;
63
+ int version = -1;
64
65
struct option options[] = {
66
OPT_SET_INT('q', "quiet", &progress,
@@ -71,6 +72,8 @@ static int cmd_bundle_create(int argc, const char **argv, const char *prefix) {
72
OPT_BOOL(0, "all-progress-implied",
73
&all_progress_implied,
74
N_("similar to --all-progress when progress meter is shown")),
75
+ OPT_INTEGER(0, "version", &version,
76
+ N_("specify bundle format version")),
77
OPT_END()
78
};
79
const char* bundle_file;
@@ -91,7 +94,7 @@ static int cmd_bundle_create(int argc, const char **argv, const char *prefix) {
94
95
if (!startup_info->have_repository)
96
die(_("Need a repository to create a bundle."));
94
- return !!create_bundle(the_repository, bundle_file, argc, argv, &pack_opts);
97
+ return !!create_bundle(the_repository, bundle_file, argc, argv, &pack_opts, version);
98
}
99
100
static int cmd_bundle_verify(int argc, const char **argv, const char *prefix) {
bundle.c
+62
-21
@@ -12,7 +12,16 @@
12
#include "refs.h"
13
#include "argv-array.h"
14
15
-static const char bundle_signature[] = "# v2 git bundle\n";
15
+
16
+static const char v2_bundle_signature[] = "# v2 git bundle\n";
17
+static const char v3_bundle_signature[] = "# v3 git bundle\n";
18
+static struct {
19
+ int version;
20
+ const char *signature;
21
+} bundle_sigs[] = {
22
+ { 2, v2_bundle_signature },
23
+ { 3, v3_bundle_signature },
24
+};
25
26
static void add_to_ref_list(const struct object_id *oid, const char *name,
27
struct ref_list *list)
@@ -23,15 +32,30 @@ static void add_to_ref_list(const struct object_id *oid, const char *name,
32
list->nr++;
33
}
34
26
-static const struct git_hash_algo *detect_hash_algo(struct strbuf *buf)
35
+static int parse_capability(struct bundle_header *header, const char *capability)
36
+{
37
+ const char *arg;
38
+ if (skip_prefix(capability, "object-format=", &arg)) {
39
+ int algo = hash_algo_by_name(arg);
40
+ if (algo == GIT_HASH_UNKNOWN)
41
+ return error(_("unrecognized bundle hash algorithm: %s"), arg);
42
+ header->hash_algo = &hash_algos[algo];
43
+ return 0;
44
+ }
45
+ return error(_("unknown capability '%s'"), capability);
46
+}
47
+
48
+static int parse_bundle_signature(struct bundle_header *header, const char *line)
49
{
28
- size_t len = strcspn(buf->buf, " \n");
29
- int algo;
50
+ int i;
51
31
- algo = hash_algo_by_length(len / 2);
32
- if (algo == GIT_HASH_UNKNOWN)
33
- return NULL;
34
- return &hash_algos[algo];
52
+ for (i = 0; i < ARRAY_SIZE(bundle_sigs); i++) {
53
+ if (!strcmp(line, bundle_sigs[i].signature)) {
54
+ header->version = bundle_sigs[i].version;
55
+ return 0;
56
+ }
57
+ }
58
+ return -1;
59
}
60
61
static int parse_bundle_header(int fd, struct bundle_header *header,
@@ -42,14 +66,16 @@ static int parse_bundle_header(int fd, struct bundle_header *header,
66
67
/* The bundle header begins with the signature */
68
if (strbuf_getwholeline_fd(&buf, fd, '\n') ||
45
- strcmp(buf.buf, bundle_signature)) {
69
+ parse_bundle_signature(header, buf.buf)) {
70
if (report_path)
47
- error(_("'%s' does not look like a v2 bundle file"),
71
+ error(_("'%s' does not look like a v2 or v3 bundle file"),
72
report_path);
73
status = -1;
74
goto abort;
75
}
76
77
+ header->hash_algo = the_hash_algo;
78
+
79
/* The bundle header ends with an empty line */
80
while (!strbuf_getwholeline_fd(&buf, fd, '\n') &&
81
buf.len && buf.buf[0] != '\n') {
@@ -57,19 +83,19 @@ static int parse_bundle_header(int fd, struct bundle_header *header,
83
int is_prereq = 0;
84
const char *p;
85
60
- if (*buf.buf == '-') {
61
- is_prereq = 1;
62
- strbuf_remove(&buf, 0, 1);
63
- }
86
strbuf_rtrim(&buf);
87
66
- if (!header->hash_algo) {
67
- header->hash_algo = detect_hash_algo(&buf);
68
- if (!header->hash_algo) {
69
- error(_("unknown hash algorithm length"));
88
+ if (header->version == 3 && *buf.buf == '@') {
89
+ if (parse_capability(header, buf.buf + 1)) {
90
status = -1;
91
break;
92
}
93
+ continue;
94
+ }
95
+
96
+ if (*buf.buf == '-') {
97
+ is_prereq = 1;
98
+ strbuf_remove(&buf, 0, 1);
99
}
100
101
/*
@@ -449,13 +475,14 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs)
475
}
476
477
int create_bundle(struct repository *r, const char *path,
452
- int argc, const char **argv, struct argv_array *pack_options)
478
+ int argc, const char **argv, struct argv_array *pack_options, int version)
479
{
480
struct lock_file lock = LOCK_INIT;
481
int bundle_fd = -1;
482
int bundle_to_stdout;
483
int ref_count = 0;
484
struct rev_info revs;
485
+ int min_version = the_hash_algo == &hash_algos[GIT_HASH_SHA1] ? 2 : 3;
486
487
bundle_to_stdout = !strcmp(path, "-");
488
if (bundle_to_stdout)
@@ -464,8 +491,22 @@ int create_bundle(struct repository *r, const char *path,
491
bundle_fd = hold_lock_file_for_update(&lock, path,
492
LOCK_DIE_ON_ERROR);
493
467
- /* write signature */
468
- write_or_die(bundle_fd, bundle_signature, strlen(bundle_signature));
494
+ if (version == -1)
495
+ version = min_version;
496
+
497
+ if (version < 2 || version > 3) {
498
+ die(_("unsupported bundle version %d"), version);
499
+ } else if (version < min_version) {
500
+ die(_("cannot write bundle version %d with algorithm %s"), version, the_hash_algo->name);
501
+ } else if (version == 2) {
502
+ write_or_die(bundle_fd, v2_bundle_signature, strlen(v2_bundle_signature));
503
+ } else {
504
+ const char *capability = "@object-format=";
505
+ write_or_die(bundle_fd, v3_bundle_signature, strlen(v3_bundle_signature));
506
+ write_or_die(bundle_fd, capability, strlen(capability));
507
+ write_or_die(bundle_fd, the_hash_algo->name, strlen(the_hash_algo->name));
508
+ write_or_die(bundle_fd, "\n", 1);
509
+ }
510
511
/* init revs to list objects for pack-objects later */
512
save_commit_buffer = 0;
bundle.h
+3
-1
@@ -13,6 +13,7 @@ struct ref_list {
13
};
14
15
struct bundle_header {
16
+ unsigned version;
17
struct ref_list prerequisites;
18
struct ref_list references;
19
const struct git_hash_algo *hash_algo;
@@ -21,7 +22,8 @@ struct bundle_header {
22
int is_bundle(const char *path, int quiet);
23
int read_bundle_header(const char *path, struct bundle_header *header);
24
int create_bundle(struct repository *r, const char *path,
24
- int argc, const char **argv, struct argv_array *pack_options);
25
+ int argc, const char **argv, struct argv_array *pack_options,
26
+ int version);
27
int verify_bundle(struct repository *r, struct bundle_header *header, int verbose);
28
#define BUNDLE_VERBOSE 1
29
int unbundle(struct repository *r, struct bundle_header *header,
t/t5510-fetch.sh
+10
-6
@@ -281,15 +281,19 @@ test_expect_success 'create bundle 1' '
281
cd "$D" &&
282
echo >file updated again by origin &&
283
git commit -a -m "tip" &&
284
- git bundle create bundle1 master^..master
284
+ git bundle create --version=3 bundle1 master^..master
285
'
286
287
test_expect_success 'header of bundle looks right' '
288
- head -n 4 "$D"/bundle1 &&
289
- head -n 1 "$D"/bundle1 | grep "^#" &&
290
- head -n 2 "$D"/bundle1 | grep "^-$OID_REGEX " &&
291
- head -n 3 "$D"/bundle1 | grep "^$OID_REGEX " &&
292
- head -n 4 "$D"/bundle1 | grep "^$"
288
+ cat >expect <<-EOF &&
289
+ # v3 git bundle
290
+ @object-format=$(test_oid algo)
291
+ -OID updated by origin
292
+ OID refs/heads/master
293
+
294
+ EOF
295
+ sed -e "s/$OID_REGEX/OID/g" -e "5q" "$D"/bundle1 >actual &&
296
+ test_cmp expect actual
297
'
298
299
test_expect_success 'create bundle 2' '
t/t5607-clone-bundle.sh
+31
@@ -4,6 +4,10 @@ test_description='some bundle related tests'
4
. ./test-lib.sh
5
6
test_expect_success 'setup' '
7
+ test_oid_cache <<-EOF &&
8
+ version sha1:2
9
+ version sha256:3
10
+ EOF
11
test_commit initial &&
12
test_tick &&
13
git tag -m tag tag &&
@@ -94,4 +98,31 @@ test_expect_success 'fetch SHA-1 from bundle' '
98
git fetch --no-tags foo/tip.bundle "$(cat hash)"
99
'
100
101
+test_expect_success 'git bundle uses expected default format' '
102
+ git bundle create bundle HEAD^.. &&
103
+ head -n1 bundle | grep "^# v$(test_oid version) git bundle$"
104
+'
105
+
106
+test_expect_success 'git bundle v3 has expected contents' '
107
+ git branch side HEAD &&
108
+ git bundle create --version=3 bundle HEAD^..side &&
109
+ head -n2 bundle >actual &&
110
+ cat >expect <<-EOF &&
111
+ # v3 git bundle
112
+ @object-format=$(test_oid algo)
113
+ EOF
114
+ test_cmp expect actual &&
115
+ git bundle verify bundle
116
+'
117
+
118
+test_expect_success 'git bundle v3 rejects unknown capabilities' '
119
+ cat >new <<-EOF &&
120
+ # v3 git bundle
121
+ @object-format=$(test_oid algo)
122
+ @unknown=silly
123
+ EOF
124
+ test_must_fail git bundle verify new 2>output &&
125
+ test_i18ngrep "unknown capability .unknown=silly." output
126
+'
127
+
128
test_done