fast-import: let importers retrieve blobs
New objects written by fast-import are not available immediately. Until a checkpoint has been started and finishes writing the pack index, any new blobs will not be accessible using standard git tools. So introduce a new way to access them: a "cat-blob" command in the command stream requests for fast-import to print a blob to stdout or a file descriptor specified by the argument to --cat-blob-fd. The value for cat-blob-fd cannot be specified in the stream because that would be a layering violation: the decision of where to direct a stream has to be made when fast-import is started anyway, so we might as well make the stream format is independent of that detail. Output uses the same format as "git cat-file --batch". Thanks to Sverre Rabbelier and Sam Vilain for guidance in designing the protocol. Based-on-patch-by: Jonathan Nieder <jrnieder@gmail.com> Signed-off-by: David Barr <david.barr@cordelta.com> Acked-by: Ramkumar Ramachandra <artagnon@gmail.com> Signed-off-by: Jonathan Nieder <jrnieder@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
David Barr committed
Nov 28, 2010 at 13:45 UTC
85c62395b152f99e8867aaf84cea93dddc03243c
3 files changed
+329
-2
Documentation/git-fast-import.txt
+41
@@ -92,6 +92,11 @@ OPTIONS
92
--(no-)-relative-marks= with the --(import|export)-marks=
93
options.
94
95
+--cat-blob-fd=<fd>::
96
+ Specify the file descriptor that will be written to
97
+ when the `cat-blob` command is encountered in the stream.
98
+ The default behaviour is to write to `stdout`.
99
+
100
--export-pack-edges=<file>::
101
After creating a packfile, print a line of data to
102
<file> listing the filename of the packfile and the last
@@ -320,6 +325,11 @@ and control the current import process. More detailed discussion
325
standard output. This command is optional and is not needed
326
to perform an import.
327
328
+`cat-blob`::
329
+ Causes fast-import to print a blob in 'cat-file --batch'
330
+ format to the file descriptor set with `--cat-blob-fd` or
331
+ `stdout` if unspecified.
332
+
333
`feature`::
334
Require that fast-import supports the specified feature, or
335
abort if it does not.
@@ -879,6 +889,29 @@ Placing a `progress` command immediately after a `checkpoint` will
889
inform the reader when the `checkpoint` has been completed and it
890
can safely access the refs that fast-import updated.
891
892
+`cat-blob`
893
+~~~~~~~~~~
894
+Causes fast-import to print a blob to a file descriptor previously
895
+arranged with the `--cat-blob-fd` argument. The command otherwise
896
+has no impact on the current import; its main purpose is to
897
+retrieve blobs that may be in fast-import's memory but not
898
+accessible from the target repository.
899
+
900
+....
901
+ 'cat-blob' SP <dataref> LF
902
+....
903
+
904
+The `<dataref>` can be either a mark reference (`:<idnum>`)
905
+set previously or a full 40-byte SHA-1 of a Git blob, preexisting or
906
+ready to be written.
907
+
908
+output uses the same format as `git cat-file --batch`:
909
+
910
+====
911
+ <sha1> SP 'blob' SP <size> LF
912
+ <contents> LF
913
+====
914
+
915
`feature`
916
~~~~~~~~~
917
Require that fast-import supports the specified feature, or abort if
@@ -905,6 +938,13 @@ import-marks::
938
second, an --import-marks= command-line option overrides
939
any "feature import-marks" command in the stream.
940
941
+cat-blob::
942
+ Ignored. Versions of fast-import not supporting the
943
+ "cat-blob" command will exit with a message indicating so.
944
+ This lets the import error out early with a clear message,
945
+ rather than wasting time on the early part of an import
946
+ before the unsupported command is detected.
947
+
948
`option`
949
~~~~~~~~
950
Processes the specified option so that git fast-import behaves in a
@@ -930,6 +970,7 @@ not be passed as option:
970
* date-format
971
* import-marks
972
* export-marks
973
+* cat-blob-fd
974
* force
975
976
Crash Reports
fast-import.c
+97
@@ -55,6 +55,8 @@ Format of STDIN stream:
55
('from' sp committish lf)?
56
lf?;
57
58
+ cat_blob ::= 'cat-blob' sp (hexsha1 | idnum) lf;
59
+
60
checkpoint ::= 'checkpoint' lf
61
lf?;
62
@@ -361,6 +363,9 @@ static uintmax_t next_mark;
363
static struct strbuf new_data = STRBUF_INIT;
364
static int seen_data_command;
365
366
+/* Where to write output of cat-blob commands */
367
+static int cat_blob_fd = STDOUT_FILENO;
368
+
369
static void parse_argv(void);
370
371
static void write_branch_report(FILE *rpt, struct branch *b)
@@ -2689,6 +2694,81 @@ static void parse_reset_branch(void)
2694
unread_command_buf = 1;
2695
}
2696
2697
+static void cat_blob_write(const char *buf, unsigned long size)
2698
+{
2699
+ if (write_in_full(cat_blob_fd, buf, size) != size)
2700
+ die_errno("Write to frontend failed");
2701
+}
2702
+
2703
+static void cat_blob(struct object_entry *oe, unsigned char sha1[20])
2704
+{
2705
+ struct strbuf line = STRBUF_INIT;
2706
+ unsigned long size;
2707
+ enum object_type type = 0;
2708
+ char *buf;
2709
+
2710
+ if (!oe || oe->pack_id == MAX_PACK_ID) {
2711
+ buf = read_sha1_file(sha1, &type, &size);
2712
+ } else {
2713
+ type = oe->type;
2714
+ buf = gfi_unpack_entry(oe, &size);
2715
+ }
2716
+
2717
+ /*
2718
+ * Output based on batch_one_object() from cat-file.c.
2719
+ */
2720
+ if (type <= 0) {
2721
+ strbuf_reset(&line);
2722
+ strbuf_addf(&line, "%s missing\n", sha1_to_hex(sha1));
2723
+ cat_blob_write(line.buf, line.len);
2724
+ strbuf_release(&line);
2725
+ free(buf);
2726
+ return;
2727
+ }
2728
+ if (!buf)
2729
+ die("Can't read object %s", sha1_to_hex(sha1));
2730
+ if (type != OBJ_BLOB)
2731
+ die("Object %s is a %s but a blob was expected.",
2732
+ sha1_to_hex(sha1), typename(type));
2733
+ strbuf_reset(&line);
2734
+ strbuf_addf(&line, "%s %s %lu\n", sha1_to_hex(sha1),
2735
+ typename(type), size);
2736
+ cat_blob_write(line.buf, line.len);
2737
+ strbuf_release(&line);
2738
+ cat_blob_write(buf, size);
2739
+ cat_blob_write("\n", 1);
2740
+ free(buf);
2741
+}
2742
+
2743
+static void parse_cat_blob(void)
2744
+{
2745
+ const char *p;
2746
+ struct object_entry *oe = oe;
2747
+ unsigned char sha1[20];
2748
+
2749
+ /* cat-blob SP <object> LF */
2750
+ p = command_buf.buf + strlen("cat-blob ");
2751
+ if (*p == ':') {
2752
+ char *x;
2753
+ oe = find_mark(strtoumax(p + 1, &x, 10));
2754
+ if (x == p + 1)
2755
+ die("Invalid mark: %s", command_buf.buf);
2756
+ if (!oe)
2757
+ die("Unknown mark: %s", command_buf.buf);
2758
+ if (*x)
2759
+ die("Garbage after mark: %s", command_buf.buf);
2760
+ hashcpy(sha1, oe->idx.sha1);
2761
+ } else {
2762
+ if (get_sha1_hex(p, sha1))
2763
+ die("Invalid SHA1: %s", command_buf.buf);
2764
+ if (p[40])
2765
+ die("Garbage after SHA1: %s", command_buf.buf);
2766
+ oe = find_object(sha1);
2767
+ }
2768
+
2769
+ cat_blob(oe, sha1);
2770
+}
2771
+
2772
static void parse_checkpoint(void)
2773
{
2774
if (object_count) {
@@ -2773,6 +2853,14 @@ static void option_export_marks(const char *marks)
2853
safe_create_leading_directories_const(export_marks_file);
2854
}
2855
2856
+static void option_cat_blob_fd(const char *fd)
2857
+{
2858
+ unsigned long n = ulong_arg("--cat-blob-fd", fd);
2859
+ if (n > (unsigned long) INT_MAX)
2860
+ die("--cat-blob-fd cannot exceed %d", INT_MAX);
2861
+ cat_blob_fd = (int) n;
2862
+}
2863
+
2864
static void option_export_pack_edges(const char *edges)
2865
{
2866
if (pack_edges)
@@ -2826,6 +2914,8 @@ static int parse_one_feature(const char *feature, int from_stream)
2914
option_import_marks(feature + 13, from_stream);
2915
} else if (!prefixcmp(feature, "export-marks=")) {
2916
option_export_marks(feature + 13);
2917
+ } else if (!strcmp(feature, "cat-blob")) {
2918
+ ; /* Don't die - this feature is supported */
2919
} else if (!prefixcmp(feature, "relative-marks")) {
2920
relative_marks_paths = 1;
2921
} else if (!prefixcmp(feature, "no-relative-marks")) {
@@ -2920,6 +3010,11 @@ static void parse_argv(void)
3010
if (parse_one_feature(a + 2, 0))
3011
continue;
3012
3013
+ if (!prefixcmp(a + 2, "cat-blob-fd=")) {
3014
+ option_cat_blob_fd(a + 2 + strlen("cat-blob-fd="));
3015
+ continue;
3016
+ }
3017
+
3018
die("unknown option %s", a);
3019
}
3020
if (i != global_argc)
@@ -2971,6 +3066,8 @@ int main(int argc, const char **argv)
3066
parse_new_tag();
3067
else if (!prefixcmp(command_buf.buf, "reset "))
3068
parse_reset_branch();
3069
+ else if (!prefixcmp(command_buf.buf, "cat-blob "))
3070
+ parse_cat_blob();
3071
else if (!strcmp("checkpoint", command_buf.buf))
3072
parse_checkpoint();
3073
else if (!prefixcmp(command_buf.buf, "progress "))
t/t9300-fast-import.sh
+191
-2
@@ -23,11 +23,18 @@ file5_data='an inline file.
23
file6_data='#!/bin/sh
24
echo "$@"'
25
26
+>empty
27
+
28
###
29
### series A
30
###
31
32
test_tick
33
+
34
+test_expect_success 'empty stream succeeds' '
35
+ git fast-import </dev/null
36
+'
37
+
38
cat >input <<INPUT_END
39
blob
40
mark :2
@@ -1632,6 +1639,190 @@ test_expect_success 'R: feature no-relative-marks should be honoured' '
1639
test_cmp marks.new non-relative.out
1640
'
1641
1642
+test_expect_success 'R: feature cat-blob supported' '
1643
+ echo "feature cat-blob" |
1644
+ git fast-import
1645
+'
1646
+
1647
+test_expect_success 'R: cat-blob-fd must be a nonnegative integer' '
1648
+ test_must_fail git fast-import --cat-blob-fd=-1 </dev/null
1649
+'
1650
+
1651
+test_expect_success 'R: print old blob' '
1652
+ blob=$(echo "yes it can" | git hash-object -w --stdin) &&
1653
+ cat >expect <<-EOF &&
1654
+ ${blob} blob 11
1655
+ yes it can
1656
+
1657
+ EOF
1658
+ echo "cat-blob $blob" |
1659
+ git fast-import --cat-blob-fd=6 6>actual &&
1660
+ test_cmp expect actual
1661
+'
1662
+
1663
+test_expect_success 'R: in-stream cat-blob-fd not respected' '
1664
+ echo hello >greeting &&
1665
+ blob=$(git hash-object -w greeting) &&
1666
+ cat >expect <<-EOF &&
1667
+ ${blob} blob 6
1668
+ hello
1669
+
1670
+ EOF
1671
+ git fast-import --cat-blob-fd=3 3>actual.3 >actual.1 <<-EOF &&
1672
+ cat-blob $blob
1673
+ EOF
1674
+ test_cmp expect actual.3 &&
1675
+ test_cmp empty actual.1 &&
1676
+ git fast-import 3>actual.3 >actual.1 <<-EOF &&
1677
+ option cat-blob-fd=3
1678
+ cat-blob $blob
1679
+ EOF
1680
+ test_cmp empty actual.3 &&
1681
+ test_cmp expect actual.1
1682
+'
1683
+
1684
+test_expect_success 'R: print new blob' '
1685
+ blob=$(echo "yep yep yep" | git hash-object --stdin) &&
1686
+ cat >expect <<-EOF &&
1687
+ ${blob} blob 12
1688
+ yep yep yep
1689
+
1690
+ EOF
1691
+ git fast-import --cat-blob-fd=6 6>actual <<-\EOF &&
1692
+ blob
1693
+ mark :1
1694
+ data <<BLOB_END
1695
+ yep yep yep
1696
+ BLOB_END
1697
+ cat-blob :1
1698
+ EOF
1699
+ test_cmp expect actual
1700
+'
1701
+
1702
+test_expect_success 'R: print new blob by sha1' '
1703
+ blob=$(echo "a new blob named by sha1" | git hash-object --stdin) &&
1704
+ cat >expect <<-EOF &&
1705
+ ${blob} blob 25
1706
+ a new blob named by sha1
1707
+
1708
+ EOF
1709
+ git fast-import --cat-blob-fd=6 6>actual <<-EOF &&
1710
+ blob
1711
+ data <<BLOB_END
1712
+ a new blob named by sha1
1713
+ BLOB_END
1714
+ cat-blob $blob
1715
+ EOF
1716
+ test_cmp expect actual
1717
+'
1718
+
1719
+test_expect_success 'setup: big file' '
1720
+ (
1721
+ echo "the quick brown fox jumps over the lazy dog" >big &&
1722
+ for i in 1 2 3
1723
+ do
1724
+ cat big big big big >bigger &&
1725
+ cat bigger bigger bigger bigger >big ||
1726
+ exit
1727
+ done
1728
+ )
1729
+'
1730
+
1731
+test_expect_success 'R: print two blobs to stdout' '
1732
+ blob1=$(git hash-object big) &&
1733
+ blob1_len=$(wc -c <big) &&
1734
+ blob2=$(echo hello | git hash-object --stdin) &&
1735
+ {
1736
+ echo ${blob1} blob $blob1_len &&
1737
+ cat big &&
1738
+ cat <<-EOF
1739
+
1740
+ ${blob2} blob 6
1741
+ hello
1742
+
1743
+ EOF
1744
+ } >expect &&
1745
+ {
1746
+ cat <<-\END_PART1 &&
1747
+ blob
1748
+ mark :1
1749
+ data <<data_end
1750
+ END_PART1
1751
+ cat big &&
1752
+ cat <<-\EOF
1753
+ data_end
1754
+ blob
1755
+ mark :2
1756
+ data <<data_end
1757
+ hello
1758
+ data_end
1759
+ cat-blob :1
1760
+ cat-blob :2
1761
+ EOF
1762
+ } |
1763
+ git fast-import >actual &&
1764
+ test_cmp expect actual
1765
+'
1766
+
1767
+test_expect_success 'setup: have pipes?' '
1768
+ rm -f frob &&
1769
+ if mkfifo frob
1770
+ then
1771
+ test_set_prereq PIPE
1772
+ fi
1773
+'
1774
+
1775
+test_expect_success PIPE 'R: copy using cat-file' '
1776
+ expect_id=$(git hash-object big) &&
1777
+ expect_len=$(wc -c <big) &&
1778
+ echo $expect_id blob $expect_len >expect.response &&
1779
+
1780
+ rm -f blobs &&
1781
+ cat >frontend <<-\FRONTEND_END &&
1782
+ #!/bin/sh
1783
+ cat <<EOF &&
1784
+ feature cat-blob
1785
+ blob
1786
+ mark :1
1787
+ data <<BLOB
1788
+ EOF
1789
+ cat big
1790
+ cat <<EOF
1791
+ BLOB
1792
+ cat-blob :1
1793
+ EOF
1794
+
1795
+ read blob_id type size <&3 &&
1796
+ echo "$blob_id $type $size" >response &&
1797
+ dd if=/dev/stdin of=blob bs=$size count=1 <&3 &&
1798
+ read newline <&3 &&
1799
+
1800
+ cat <<EOF &&
1801
+ commit refs/heads/copied
1802
+ committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
1803
+ data <<COMMIT
1804
+ copy big file as file3
1805
+ COMMIT
1806
+ M 644 inline file3
1807
+ data <<BLOB
1808
+ EOF
1809
+ cat blob &&
1810
+ cat <<EOF
1811
+ BLOB
1812
+ EOF
1813
+ FRONTEND_END
1814
+
1815
+ mkfifo blobs &&
1816
+ (
1817
+ export GIT_COMMITTER_NAME GIT_COMMITTER_EMAIL GIT_COMMITTER_DATE &&
1818
+ sh frontend 3<blobs |
1819
+ git fast-import --cat-blob-fd=3 3>blobs
1820
+ ) &&
1821
+ git show copied:file3 >actual &&
1822
+ test_cmp expect.response response &&
1823
+ test_cmp big actual
1824
+'
1825
+
1826
cat >input << EOF
1827
option git quiet
1828
blob
@@ -1640,8 +1831,6 @@ hi
1831
1832
EOF
1833
1643
-touch empty
1644
-
1834
test_expect_success 'R: quiet option results in no stats being output' '
1835
cat input | git fast-import 2> output &&
1836
test_cmp empty output