@matheusbsbs / matheus-bagliano / commits / c44cdc7eef
Fork 0

Converted fast-import to a text based protocol.

Frontend clients can now send a text stream to fast-import rather than a binary stream. This should facilitate developing frontend software as the data stream is easier to view, manipulate and debug my hand and Mark-I eyeball. Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

Shawn O. Pearce committed Aug 14, 2006 at 20:16 UTC c44cdc7eef212ec09901eb2e0996476e0468ed88
1 file changed +326 -160
fast-import.c
+326 -160
@@ -4,57 +4,93 @@ Format of STDIN stream:
4 stream ::= cmd*;
5
6 cmd ::= new_blob
7 - | new_commit
7 | new_branch
8 + | new_commit
9 | new_tag
10 ;
11
12 - new_blob ::= 'blob' blob_data;
12 + new_blob ::= 'blob' lf
13 + mark?
14 + file_content;
15 + file_content ::= data;
16
14 - new_commit ::= 'comt' ref_name author_committer_msg
15 - file_change*
16 - '0';
17 + new_branch ::= 'branch' sp ref_str lf
18 + ('from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf)?
19 + lf;
20
18 - new_branch ::= 'brch' dst_ref_name src_ref_name;
19 - dst_ref_name ::= ref_name;
20 - src_ref_name ::= ref_name | sha1_exp;
21 -
22 - new_tag ::= 'tagg' ref_name tag_name tagger_msg;
21 + new_commit ::= 'commit' sp ref_str lf
22 + mark?
23 + ('author' sp name '<' email '>' ts tz lf)?
24 + 'committer' sp name '<' email '>' ts tz lf
25 + commit_msg
26 + file_change*
27 + lf;
28 + commit_msg ::= data;
29
24 - file_change ::= 'M' path_name hexsha1
25 - | 'D' path_name
30 + file_change ::= 'M' sp mode sp (hexsha1 | idnum) sp path_str lf
31 + | 'D' sp path_str lf
32 ;
27 -
28 - author_committer_msg ::= len32
29 - 'author' sp name '<' email '>' ts tz lf
30 - 'committer' sp name '<' email '>' ts tz lf
31 - lf
32 - binary_data;
33 -
34 - tagger_msg ::= len32
35 - 'tagger' sp name '<' email '>' ts tz lf
36 - lf
37 - binary_data;
38 -
39 - blob_data ::= len32 binary_data; # max len is 2^32-1
40 - path_name ::= len32 path; # max len is PATH_MAX-1
41 - ref_name ::= len32 ref; # max len is PATH_MAX-1
42 - tag_name ::= len32 tag; # max len is PATH_MAX-1
43 - sha1_exp ::= len32 sha1exp; # max len is PATH_MAX-1
44 -
45 - len32 ::= # unsigned 32 bit value, native format;
33 + mode ::= '644' | '755';
34 +
35 + new_tag ::= 'tag' sp tag_str lf
36 + 'from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf
37 + 'tagger' sp name '<' email '>' ts tz lf
38 + tag_msg;
39 + tag_msg ::= data;
40 +
41 + # note: the first idnum in a stream should be 1 and subsequent
42 + # idnums should not have gaps between values as this will cause
43 + # the stream parser to reserve space for the gapped values. An
44 + # idnum can be updated in the future to a new object by issuing
45 + # a new mark directive with the old idnum.
46 + #
47 + mark ::= 'mark' sp idnum lf;
48 +
49 + # note: declen indicates the length of binary_data in bytes.
50 + # declen does not include the lf preceeding or trailing the
51 + # binary data.
52 + #
53 + data ::= 'data' sp declen lf
54 + binary_data
55 + lf;
56 +
57 + # note: quoted strings are C-style quoting supporting \c for
58 + # common escapes of 'c' (e..g \n, \t, \\, \") or \nnn where nnn
59 + # is the signed byte value in octal. Note that the only
60 + # characters which must actually be escaped to protect the
61 + # stream formatting is: \, " and LF. Otherwise these values
62 + # are UTF8.
63 + #
64 + ref_str ::= ref | '"' quoted(ref) '"' ;
65 + sha1exp_str ::= sha1exp | '"' quoted(sha1exp) '"' ;
66 + tag_str ::= tag | '"' quoted(tag) '"' ;
67 + path_str ::= path | '"' quoted(path) '"' ;
68 +
69 + declen ::= # unsigned 32 bit value, ascii base10 notation;
70 binary_data ::= # file content, not interpreted;
71 +
72 sp ::= # ASCII space character;
73 lf ::= # ASCII newline (LF) character;
49 - path ::= # GIT style file path, e.g. "a/b/c";
50 - ref ::= # GIT ref name, e.g. "refs/heads/MOZ_GECKO_EXPERIMENT";
51 - tag ::= # GIT tag name, e.g. "FIREFOX_1_5";
74 +
75 + # note: a colon (':') must precede the numerical value assigned to
76 + # an idnum. This is to distinguish it from a ref or tag name as
77 + # GIT does not permit ':' in ref or tag strings.
78 + #
79 + idnum ::= ':' declen;
80 + path ::= # GIT style file path, e.g. "a/b/c";
81 + ref ::= # GIT ref name, e.g. "refs/heads/MOZ_GECKO_EXPERIMENT";
82 + tag ::= # GIT tag name, e.g. "FIREFOX_1_5";
83 sha1exp ::= # Any valid GIT SHA1 expression;
84 hexsha1 ::= # SHA1 in hexadecimal format;
54 - name ::= # valid GIT author/committer name;
85 +
86 + # note: name and email are UTF8 strings, however name must not
87 + # contain '<' or lf and email must not contain any of the
88 + # following: '<', '>', lf.
89 + #
90 + name ::= # valid GIT author/committer name;
91 email ::= # valid GIT author/committer email;
56 - ts ::= # time since the epoch in seconds, ascii decimal;
57 - tz ::= # GIT style timezone;
92 + ts ::= # time since the epoch in seconds, ascii base10 notation;
93 + tz ::= # GIT style timezone;
94 */
95
96 #include "builtin.h"
@@ -66,6 +102,8 @@ Format of STDIN stream:
102 #include "pack.h"
103 #include "refs.h"
104 #include "csum-file.h"
105 +#include "strbuf.h"
106 +#include "quote.h"
107
108 struct object_entry
109 {
@@ -153,7 +191,7 @@ static size_t mem_pool_alloc = 2*1024*1024 - sizeof(struct mem_pool);
191 static size_t total_allocd;
192 static struct mem_pool *mem_pool;
193
156 -/* atom management */
194 +/* Atom management */
195 static unsigned int atom_table_sz = 4451;
196 static unsigned int atom_cnt;
197 static struct atom_str **atom_table;
@@ -184,6 +222,10 @@ static unsigned int branch_table_sz = 1039;
222 static struct branch **branch_table;
223 static struct branch *active_branches;
224
225 +/* Input stream parsing */
226 +static struct strbuf command_buf;
227 +static unsigned long command_mark;
228 +
229
230 static void alloc_objects(int cnt)
231 {
@@ -330,6 +372,8 @@ static struct branch* new_branch(const char *name)
372
373 if (b)
374 die("Invalid attempt to create duplicate branch: %s", name);
375 + if (check_ref_format(name))
376 + die("Branch name doesn't conform to GIT standards: %s", name);
377
378 b = pool_calloc(1, sizeof(struct branch));
379 b->name = pool_strdup(name);
@@ -433,22 +477,6 @@ static void yread(int fd, void *buffer, size_t length)
477 }
478 }
479
436 -static int optional_read(int fd, void *buffer, size_t length)
437 -{
438 - ssize_t ret = 0;
439 - while (ret < length) {
440 - ssize_t size = xread(fd, (char *) buffer + ret, length - ret);
441 - if (!size && !ret)
442 - return 1;
443 - if (!size)
444 - die("Read from descriptor %i: end of stream", fd);
445 - if (size < 0)
446 - die("Read from descriptor %i: %s", fd, strerror(errno));
447 - ret += size;
448 - }
449 - return 0;
450 -}
451 -
480 static void ywrite(int fd, void *buffer, size_t length)
481 {
482 ssize_t ret = 0;
@@ -462,24 +490,9 @@ static void ywrite(int fd, void *buffer, size_t length)
490 }
491 }
492
465 -static const char* read_path()
466 -{
467 - static char sn[PATH_MAX];
468 - unsigned long slen;
469 -
470 - yread(0, &slen, 4);
471 - if (!slen)
472 - die("Expected string command parameter, didn't find one");
473 - if (slen > (PATH_MAX - 1))
474 - die("Can't handle excessive string length %lu", slen);
475 - yread(0, sn, slen);
476 - sn[slen] = 0;
477 - return sn;
478 -}
479 -
480 -static unsigned long encode_header(
493 +static size_t encode_header(
494 enum object_type type,
482 - unsigned long size,
495 + size_t size,
496 unsigned char *hdr)
497 {
498 int n = 1;
@@ -503,7 +516,7 @@ static unsigned long encode_header(
516 static int store_object(
517 enum object_type type,
518 void *dat,
506 - unsigned long datlen,
519 + size_t datlen,
520 struct last_object *last,
521 unsigned char *sha1out)
522 {
@@ -896,15 +909,57 @@ static void dump_branches()
909 }
910 }
911
912 +static void read_next_command()
913 +{
914 + read_line(&command_buf, stdin, '\n');
915 +}
916 +
917 +static void cmd_mark()
918 +{
919 + if (!strncmp("mark :", command_buf.buf, 6)) {
920 + command_mark = strtoul(command_buf.buf + 6, NULL, 10);
921 + read_next_command();
922 + }
923 + else
924 + command_mark = 0;
925 +}
926 +
927 +static void* cmd_data (size_t *size)
928 +{
929 + size_t n = 0;
930 + void *buffer;
931 + size_t length;
932 +
933 + if (strncmp("data ", command_buf.buf, 5))
934 + die("Expected 'data n' command, found: %s", command_buf.buf);
935 +
936 + length = strtoul(command_buf.buf + 5, NULL, 10);
937 + buffer = xmalloc(length);
938 +
939 + while (n < length) {
940 + size_t s = fread((char*)buffer + n, 1, length - n, stdin);
941 + if (!s && feof(stdin))
942 + die("EOF in data (%lu bytes remaining)", length - n);
943 + n += s;
944 + }
945 +
946 + if (fgetc(stdin) != '\n')
947 + die("An lf did not trail the binary data as expected.");
948 +
949 + *size = length;
950 + return buffer;
951 +}
952 +
953 static void cmd_new_blob()
954 {
901 - unsigned long datlen;
902 - unsigned char sha1[20];
955 + size_t datlen;
956 void *dat;
957 + unsigned char sha1[20];
958 +
959 + read_next_command();
960 + cmd_mark();
961 + dat = cmd_data(&datlen);
962
905 - yread(0, &datlen, 4);
906 - dat = xmalloc(datlen);
907 - yread(0, dat, datlen);
963 if (store_object(OBJ_BLOB, dat, datlen, &last_blob, sha1))
964 free(dat);
965 }
@@ -949,122 +1004,231 @@ static void load_branch(struct branch *b)
1004
1005 static void file_change_m(struct branch *b)
1006 {
952 - const char *path = read_path();
1007 + const char *p = command_buf.buf + 2;
1008 + char *p_uq;
1009 + const char *endp;
1010 struct object_entry *oe;
954 - char hexsha1[41];
1011 unsigned char sha1[20];
1012 + unsigned int mode;
1013 char type[20];
1014
958 - yread(0, hexsha1, 40);
959 - hexsha1[40] = 0;
1015 + p = get_mode(p, &mode);
1016 + if (!p)
1017 + die("Corrupt mode: %s", command_buf.buf);
1018 + switch (mode) {
1019 + case S_IFREG | 0644:
1020 + case S_IFREG | 0755:
1021 + case 0644:
1022 + case 0755:
1023 + /* ok */
1024 + break;
1025 + default:
1026 + die("Corrupt mode: %s", command_buf.buf);
1027 + }
1028 +
1029 + if (get_sha1_hex(p, sha1))
1030 + die("Invalid SHA1: %s", command_buf.buf);
1031 + p += 40;
1032 + if (*p++ != ' ')
1033 + die("Missing space after SHA1: %s", command_buf.buf);
1034 +
1035 + p_uq = unquote_c_style(p, &endp);
1036 + if (p_uq) {
1037 + if (*endp)
1038 + die("Garbage after path in: %s", command_buf.buf);
1039 + p = p_uq;
1040 + }
1041
961 - if (get_sha1_hex(hexsha1, sha1))
962 - die("Invalid sha1 %s for %s", hexsha1, path);
1042 oe = find_object(sha1);
1043 if (oe) {
1044 if (oe->type != OBJ_BLOB)
966 - die("%s is a %s not a blob (for %s)", hexsha1, type_names[oe->type], path);
1045 + die("Not a blob (actually a %s): %s",
1046 + command_buf.buf, type_names[oe->type]);
1047 } else {
1048 if (sha1_object_info(sha1, type, NULL))
969 - die("No blob %s for %s", hexsha1, path);
1049 + die("Blob not found: %s", command_buf.buf);
1050 if (strcmp(blob_type, type))
971 - die("%s is a %s not a blob (for %s)", hexsha1, type, path);
1051 + die("Not a blob (actually a %s): %s",
1052 + command_buf.buf, type);
1053 }
1054
974 - tree_content_set(&b->branch_tree, path, sha1, S_IFREG | 0644);
1055 + tree_content_set(&b->branch_tree, p, sha1, S_IFREG | mode);
1056 +
1057 + if (p_uq)
1058 + free(p_uq);
1059 }
1060
1061 static void file_change_d(struct branch *b)
1062 {
979 - tree_content_remove(&b->branch_tree, read_path());
1063 + const char *p = command_buf.buf + 2;
1064 + char *p_uq;
1065 + const char *endp;
1066 +
1067 + p_uq = unquote_c_style(p, &endp);
1068 + if (p_uq) {
1069 + if (*endp)
1070 + die("Garbage after path in: %s", command_buf.buf);
1071 + p = p_uq;
1072 + }
1073 + tree_content_remove(&b->branch_tree, p);
1074 + if (p_uq)
1075 + free(p_uq);
1076 }
1077
1078 static void cmd_new_commit()
1079 {
984 - static const unsigned int max_hdr_len = 94;
985 - const char *name = read_path();
986 - struct branch *b = lookup_branch(name);
987 - unsigned int acmsglen;
988 - char *body, *c;
989 -
1080 + struct branch *b;
1081 + void *msg;
1082 + size_t msglen;
1083 + char *str_uq;
1084 + const char *endp;
1085 + char *sp;
1086 + char *author = NULL;
1087 + char *committer = NULL;
1088 + char *body;
1089 +
1090 + /* Obtain the branch name from the rest of our command */
1091 + sp = strchr(command_buf.buf, ' ') + 1;
1092 + str_uq = unquote_c_style(sp, &endp);
1093 + if (str_uq) {
1094 + if (*endp)
1095 + die("Garbage after ref in: %s", command_buf.buf);
1096 + sp = str_uq;
1097 + }
1098 + b = lookup_branch(sp);
1099 if (!b)
991 - die("Branch not declared: %s", name);
1100 + die("Branch not declared: %s", sp);
1101 + if (str_uq)
1102 + free(str_uq);
1103 +
1104 + read_next_command();
1105 + cmd_mark();
1106 + if (!strncmp("author ", command_buf.buf, 7)) {
1107 + author = strdup(command_buf.buf);
1108 + read_next_command();
1109 + }
1110 + if (!strncmp("committer ", command_buf.buf, 10)) {
1111 + committer = strdup(command_buf.buf);
1112 + read_next_command();
1113 + }
1114 + if (!committer)
1115 + die("Expected committer but didn't get one");
1116 + msg = cmd_data(&msglen);
1117 +
1118 + /* ensure the branch is active/loaded */
1119 if (!b->branch_tree.tree) {
1120 unload_one_branch();
1121 load_branch(b);
1122 }
1123
997 - /* author_committer_msg */
998 - yread(0, &acmsglen, 4);
999 - body = xmalloc(acmsglen + max_hdr_len);
1000 - c = body + max_hdr_len;
1001 - yread(0, c, acmsglen);
1002 -
1003 - /* oddly enough this is all that fsck-objects cares about */
1004 - if (memcmp(c, "author ", 7))
1005 - die("Invalid commit format on branch %s", name);
1006 -
1124 /* file_change* */
1125 for (;;) {
1009 - unsigned char cmd;
1010 - yread(0, &cmd, 1);
1011 - if (cmd == '0')
1126 + read_next_command();
1127 + if (1 == command_buf.len)
1128 break;
1013 - else if (cmd == 'M')
1129 + else if (!strncmp("M ", command_buf.buf, 2))
1130 file_change_m(b);
1015 - else if (cmd == 'D')
1131 + else if (!strncmp("D ", command_buf.buf, 2))
1132 file_change_d(b);
1133 else
1018 - die("Unsupported file_change: %c", cmd);
1134 + die("Unsupported file_change: %s", command_buf.buf);
1135 }
1136
1021 - if (memcmp(b->sha1, null_sha1, 20)) {
1022 - sprintf(c - 48, "parent %s", sha1_to_hex(b->sha1));
1023 - *(c - 1) = '\n';
1024 - c -= 48;
1025 - }
1137 + /* build the tree and the commit */
1138 store_tree(&b->branch_tree);
1027 - sprintf(c - 46, "tree %s", sha1_to_hex(b->branch_tree.sha1));
1028 - *(c - 1) = '\n';
1029 - c -= 46;
1030 -
1031 - store_object(OBJ_COMMIT,
1032 - c, (body + max_hdr_len + acmsglen) - c,
1033 - NULL, b->sha1);
1139 + body = xmalloc(97 + msglen
1140 + + (author
1141 + ? strlen(author) + strlen(committer)
1142 + : 2 * strlen(committer)));
1143 + sp = body;
1144 + sp += sprintf(sp, "tree %s\n", sha1_to_hex(b->branch_tree.sha1));
1145 + if (memcmp(b->sha1, null_sha1, 20))
1146 + sp += sprintf(sp, "parent %s\n", sha1_to_hex(b->sha1));
1147 + if (author)
1148 + sp += sprintf(sp, "%s\n", author);
1149 + else
1150 + sp += sprintf(sp, "author %s\n", committer + 10);
1151 + sp += sprintf(sp, "%s\n\n", committer);
1152 + memcpy(sp, msg, msglen);
1153 + sp += msglen;
1154 + if (author)
1155 + free(author);
1156 + free(committer);
1157 + free(msg);
1158 +
1159 + store_object(OBJ_COMMIT, body, sp - body, NULL, b->sha1);
1160 free(body);
1161 b->last_commit = object_count_by_type[OBJ_COMMIT];
1162 }
1163
1164 static void cmd_new_branch()
1165 {
1040 - struct branch *b = new_branch(read_path());
1041 - const char *base = read_path();
1042 - struct branch *s = lookup_branch(base);
1043 -
1044 - if (!strcmp(b->name, base))
1045 - die("Can't create a branch from itself: %s", base);
1046 - else if (s) {
1047 - memcpy(b->sha1, s->sha1, 20);
1048 - memcpy(b->branch_tree.sha1, s->branch_tree.sha1, 20);
1166 + struct branch *b;
1167 + char *str_uq;
1168 + const char *endp;
1169 + char *sp;
1170 +
1171 + /* Obtain the new branch name from the rest of our command */
1172 + sp = strchr(command_buf.buf, ' ') + 1;
1173 + str_uq = unquote_c_style(sp, &endp);
1174 + if (str_uq) {
1175 + if (*endp)
1176 + die("Garbage after ref in: %s", command_buf.buf);
1177 + sp = str_uq;
1178 }
1050 - else if (!get_sha1(base, b->sha1)) {
1051 - if (!memcmp(b->sha1, null_sha1, 20))
1052 - memcpy(b->branch_tree.sha1, null_sha1, 20);
1053 - else {
1054 - unsigned long size;
1055 - char *buf;
1056 -
1057 - buf = read_object_with_reference(b->sha1,
1058 - type_names[OBJ_COMMIT], &size, b->sha1);
1059 - if (!buf || size < 46)
1060 - die("Not a valid commit: %s", base);
1061 - if (memcmp("tree ", buf, 5)
1062 - || get_sha1_hex(buf + 5, b->branch_tree.sha1))
1063 - die("The commit %s is corrupt", sha1_to_hex(b->sha1));
1064 - free(buf);
1179 + b = new_branch(sp);
1180 + if (str_uq)
1181 + free(str_uq);
1182 + read_next_command();
1183 +
1184 + /* from ... */
1185 + if (!strncmp("from ", command_buf.buf, 5)) {
1186 + const char *from;
1187 + struct branch *s;
1188 +
1189 + from = strchr(command_buf.buf, ' ') + 1;
1190 + str_uq = unquote_c_style(from, &endp);
1191 + if (str_uq) {
1192 + if (*endp)
1193 + die("Garbage after string in: %s", command_buf.buf);
1194 + from = str_uq;
1195 }
1066 - } else
1067 - die("Not a SHA1 or branch: %s", base);
1196 +
1197 + s = lookup_branch(from);
1198 + if (b == s)
1199 + die("Can't create a branch from itself: %s", b->name);
1200 + else if (s) {
1201 + memcpy(b->sha1, s->sha1, 20);
1202 + memcpy(b->branch_tree.sha1, s->branch_tree.sha1, 20);
1203 + } else if (!get_sha1(from, b->sha1)) {
1204 + if (!memcmp(b->sha1, null_sha1, 20))
1205 + memcpy(b->branch_tree.sha1, null_sha1, 20);
1206 + else {
1207 + unsigned long size;
1208 + char *buf;
1209 +
1210 + buf = read_object_with_reference(b->sha1,
1211 + type_names[OBJ_COMMIT], &size, b->sha1);
1212 + if (!buf || size < 46)
1213 + die("Not a valid commit: %s", from);
1214 + if (memcmp("tree ", buf, 5)
1215 + || get_sha1_hex(buf + 5, b->branch_tree.sha1))
1216 + die("The commit %s is corrupt", sha1_to_hex(b->sha1));
1217 + free(buf);
1218 + }
1219 + } else
1220 + die("Invalid ref name or SHA1 expression: %s", from);
1221 +
1222 + if (str_uq)
1223 + free(str_uq);
1224 + read_next_command();
1225 + } else {
1226 + memcpy(b->sha1, null_sha1, 20);
1227 + memcpy(b->branch_tree.sha1, null_sha1, 20);
1228 + }
1229 +
1230 + if (command_buf.eof || command_buf.len > 1)
1231 + die("An lf did not terminate the branch command as expected.");
1232 }
1233
1234 int main(int argc, const char **argv)
@@ -1087,26 +1251,28 @@ int main(int argc, const char **argv)
1251 if (pack_fd < 0)
1252 die("Can't create %s: %s", pack_name, strerror(errno));
1253
1254 + init_pack_header();
1255 alloc_objects(est_obj_cnt);
1256 + strbuf_init(&command_buf);
1257
1258 atom_table = xcalloc(atom_table_sz, sizeof(struct atom_str*));
1259 branch_table = xcalloc(branch_table_sz, sizeof(struct branch*));
1260 avail_tree_table = xcalloc(avail_tree_table_sz, sizeof(struct avail_tree_content*));
1261
1096 - init_pack_header();
1262 for (;;) {
1098 - unsigned long cmd;
1099 - if (optional_read(0, &cmd, 4))
1263 + read_next_command();
1264 + if (command_buf.eof)
1265 break;
1101 -
1102 - switch (ntohl(cmd)) {
1103 - case 'blob': cmd_new_blob(); break;
1104 - case 'comt': cmd_new_commit(); break;
1105 - case 'brch': cmd_new_branch(); break;
1106 - default:
1107 - die("Invalid command %lu", cmd);
1108 - }
1266 + else if (!strcmp("blob", command_buf.buf))
1267 + cmd_new_blob();
1268 + else if (!strncmp("branch ", command_buf.buf, 7))
1269 + cmd_new_branch();
1270 + else if (!strncmp("commit ", command_buf.buf, 7))
1271 + cmd_new_commit();
1272 + else
1273 + die("Unsupported command: %s", command_buf.buf);
1274 }
1275 +
1276 fixup_header_footer();
1277 close(pack_fd);
1278 write_index(idx_name);