@matheusbsbs / matheus-bagliano / commits / 74543a0423

pkt-line: provide a LARGE_PACKET_MAX static buffer

Most of the callers of packet_read_line just read into a static 1000-byte buffer (callers which handle arbitrary binary data already use LARGE_PACKET_MAX). This works fine in practice, because: 1. The only variable-sized data in these lines is a ref name, and refs tend to be a lot shorter than 1000 characters. 2. When sending ref lines, git-core always limits itself to 1000 byte packets. However, the only limit given in the protocol specification in Documentation/technical/protocol-common.txt is LARGE_PACKET_MAX; the 1000 byte limit is mentioned only in pack-protocol.txt, and then only describing what we write, not as a specific limit for readers. This patch lets us bump the 1000-byte limit to LARGE_PACKET_MAX. Even though git-core will never write a packet where this makes a difference, there are two good reasons to do this: 1. Other git implementations may have followed protocol-common.txt and used a larger maximum size. We don't bump into it in practice because it would involve very long ref names. 2. We may want to increase the 1000-byte limit one day. Since packets are transferred before any capabilities, it's difficult to do this in a backwards-compatible way. But if we bump the size of buffer the readers can handle, eventually older versions of git will be obsolete enough that we can justify bumping the writers, as well. We don't have plans to do this anytime soon, but there is no reason not to start the clock ticking now. Just bumping all of the reading bufs to LARGE_PACKET_MAX would waste memory. Instead, since most readers just read into a temporary buffer anyway, let's provide a single static buffer that all callers can use. We can further wrap this detail away by having the packet_read_line wrapper just use the buffer transparently and return a pointer to the static storage. That covers most of the cases, and the remaining ones already read into their own LARGE_PACKET_MAX buffers. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Jeff King committed Feb 20, 2013 at 15:02 UTC 74543a0423c96130b3b07946c20b10735c3b5b15

11 files changed +47 -45

builtin/archive.c

+7 -8

@@ -27,8 +27,8 @@ static int run_remote_archiver(int argc, const char **argv,

27 const char *remote, const char *exec,

28 const char *name_hint)

29 {

30 - char buf[LARGE_PACKET_MAX];

31 - int fd[2], i, len, rv;

30 + char *buf;

31 + int fd[2], i, rv;

32 struct transport *transport;

33 struct remote *_remote;

@@ -53,19 +53,18 @@ static int run_remote_archiver(int argc, const char **argv,

53 packet_write(fd[1], "argument %s\n", argv[i]);

54 packet_flush(fd[1]);

56 - len = packet_read_line(fd[0], buf, sizeof(buf));

57 - if (!len)

56 + buf = packet_read_line(fd[0], NULL);

57 + if (!buf)

58 die(_("git archive: expected ACK/NAK, got EOF"));

59 if (strcmp(buf, "ACK")) {

60 - if (len > 5 && !prefixcmp(buf, "NACK "))

60 + if (!prefixcmp(buf, "NACK "))

61 die(_("git archive: NACK %s"), buf + 5);

62 - if (len > 4 && !prefixcmp(buf, "ERR "))

62 + if (!prefixcmp(buf, "ERR "))

63 die(_("remote error: %s"), buf + 4);

64 die(_("git archive: protocol error"));

65 }

67 - len = packet_read_line(fd[0], buf, sizeof(buf));

68 - if (len)

67 + if (packet_read_line(fd[0], NULL))

68 die(_("git archive: expected a flush"));

70 /* Now, start reading from fd[0] and spit it out to stdout */

builtin/fetch-pack.c

+3 -4

@@ -100,12 +100,11 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)

100 /* in stateless RPC mode we use pkt-line to read

101 * from stdin, until we get a flush packet

102 */

103 - static char line[1000];

103 for (;;) {

105 - int n = packet_read_line(0, line, sizeof(line));

106 - if (!n)

104 + char *line = packet_read_line(0, NULL);

105 + if (!line)

106 break;

108 - string_list_append(&sought, xmemdupz(line, n));

107 + string_list_append(&sought, xstrdup(line));

108 }

109 }

110 else {

builtin/receive-pack.c

+3 -3

@@ -754,14 +754,14 @@ static struct command *read_head_info(void)

754 struct command *commands = NULL;

755 struct command **p = &commands;

756 for (;;) {

757 - static char line[1000];

757 + char *line;

758 unsigned char old_sha1[20], new_sha1[20];

759 struct command *cmd;

760 char *refname;

761 int len, reflen;

762

763 - len = packet_read_line(0, line, sizeof(line));

764 - if (!len)

763 + line = packet_read_line(0, &len);

764 + if (!line)

765 break;

766 if (len < 83 ||

767 line[40] != ' ' ||

builtin/upload-archive.c

+2 -5

@@ -21,8 +21,6 @@ int cmd_upload_archive_writer(int argc, const char **argv, const char *prefix)

21 {

22 struct argv_array sent_argv = ARGV_ARRAY_INIT;

23 const char *arg_cmd = "argument ";

24 - char buf[4096];

25 - int len;

25 if (argc != 2)

26 usage(upload_archive_usage);

@@ -33,9 +31,8 @@ int cmd_upload_archive_writer(int argc, const char **argv, const char *prefix)

31 /* put received options in sent_argv[] */

32 argv_array_push(&sent_argv, "git-upload-archive");

33 for (;;) {

36 - /* This will die if not enough free space in buf */

37 - len = packet_read_line(0, buf, sizeof(buf));

38 - if (len == 0)

34 + char *buf = packet_read_line(0, NULL);

35 + if (!buf)

36 break; /* got a flush */

37 if (sent_argv.argc > MAX_ARGS)

38 die("Too many options (>%d)", MAX_ARGS - 1);

connect.c

+2 -2

@@ -72,11 +72,11 @@ struct ref **get_remote_heads(int in, struct ref **list,

72 for (;;) {

73 struct ref *ref;

74 unsigned char old_sha1[20];

75 - static char buffer[1000];

75 char *name;

76 int len, name_len;

77 + char *buffer = packet_buffer;

79 - len = packet_read(in, buffer, sizeof(buffer),

79 + len = packet_read(in, packet_buffer, sizeof(packet_buffer),

80 PACKET_READ_GENTLE_ON_EOF |

81 PACKET_READ_CHOMP_NEWLINE);

82 if (len < 0)

daemon.c

+2 -2

@@ -604,7 +604,7 @@ static void parse_host_arg(char *extra_args, int buflen)

604

605 static int execute(void)

606 {

607 - static char line[1000];

607 + char *line = packet_buffer;

608 int pktlen, len, i;

609 char *addr = getenv("REMOTE_ADDR"), *port = getenv("REMOTE_PORT");

610

@@ -612,7 +612,7 @@ static int execute(void)

612 loginfo("Connection from %s:%s", addr, port);

613

614 alarm(init_timeout ? init_timeout : timeout);

615 - pktlen = packet_read(0, line, sizeof(line), 0);

615 + pktlen = packet_read(0, packet_buffer, sizeof(packet_buffer), 0);

616 alarm(0);

617

618 len = strlen(line);

fetch-pack.c

+6 -6

@@ -172,8 +172,8 @@ static void consume_shallow_list(struct fetch_pack_args *args, int fd)

172 * shallow and unshallow commands every time there

173 * is a block of have lines exchanged.

174 */

175 - char line[1000];

176 - while (packet_read_line(fd, line, sizeof(line))) {

175 + char *line;

176 + while ((line = packet_read_line(fd, NULL))) {

177 if (!prefixcmp(line, "shallow "))

178 continue;

179 if (!prefixcmp(line, "unshallow "))

@@ -215,8 +215,8 @@ static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)

215

216 static enum ack_type get_ack(int fd, unsigned char *result_sha1)

217 {

218 - static char line[1000];

219 - int len = packet_read_line(fd, line, sizeof(line));

218 + int len;

219 + char *line = packet_read_line(fd, &len);

220

221 if (!len)

222 die("git fetch-pack: expected ACK/NAK, got EOF");

@@ -346,11 +346,11 @@ static int find_common(struct fetch_pack_args *args,

346 state_len = req_buf.len;

347

348 if (args->depth > 0) {

349 - char line[1024];

349 + char *line;

350 unsigned char sha1[20];

351

352 send_request(args, fd[1], &req_buf);

353 - while (packet_read_line(fd[0], line, sizeof(line))) {

353 + while ((line = packet_read_line(fd[0], NULL))) {

354 if (!prefixcmp(line, "shallow ")) {

355 if (get_sha1_hex(line + 8, sha1))

356 die("invalid shallow line: %s", line);

pkt-line.c

+7 -2

@@ -1,6 +1,7 @@

1 #include "cache.h"

2 #include "pkt-line.h"

4 +char packet_buffer[LARGE_PACKET_MAX];

5 static const char *packet_trace_prefix = "git";

6 static const char trace_key[] = "GIT_TRACE_PACKET";

@@ -174,9 +175,13 @@ int packet_read(int fd, char *buffer, unsigned size, int options)

175 return len;

176 }

177

177 -int packet_read_line(int fd, char *buffer, unsigned size)

178 +char *packet_read_line(int fd, int *len_p)

179 {

179 - return packet_read(fd, buffer, size, PACKET_READ_CHOMP_NEWLINE);

180 + int len = packet_read(fd, packet_buffer, sizeof(packet_buffer),

181 + PACKET_READ_CHOMP_NEWLINE);

182 + if (len_p)

183 + *len_p = len;

184 + return len ? packet_buffer : NULL;

185 }

186

187 int packet_get_line(struct strbuf *out,

pkt-line.h

+7 -2

@@ -54,12 +54,17 @@ int packet_read(int fd, char *buffer, unsigned size, int options);

55 /*

56 * Convenience wrapper for packet_read that is not gentle, and sets the

57 - * CHOMP_NEWLINE option.

57 + * CHOMP_NEWLINE option. The return value is NULL for a flush packet,

58 + * and otherwise points to a static buffer (that may be overwritten by

59 + * subsequent calls). If the size parameter is not NULL, the length of the

60 + * packet is written to it.

61 */

59 -int packet_read_line(int fd, char *buffer, unsigned size);

62 +char *packet_read_line(int fd, int *size);

63 +

65 #define DEFAULT_PACKET_MAX 1000

66 #define LARGE_PACKET_MAX 65520

67 +extern char packet_buffer[LARGE_PACKET_MAX];

69 int packet_get_line(struct strbuf *out, char **src_buf, size_t *src_len);

send-pack.c

+3 -4

@@ -106,9 +106,8 @@ static int pack_objects(int fd, struct ref *refs, struct extra_have_objects *ext

106 static int receive_status(int in, struct ref *refs)

107 {

108 struct ref *hint;

109 - char line[1000];

109 int ret = 0;

111 - int len = packet_read_line(in, line, sizeof(line));

110 + char *line = packet_read_line(in, NULL);

111 if (prefixcmp(line, "unpack "))

112 return error("did not receive remote status");

113 if (strcmp(line, "unpack ok")) {

@@ -119,8 +118,8 @@ static int receive_status(int in, struct ref *refs)

118 while (1) {

119 char *refname;

120 char *msg;

122 - len = packet_read_line(in, line, sizeof(line));

123 - if (!len)

121 + line = packet_read_line(in, NULL);

122 + if (!line)

123 break;

124 if (prefixcmp(line, "ok ") && prefixcmp(line, "ng ")) {

125 error("invalid ref status from remote: %s", line);

upload-pack.c

+5 -7

@@ -408,7 +408,6 @@ static int ok_to_give_up(void)

408

409 static int get_common_commits(void)

410 {

411 - static char line[1000];

411 unsigned char sha1[20];

412 char last_hex[41];

413 int got_common = 0;

@@ -418,10 +417,10 @@ static int get_common_commits(void)

417 save_commit_buffer = 0;

418

419 for (;;) {

421 - int len = packet_read_line(0, line, sizeof(line));

420 + char *line = packet_read_line(0, NULL);

421 reset_timeout();

422

424 - if (!len) {

423 + if (!line) {

424 if (multi_ack == 2 && got_common

425 && !got_other && ok_to_give_up()) {

426 sent_ready = 1;

@@ -567,8 +566,7 @@ error:

566 static void receive_needs(void)

567 {

568 struct object_array shallows = OBJECT_ARRAY_INIT;

570 - static char line[1000];

571 - int len, depth = 0;

569 + int depth = 0;

570 int has_non_tip = 0;

571

572 shallow_nr = 0;

@@ -576,9 +574,9 @@ static void receive_needs(void)

574 struct object *o;

575 const char *features;

576 unsigned char sha1_buf[20];

579 - len = packet_read_line(0, line, sizeof(line));

577 + char *line = packet_read_line(0, NULL);

578 reset_timeout();

581 - if (!len)

579 + if (!line)

580 break;

581

582 if (!prefixcmp(line, "shallow ")) {