From b7b021701cea6c9e54c826e1f6bc5faa9d3fef53 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Wed, 20 Feb 2013 14:53:33 -0500 Subject: upload-pack: use get_sha1_hex to parse "shallow" lines When we receive a line like "shallow " from the client, we feed the part to get_sha1. This is a mistake, as the argument on a shallow line is defined by Documentation/technical/pack-protocol.txt to contain an "obj-id". This is never defined in the BNF, but it is clear from the text and from the other uses that it is meant to be a hex sha1, not an arbitrary identifier (and that is what fetch-pack has always sent). We should be using get_sha1_hex instead, which doesn't allow the client to request arbitrary junk like "HEAD@{yesterday}". Because this is just marking shallow objects, the client couldn't actually do anything interesting (like fetching objects from unreachable reflog entries), but we should keep our parsing tight to be on the safe side. Because get_sha1 is for the most part a superset of get_sha1_hex, in theory the only behavior change should be disallowing non-hex object references. However, there is one interesting exception: get_sha1 will only parse a 40-character hex sha1 if the string has exactly 40 characters, whereas get_sha1_hex will just eat the first 40 characters, leaving the rest. That means that current versions of git-upload-pack will not accept a "shallow" packet that has a trailing newline, even though the protocol documentation is clear that newlines are allowed (even encouraged) in non-binary parts of the protocol. This never mattered in practice, though, because fetch-pack, contrary to the protocol documentation, does not include a newline in its shallow lines. JGit follows its lead (though it correctly is strict on the parsing end about wanting a hex object id). We do not adjust fetch-pack to send newlines here, as it would break communication with older versions of git (and there is no actual benefit to doing so, except for consistency with other parts of the protocol). Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- upload-pack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'upload-pack.c') diff --git a/upload-pack.c b/upload-pack.c index 30146a04f7..b058e8de50 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -596,7 +596,7 @@ static void receive_needs(void) if (!prefixcmp(line, "shallow ")) { unsigned char sha1[20]; struct object *object; - if (get_sha1(line + 8, sha1)) + if (get_sha1_hex(line + 8, sha1)) die("invalid shallow line: %s", line); object = parse_object(sha1); if (!object) -- cgit v1.3 From e58e57e49eb990e38df19628a744c71b44397ef1 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Wed, 20 Feb 2013 14:54:57 -0500 Subject: upload-pack: do not add duplicate objects to shallow list When the client tells us it has a shallow object via "shallow ", we make sure we have the object, mark it with a flag, then add it to a dynamic array of shallow objects. This means that a client can get us to allocate arbitrary amounts of memory just by flooding us with shallow lines (whether they have the objects or not). You can demonstrate it easily with: yes '0035shallow e83c5163316f89bfbde7d9ab23ca2e25604af290' | git-upload-pack git.git We already protect against duplicates in want lines by checking if our flag is already set; let's do the same thing here. Note that a client can still get us to allocate some amount of memory by marking every object in the repo as "shallow" (or "want"). But this at least bounds it with the number of objects in the repository, which is not under the control of an upload-pack client. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- upload-pack.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'upload-pack.c') diff --git a/upload-pack.c b/upload-pack.c index b058e8de50..bd6f25519c 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -603,8 +603,10 @@ static void receive_needs(void) die("did not find object for %s", line); if (object->type != OBJ_COMMIT) die("invalid shallow object %s", sha1_to_hex(sha1)); - object->flags |= CLIENT_SHALLOW; - add_object_array(object, NULL, &shallows); + if (!(object->flags & CLIENT_SHALLOW)) { + object->flags |= CLIENT_SHALLOW; + add_object_array(object, NULL, &shallows); + } continue; } if (!prefixcmp(line, "deepen ")) { -- cgit v1.3 From 97a83fa839d818130bfeca060279c4c355036785 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Wed, 20 Feb 2013 14:55:28 -0500 Subject: upload-pack: remove packet debugging harness If you set the GIT_DEBUG_SEND_PACK environment variable, upload-pack will dump lines it receives in the receive_needs phase to a descriptor. This debugging harness is a strict subset of what GIT_TRACE_PACKET can do. Let's just drop it in favor of that. A few tests used GIT_DEBUG_SEND_PACK to confirm which objects get sent; we have to adapt them to the new output format. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- t/t5503-tagfollow.sh | 38 +++++++++++++++++--------------------- t/t5700-clone-reference.sh | 10 +++++----- upload-pack.c | 9 --------- 3 files changed, 22 insertions(+), 35 deletions(-) (limited to 'upload-pack.c') diff --git a/t/t5503-tagfollow.sh b/t/t5503-tagfollow.sh index 60de2d6ede..d181c96815 100755 --- a/t/t5503-tagfollow.sh +++ b/t/t5503-tagfollow.sh @@ -5,7 +5,7 @@ test_description='test automatic tag following' . ./test-lib.sh if ! test_have_prereq NOT_MINGW; then - say "GIT_DEBUG_SEND_PACK not supported - skipping tests" + say "GIT_TRACE_PACKET not supported - skipping tests" fi # End state of the repository: @@ -42,21 +42,26 @@ U=UPLOAD_LOG test_expect_success NOT_MINGW 'setup expect' ' cat - <expect -#S want $A -#E EOF ' +get_needs () { + perl -alne ' + next unless $F[1] eq "upload-pack<"; + last if $F[2] eq "0000"; + print $F[2], " ", $F[3]; + ' "$@" +} + test_expect_success NOT_MINGW 'fetch A (new commit : 1 connection)' ' rm -f $U && ( cd cloned && - GIT_DEBUG_SEND_PACK=3 git fetch 3>../$U && + GIT_TRACE_PACKET=3 git fetch 3>../$U && test $A = $(git rev-parse --verify origin/master) ) && - test -s $U && - cut -d" " -f1,2 $U >actual && + get_needs $U >actual && test_cmp expect actual ' @@ -74,10 +79,8 @@ test_expect_success NOT_MINGW "create tag T on A, create C on branch cat" ' test_expect_success NOT_MINGW 'setup expect' ' cat - <expect -#S want $C want $T -#E EOF ' @@ -85,13 +88,12 @@ test_expect_success NOT_MINGW 'fetch C, T (new branch, tag : 1 connection)' ' rm -f $U && ( cd cloned && - GIT_DEBUG_SEND_PACK=3 git fetch 3>../$U && + GIT_TRACE_PACKET=3 git fetch 3>../$U && test $C = $(git rev-parse --verify origin/cat) && test $T = $(git rev-parse --verify tag1) && test $A = $(git rev-parse --verify tag1^0) ) && - test -s $U && - cut -d" " -f1,2 $U >actual && + get_needs $U >actual && test_cmp expect actual ' @@ -113,10 +115,8 @@ test_expect_success NOT_MINGW "create commits O, B, tag S on B" ' test_expect_success NOT_MINGW 'setup expect' ' cat - <expect -#S want $B want $S -#E EOF ' @@ -124,22 +124,19 @@ test_expect_success NOT_MINGW 'fetch B, S (commit and tag : 1 connection)' ' rm -f $U && ( cd cloned && - GIT_DEBUG_SEND_PACK=3 git fetch 3>../$U && + GIT_TRACE_PACKET=3 git fetch 3>../$U && test $B = $(git rev-parse --verify origin/master) && test $B = $(git rev-parse --verify tag2^0) && test $S = $(git rev-parse --verify tag2) ) && - test -s $U && - cut -d" " -f1,2 $U >actual && + get_needs $U >actual && test_cmp expect actual ' test_expect_success NOT_MINGW 'setup expect' ' cat - <expect -#S want $B want $S -#E EOF ' @@ -151,15 +148,14 @@ test_expect_success NOT_MINGW 'new clone fetch master and tags' ' cd clone2 && git init && git remote add origin .. && - GIT_DEBUG_SEND_PACK=3 git fetch 3>../$U && + GIT_TRACE_PACKET=3 git fetch 3>../$U && test $B = $(git rev-parse --verify origin/master) && test $S = $(git rev-parse --verify tag2) && test $B = $(git rev-parse --verify tag2^0) && test $T = $(git rev-parse --verify tag1) && test $A = $(git rev-parse --verify tag1^0) ) && - test -s $U && - cut -d" " -f1,2 $U >actual && + get_needs $U >actual && test_cmp expect actual ' diff --git a/t/t5700-clone-reference.sh b/t/t5700-clone-reference.sh index c47d450cc3..9cd3b4d10d 100755 --- a/t/t5700-clone-reference.sh +++ b/t/t5700-clone-reference.sh @@ -55,10 +55,10 @@ cd "$base_dir" rm -f "$U.D" test_expect_success 'cloning with reference (no -l -s)' \ -'GIT_DEBUG_SEND_PACK=3 git clone --reference B "file://$(pwd)/A" D 3>"$U.D"' +'GIT_TRACE_PACKET=3 git clone --reference B "file://$(pwd)/A" D 3>"$U.D"' test_expect_success 'fetched no objects' \ -'! grep "^want" "$U.D"' +'! grep " want" "$U.D"' cd "$base_dir" @@ -173,12 +173,12 @@ test_expect_success 'fetch with incomplete alternates' ' ( cd K && git remote add J "file://$base_dir/J" && - GIT_DEBUG_SEND_PACK=3 git fetch J 3>"$U.K" + GIT_TRACE_PACKET=3 git fetch J 3>"$U.K" ) && master_object=$(cd A && git for-each-ref --format="%(objectname)" refs/heads/master) && - ! grep "^want $master_object" "$U.K" && + ! grep " want $master_object" "$U.K" && tag_object=$(cd A && git for-each-ref --format="%(objectname)" refs/tags/HEAD) && - ! grep "^want $tag_object" "$U.K" + ! grep " want $tag_object" "$U.K" ' test_done diff --git a/upload-pack.c b/upload-pack.c index bd6f25519c..0edc79e4a4 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -42,7 +42,6 @@ static unsigned int timeout; * otherwise maximum packet size (up to 65520 bytes). */ static int use_sideband; -static int debug_fd; static int advertise_refs; static int stateless_rpc; @@ -580,8 +579,6 @@ static void receive_needs(void) int has_non_tip = 0; shallow_nr = 0; - if (debug_fd) - write_str_in_full(debug_fd, "#S\n"); for (;;) { struct object *o; const char *features; @@ -590,8 +587,6 @@ static void receive_needs(void) reset_timeout(); if (!len) break; - if (debug_fd) - write_in_full(debug_fd, line, len); if (!prefixcmp(line, "shallow ")) { unsigned char sha1[20]; @@ -653,8 +648,6 @@ static void receive_needs(void) add_object_array(o, NULL, &want_obj); } } - if (debug_fd) - write_str_in_full(debug_fd, "#E\n"); /* * We have sent all our refs already, and the other end @@ -845,8 +838,6 @@ int main(int argc, char **argv) if (is_repository_shallow()) die("attempt to fetch/clone from a shallow repository"); git_config(upload_pack_config, NULL); - if (getenv("GIT_DEBUG_SEND_PACK")) - debug_fd = atoi(getenv("GIT_DEBUG_SEND_PACK")); upload_pack(); return 0; } -- cgit v1.3 From cdf4fb8e332f9641ac1ca95e999fe98251d31392 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Wed, 20 Feb 2013 15:01:56 -0500 Subject: pkt-line: drop safe_write function This is just write_or_die by another name. The one distinction is that write_or_die will treat EPIPE specially by suppressing error messages. That's fine, as we die by SIGPIPE anyway (and in the off chance that it is disabled, write_or_die will simulate it). Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- builtin/receive-pack.c | 2 +- builtin/send-pack.c | 2 +- fetch-pack.c | 2 +- http-backend.c | 8 ++++---- pkt-line.c | 21 ++------------------- pkt-line.h | 1 - remote-curl.c | 4 ++-- send-pack.c | 2 +- sideband.c | 9 +++++---- upload-pack.c | 3 ++- 10 files changed, 19 insertions(+), 35 deletions(-) (limited to 'upload-pack.c') diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index 62ba6e7a3d..9129563782 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -932,7 +932,7 @@ static void report(struct command *commands, const char *unpack_status) if (use_sideband) send_sideband(1, 1, buf.buf, buf.len, use_sideband); else - safe_write(1, buf.buf, buf.len); + write_or_die(1, buf.buf, buf.len); strbuf_release(&buf); } diff --git a/builtin/send-pack.c b/builtin/send-pack.c index 57a46b2654..87785197cd 100644 --- a/builtin/send-pack.c +++ b/builtin/send-pack.c @@ -79,7 +79,7 @@ static void print_helper_status(struct ref *ref) } strbuf_addch(&buf, '\n'); - safe_write(1, buf.buf, buf.len); + write_or_die(1, buf.buf, buf.len); } strbuf_release(&buf); } diff --git a/fetch-pack.c b/fetch-pack.c index 27a3e80364..b53a18f923 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -247,7 +247,7 @@ static void send_request(struct fetch_pack_args *args, send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX); packet_flush(fd); } else - safe_write(fd, buf->buf, buf->len); + write_or_die(fd, buf->buf, buf->len); } static void insert_one_alternate_ref(const struct ref *ref, void *unused) diff --git a/http-backend.c b/http-backend.c index f50e77fb28..8144f3ad5e 100644 --- a/http-backend.c +++ b/http-backend.c @@ -70,7 +70,7 @@ static void format_write(int fd, const char *fmt, ...) if (n >= sizeof(buffer)) die("protocol error: impossibly long line"); - safe_write(fd, buffer, n); + write_or_die(fd, buffer, n); } static void http_status(unsigned code, const char *msg) @@ -111,7 +111,7 @@ static void hdr_cache_forever(void) static void end_headers(void) { - safe_write(1, "\r\n", 2); + write_or_die(1, "\r\n", 2); } __attribute__((format (printf, 1, 2))) @@ -157,7 +157,7 @@ static void send_strbuf(const char *type, struct strbuf *buf) hdr_int(content_length, buf->len); hdr_str(content_type, type); end_headers(); - safe_write(1, buf->buf, buf->len); + write_or_die(1, buf->buf, buf->len); } static void send_local_file(const char *the_type, const char *name) @@ -185,7 +185,7 @@ static void send_local_file(const char *the_type, const char *name) die_errno("Cannot read '%s'", p); if (!n) break; - safe_write(1, buf, n); + write_or_die(1, buf, n); } close(fd); free(buf); diff --git a/pkt-line.c b/pkt-line.c index 5138f47b64..699c2dd3b9 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -46,23 +46,6 @@ static void packet_trace(const char *buf, unsigned int len, int write) strbuf_release(&out); } -ssize_t safe_write(int fd, const void *buf, ssize_t n) -{ - ssize_t nn = n; - while (n) { - int ret = xwrite(fd, buf, n); - if (ret > 0) { - buf = (char *) buf + ret; - n -= ret; - continue; - } - if (!ret) - die("write error (disk full?)"); - die_errno("write error"); - } - return nn; -} - /* * If we buffered things up above (we don't, but we should), * we'd flush it here @@ -70,7 +53,7 @@ ssize_t safe_write(int fd, const void *buf, ssize_t n) void packet_flush(int fd) { packet_trace("0000", 4, 1); - safe_write(fd, "0000", 4); + write_or_die(fd, "0000", 4); } void packet_buf_flush(struct strbuf *buf) @@ -106,7 +89,7 @@ void packet_write(int fd, const char *fmt, ...) va_start(args, fmt); n = format_packet(fmt, args); va_end(args); - safe_write(fd, buffer, n); + write_or_die(fd, buffer, n); } void packet_buf_write(struct strbuf *buf, const char *fmt, ...) diff --git a/pkt-line.h b/pkt-line.h index 7a67e9c65b..3b6c19c4e4 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -27,6 +27,5 @@ void packet_buf_write(struct strbuf *buf, const char *fmt, ...) __attribute__((f int packet_read_line(int fd, char *buffer, unsigned size); int packet_read(int fd, char *buffer, unsigned size); int packet_get_line(struct strbuf *out, char **src_buf, size_t *src_len); -ssize_t safe_write(int, const void *, ssize_t); #endif diff --git a/remote-curl.c b/remote-curl.c index 933c69ac26..7be4b53495 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -685,7 +685,7 @@ static int fetch_git(struct discovery *heads, err = rpc_service(&rpc, heads); if (rpc.result.len) - safe_write(1, rpc.result.buf, rpc.result.len); + write_or_die(1, rpc.result.buf, rpc.result.len); strbuf_release(&rpc.result); strbuf_release(&preamble); free(depth_arg); @@ -805,7 +805,7 @@ static int push_git(struct discovery *heads, int nr_spec, char **specs) err = rpc_service(&rpc, heads); if (rpc.result.len) - safe_write(1, rpc.result.buf, rpc.result.len); + write_or_die(1, rpc.result.buf, rpc.result.len); strbuf_release(&rpc.result); free(argv); return err; diff --git a/send-pack.c b/send-pack.c index e91cbe2c95..bde796b1bb 100644 --- a/send-pack.c +++ b/send-pack.c @@ -280,7 +280,7 @@ int send_pack(struct send_pack_args *args, send_sideband(out, -1, req_buf.buf, req_buf.len, LARGE_PACKET_MAX); } } else { - safe_write(out, req_buf.buf, req_buf.len); + write_or_die(out, req_buf.buf, req_buf.len); packet_flush(out); } strbuf_release(&req_buf); diff --git a/sideband.c b/sideband.c index d5ffa1c891..8f7b25bf79 100644 --- a/sideband.c +++ b/sideband.c @@ -1,3 +1,4 @@ +#include "cache.h" #include "pkt-line.h" #include "sideband.h" @@ -108,7 +109,7 @@ int recv_sideband(const char *me, int in_stream, int out) } while (len); continue; case 1: - safe_write(out, buf + pf+1, len); + write_or_die(out, buf + pf+1, len); continue; default: fprintf(stderr, "%s: protocol error: bad band #%d\n", @@ -138,12 +139,12 @@ ssize_t send_sideband(int fd, int band, const char *data, ssize_t sz, int packet if (0 <= band) { sprintf(hdr, "%04x", n + 5); hdr[4] = band; - safe_write(fd, hdr, 5); + write_or_die(fd, hdr, 5); } else { sprintf(hdr, "%04x", n + 4); - safe_write(fd, hdr, 4); + write_or_die(fd, hdr, 4); } - safe_write(fd, p, n); + write_or_die(fd, p, n); p += n; sz -= n; } diff --git a/upload-pack.c b/upload-pack.c index 0edc79e4a4..afc2d9279c 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -69,7 +69,8 @@ static ssize_t send_client_data(int fd, const char *data, ssize_t sz) xwrite(fd, data, sz); return sz; } - return safe_write(fd, data, sz); + write_or_die(fd, data, sz); + return sz; } static FILE *pack_pipe = NULL; -- cgit v1.3 From 819b929d3389f6007e1c469d9060e7876caeb97f Mon Sep 17 00:00:00 2001 From: Jeff King Date: Wed, 20 Feb 2013 15:02:28 -0500 Subject: pkt-line: teach packet_read_line to chomp newlines The packets sent during ref negotiation are all terminated by newline; even though the code to chomp these newlines is short, we end up doing it in a lot of places. This patch teaches packet_read_line to auto-chomp the trailing newline; this lets us get rid of a lot of inline chomping code. As a result, some call-sites which are not reading line-oriented data (e.g., when reading chunks of packfiles alongside sideband) transition away from packet_read_line to the generic packet_read interface. This patch converts all of the existing callsites. Since the function signature of packet_read_line does not change (but its behavior does), there is a possibility of new callsites being introduced in later commits, silently introducing an incompatibility. However, since a later patch in this series will change the signature, such a commit would have to be merged directly into this commit, not to the tip of the series; we can therefore ignore the issue. This is an internal cleanup and should produce no change of behavior in the normal case. However, there is one corner case to note. Callers of packet_read_line have never been able to tell the difference between a flush packet ("0000") and an empty packet ("0004"), as both cause packet_read_line to return a length of 0. Readers treat them identically, even though Documentation/technical/protocol-common.txt says we must not; it also says that implementations should not send an empty pkt-line. By stripping out the newline before the result gets to the caller, we will now treat the newline-only packet ("0005\n") the same as an empty packet, which in turn gets treated like a flush packet. In practice this doesn't matter, as neither empty nor newline-only packets are part of git's protocols (at least not for the line-oriented bits, and readers who are not expecting line-oriented packets will be calling packet_read directly, anyway). But even if we do decide to care about the distinction later, it is orthogonal to this patch. The right place to tighten would be to stop treating empty packets as flush packets, and this change does not make doing so any harder. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- builtin/archive.c | 2 -- builtin/fetch-pack.c | 2 -- builtin/receive-pack.c | 2 -- builtin/upload-archive.c | 4 ---- connect.c | 5 ++--- daemon.c | 2 +- fetch-pack.c | 2 -- pkt-line.c | 7 ++++++- pkt-line.h | 9 ++++++++- remote-curl.c | 6 +++--- send-pack.c | 6 +----- sideband.c | 2 +- upload-pack.c | 8 -------- 13 files changed, 22 insertions(+), 35 deletions(-) (limited to 'upload-pack.c') diff --git a/builtin/archive.c b/builtin/archive.c index 9a1cfd3dac..d381ac4147 100644 --- a/builtin/archive.c +++ b/builtin/archive.c @@ -56,8 +56,6 @@ static int run_remote_archiver(int argc, const char **argv, len = packet_read_line(fd[0], buf, sizeof(buf)); if (!len) die(_("git archive: expected ACK/NAK, got EOF")); - if (buf[len-1] == '\n') - buf[--len] = 0; if (strcmp(buf, "ACK")) { if (len > 5 && !prefixcmp(buf, "NACK ")) die(_("git archive: NACK %s"), buf + 5); diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 940ae35dc2..f73664f433 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -105,8 +105,6 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) int n = packet_read_line(0, line, sizeof(line)); if (!n) break; - if (line[n-1] == '\n') - n--; string_list_append(&sought, xmemdupz(line, n)); } } diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index 9129563782..6679e636c7 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -763,8 +763,6 @@ static struct command *read_head_info(void) len = packet_read_line(0, line, sizeof(line)); if (!len) break; - if (line[len-1] == '\n') - line[--len] = 0; if (len < 83 || line[40] != ' ' || line[81] != ' ' || diff --git a/builtin/upload-archive.c b/builtin/upload-archive.c index 1517dec406..d90f0aba44 100644 --- a/builtin/upload-archive.c +++ b/builtin/upload-archive.c @@ -40,10 +40,6 @@ int cmd_upload_archive_writer(int argc, const char **argv, const char *prefix) if (sent_argv.argc > MAX_ARGS) die("Too many options (>%d)", MAX_ARGS - 1); - if (buf[len-1] == '\n') { - buf[--len] = 0; - } - if (prefixcmp(buf, arg_cmd)) die("'argument' token or flush expected"); argv_array_push(&sent_argv, buf + strlen(arg_cmd)); diff --git a/connect.c b/connect.c index 0aa202f885..fe8eb01ae2 100644 --- a/connect.c +++ b/connect.c @@ -77,14 +77,13 @@ struct ref **get_remote_heads(int in, struct ref **list, int len, name_len; len = packet_read(in, buffer, sizeof(buffer), - PACKET_READ_GENTLE_ON_EOF); + PACKET_READ_GENTLE_ON_EOF | + PACKET_READ_CHOMP_NEWLINE); if (len < 0) die_initial_contact(got_at_least_one_head); if (!len) break; - if (buffer[len-1] == '\n') - buffer[--len] = 0; if (len > 4 && !prefixcmp(buffer, "ERR ")) die("remote error: %s", buffer + 4); diff --git a/daemon.c b/daemon.c index 4602b46a5c..4f5cd61558 100644 --- a/daemon.c +++ b/daemon.c @@ -612,7 +612,7 @@ static int execute(void) loginfo("Connection from %s:%s", addr, port); alarm(init_timeout ? init_timeout : timeout); - pktlen = packet_read_line(0, line, sizeof(line)); + pktlen = packet_read(0, line, sizeof(line), 0); alarm(0); len = strlen(line); diff --git a/fetch-pack.c b/fetch-pack.c index b53a18f923..f830db224b 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -220,8 +220,6 @@ static enum ack_type get_ack(int fd, unsigned char *result_sha1) if (!len) die("git fetch-pack: expected ACK/NAK, got EOF"); - if (line[len-1] == '\n') - line[--len] = 0; if (!strcmp(line, "NAK")) return NAK; if (!prefixcmp(line, "ACK ")) { diff --git a/pkt-line.c b/pkt-line.c index 8700cf8add..dc11c407cd 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -164,6 +164,11 @@ int packet_read(int fd, char *buffer, unsigned size, int options) ret = safe_read(fd, buffer, len, options); if (ret < 0) return ret; + + if ((options & PACKET_READ_CHOMP_NEWLINE) && + len && buffer[len-1] == '\n') + len--; + buffer[len] = 0; packet_trace(buffer, len, 0); return len; @@ -171,7 +176,7 @@ int packet_read(int fd, char *buffer, unsigned size, int options) int packet_read_line(int fd, char *buffer, unsigned size) { - return packet_read(fd, buffer, size, 0); + return packet_read(fd, buffer, size, PACKET_READ_CHOMP_NEWLINE); } int packet_get_line(struct strbuf *out, diff --git a/pkt-line.h b/pkt-line.h index 8cd326c922..5d2fb423d6 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -44,11 +44,18 @@ void packet_buf_write(struct strbuf *buf, const char *fmt, ...) __attribute__((f * If options does contain PACKET_READ_GENTLE_ON_EOF, we will not die on * condition 4 (truncated input), but instead return -1. However, we will still * die for the other 3 conditions. + * + * If options contains PACKET_READ_CHOMP_NEWLINE, a trailing newline (if + * present) is removed from the buffer before returning. */ #define PACKET_READ_GENTLE_ON_EOF (1u<<0) +#define PACKET_READ_CHOMP_NEWLINE (1u<<1) int packet_read(int fd, char *buffer, unsigned size, int options); -/* Historical convenience wrapper for packet_read that sets no options */ +/* + * Convenience wrapper for packet_read that is not gentle, and sets the + * CHOMP_NEWLINE option. + */ int packet_read_line(int fd, char *buffer, unsigned size); int packet_get_line(struct strbuf *out, char **src_buf, size_t *src_len); diff --git a/remote-curl.c b/remote-curl.c index 7be4b53495..b28f965048 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -308,7 +308,7 @@ static size_t rpc_out(void *ptr, size_t eltsize, if (!avail) { rpc->initial_buffer = 0; - avail = packet_read_line(rpc->out, rpc->buf, rpc->alloc); + avail = packet_read(rpc->out, rpc->buf, rpc->alloc, 0); if (!avail) return 0; rpc->pos = 0; @@ -425,7 +425,7 @@ static int post_rpc(struct rpc_state *rpc) break; } - n = packet_read_line(rpc->out, buf, left); + n = packet_read(rpc->out, buf, left, 0); if (!n) break; rpc->len += n; @@ -579,7 +579,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads) rpc->hdr_accept = strbuf_detach(&buf, NULL); while (!err) { - int n = packet_read_line(rpc->out, rpc->buf, rpc->alloc); + int n = packet_read(rpc->out, rpc->buf, rpc->alloc, 0); if (!n) break; rpc->pos = 0; diff --git a/send-pack.c b/send-pack.c index bde796b1bb..8c230bf6c9 100644 --- a/send-pack.c +++ b/send-pack.c @@ -111,10 +111,7 @@ static int receive_status(int in, struct ref *refs) int len = packet_read_line(in, line, sizeof(line)); if (prefixcmp(line, "unpack ")) return error("did not receive remote status"); - if (strcmp(line, "unpack ok\n")) { - char *p = line + strlen(line) - 1; - if (*p == '\n') - *p = '\0'; + if (strcmp(line, "unpack ok")) { error("unpack failed: %s", line + 7); ret = -1; } @@ -131,7 +128,6 @@ static int receive_status(int in, struct ref *refs) break; } - line[strlen(line)-1] = '\0'; refname = line + 3; msg = strchr(refname, ' '); if (msg) diff --git a/sideband.c b/sideband.c index 8f7b25bf79..15cc1aec22 100644 --- a/sideband.c +++ b/sideband.c @@ -38,7 +38,7 @@ int recv_sideband(const char *me, int in_stream, int out) while (1) { int band, len; - len = packet_read_line(in_stream, buf + pf, LARGE_PACKET_MAX); + len = packet_read(in_stream, buf + pf, LARGE_PACKET_MAX, 0); if (len == 0) break; if (len < 1) { diff --git a/upload-pack.c b/upload-pack.c index afc2d9279c..6e6d166876 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -50,13 +50,6 @@ static void reset_timeout(void) alarm(timeout); } -static int strip(char *line, int len) -{ - if (len && line[len-1] == '\n') - line[--len] = 0; - return len; -} - static ssize_t send_client_data(int fd, const char *data, ssize_t sz) { if (use_sideband) @@ -447,7 +440,6 @@ static int get_common_commits(void) got_other = 0; continue; } - strip(line, len); if (!prefixcmp(line, "have ")) { switch (got_sha1(line+5, sha1)) { case -1: /* they have what we do not */ -- cgit v1.3 From 74543a0423c96130b3b07946c20b10735c3b5b15 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Wed, 20 Feb 2013 15:02:57 -0500 Subject: pkt-line: provide a LARGE_PACKET_MAX static buffer Most of the callers of packet_read_line just read into a static 1000-byte buffer (callers which handle arbitrary binary data already use LARGE_PACKET_MAX). This works fine in practice, because: 1. The only variable-sized data in these lines is a ref name, and refs tend to be a lot shorter than 1000 characters. 2. When sending ref lines, git-core always limits itself to 1000 byte packets. However, the only limit given in the protocol specification in Documentation/technical/protocol-common.txt is LARGE_PACKET_MAX; the 1000 byte limit is mentioned only in pack-protocol.txt, and then only describing what we write, not as a specific limit for readers. This patch lets us bump the 1000-byte limit to LARGE_PACKET_MAX. Even though git-core will never write a packet where this makes a difference, there are two good reasons to do this: 1. Other git implementations may have followed protocol-common.txt and used a larger maximum size. We don't bump into it in practice because it would involve very long ref names. 2. We may want to increase the 1000-byte limit one day. Since packets are transferred before any capabilities, it's difficult to do this in a backwards-compatible way. But if we bump the size of buffer the readers can handle, eventually older versions of git will be obsolete enough that we can justify bumping the writers, as well. We don't have plans to do this anytime soon, but there is no reason not to start the clock ticking now. Just bumping all of the reading bufs to LARGE_PACKET_MAX would waste memory. Instead, since most readers just read into a temporary buffer anyway, let's provide a single static buffer that all callers can use. We can further wrap this detail away by having the packet_read_line wrapper just use the buffer transparently and return a pointer to the static storage. That covers most of the cases, and the remaining ones already read into their own LARGE_PACKET_MAX buffers. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- builtin/archive.c | 15 +++++++-------- builtin/fetch-pack.c | 7 +++---- builtin/receive-pack.c | 6 +++--- builtin/upload-archive.c | 7 ++----- connect.c | 4 ++-- daemon.c | 4 ++-- fetch-pack.c | 12 ++++++------ pkt-line.c | 9 +++++++-- pkt-line.h | 9 +++++++-- send-pack.c | 7 +++---- upload-pack.c | 12 +++++------- 11 files changed, 47 insertions(+), 45 deletions(-) (limited to 'upload-pack.c') diff --git a/builtin/archive.c b/builtin/archive.c index d381ac4147..49178f159e 100644 --- a/builtin/archive.c +++ b/builtin/archive.c @@ -27,8 +27,8 @@ static int run_remote_archiver(int argc, const char **argv, const char *remote, const char *exec, const char *name_hint) { - char buf[LARGE_PACKET_MAX]; - int fd[2], i, len, rv; + char *buf; + int fd[2], i, rv; struct transport *transport; struct remote *_remote; @@ -53,19 +53,18 @@ static int run_remote_archiver(int argc, const char **argv, packet_write(fd[1], "argument %s\n", argv[i]); packet_flush(fd[1]); - len = packet_read_line(fd[0], buf, sizeof(buf)); - if (!len) + buf = packet_read_line(fd[0], NULL); + if (!buf) die(_("git archive: expected ACK/NAK, got EOF")); if (strcmp(buf, "ACK")) { - if (len > 5 && !prefixcmp(buf, "NACK ")) + if (!prefixcmp(buf, "NACK ")) die(_("git archive: NACK %s"), buf + 5); - if (len > 4 && !prefixcmp(buf, "ERR ")) + if (!prefixcmp(buf, "ERR ")) die(_("remote error: %s"), buf + 4); die(_("git archive: protocol error")); } - len = packet_read_line(fd[0], buf, sizeof(buf)); - if (len) + if (packet_read_line(fd[0], NULL)) die(_("git archive: expected a flush")); /* Now, start reading from fd[0] and spit it out to stdout */ diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index f73664f433..c21cc2c778 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -100,12 +100,11 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) /* in stateless RPC mode we use pkt-line to read * from stdin, until we get a flush packet */ - static char line[1000]; for (;;) { - int n = packet_read_line(0, line, sizeof(line)); - if (!n) + char *line = packet_read_line(0, NULL); + if (!line) break; - string_list_append(&sought, xmemdupz(line, n)); + string_list_append(&sought, xstrdup(line)); } } else { diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index 6679e636c7..ccebd74f16 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -754,14 +754,14 @@ static struct command *read_head_info(void) struct command *commands = NULL; struct command **p = &commands; for (;;) { - static char line[1000]; + char *line; unsigned char old_sha1[20], new_sha1[20]; struct command *cmd; char *refname; int len, reflen; - len = packet_read_line(0, line, sizeof(line)); - if (!len) + line = packet_read_line(0, &len); + if (!line) break; if (len < 83 || line[40] != ' ' || diff --git a/builtin/upload-archive.c b/builtin/upload-archive.c index d90f0aba44..af2da35e7d 100644 --- a/builtin/upload-archive.c +++ b/builtin/upload-archive.c @@ -21,8 +21,6 @@ int cmd_upload_archive_writer(int argc, const char **argv, const char *prefix) { struct argv_array sent_argv = ARGV_ARRAY_INIT; const char *arg_cmd = "argument "; - char buf[4096]; - int len; if (argc != 2) usage(upload_archive_usage); @@ -33,9 +31,8 @@ int cmd_upload_archive_writer(int argc, const char **argv, const char *prefix) /* put received options in sent_argv[] */ argv_array_push(&sent_argv, "git-upload-archive"); for (;;) { - /* This will die if not enough free space in buf */ - len = packet_read_line(0, buf, sizeof(buf)); - if (len == 0) + char *buf = packet_read_line(0, NULL); + if (!buf) break; /* got a flush */ if (sent_argv.argc > MAX_ARGS) die("Too many options (>%d)", MAX_ARGS - 1); diff --git a/connect.c b/connect.c index fe8eb01ae2..611ffb4419 100644 --- a/connect.c +++ b/connect.c @@ -72,11 +72,11 @@ struct ref **get_remote_heads(int in, struct ref **list, for (;;) { struct ref *ref; unsigned char old_sha1[20]; - static char buffer[1000]; char *name; int len, name_len; + char *buffer = packet_buffer; - len = packet_read(in, buffer, sizeof(buffer), + len = packet_read(in, packet_buffer, sizeof(packet_buffer), PACKET_READ_GENTLE_ON_EOF | PACKET_READ_CHOMP_NEWLINE); if (len < 0) diff --git a/daemon.c b/daemon.c index 4f5cd61558..3f70e79b8e 100644 --- a/daemon.c +++ b/daemon.c @@ -604,7 +604,7 @@ static void parse_host_arg(char *extra_args, int buflen) static int execute(void) { - static char line[1000]; + char *line = packet_buffer; int pktlen, len, i; char *addr = getenv("REMOTE_ADDR"), *port = getenv("REMOTE_PORT"); @@ -612,7 +612,7 @@ static int execute(void) loginfo("Connection from %s:%s", addr, port); alarm(init_timeout ? init_timeout : timeout); - pktlen = packet_read(0, line, sizeof(line), 0); + pktlen = packet_read(0, packet_buffer, sizeof(packet_buffer), 0); alarm(0); len = strlen(line); diff --git a/fetch-pack.c b/fetch-pack.c index f830db224b..66ff9add89 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -172,8 +172,8 @@ static void consume_shallow_list(struct fetch_pack_args *args, int fd) * shallow and unshallow commands every time there * is a block of have lines exchanged. */ - char line[1000]; - while (packet_read_line(fd, line, sizeof(line))) { + char *line; + while ((line = packet_read_line(fd, NULL))) { if (!prefixcmp(line, "shallow ")) continue; if (!prefixcmp(line, "unshallow ")) @@ -215,8 +215,8 @@ static int write_shallow_commits(struct strbuf *out, int use_pack_protocol) static enum ack_type get_ack(int fd, unsigned char *result_sha1) { - static char line[1000]; - int len = packet_read_line(fd, line, sizeof(line)); + int len; + char *line = packet_read_line(fd, &len); if (!len) die("git fetch-pack: expected ACK/NAK, got EOF"); @@ -346,11 +346,11 @@ static int find_common(struct fetch_pack_args *args, state_len = req_buf.len; if (args->depth > 0) { - char line[1024]; + char *line; unsigned char sha1[20]; send_request(args, fd[1], &req_buf); - while (packet_read_line(fd[0], line, sizeof(line))) { + while ((line = packet_read_line(fd[0], NULL))) { if (!prefixcmp(line, "shallow ")) { if (get_sha1_hex(line + 8, sha1)) die("invalid shallow line: %s", line); diff --git a/pkt-line.c b/pkt-line.c index dc11c407cd..55fb688899 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -1,6 +1,7 @@ #include "cache.h" #include "pkt-line.h" +char packet_buffer[LARGE_PACKET_MAX]; static const char *packet_trace_prefix = "git"; static const char trace_key[] = "GIT_TRACE_PACKET"; @@ -174,9 +175,13 @@ int packet_read(int fd, char *buffer, unsigned size, int options) return len; } -int packet_read_line(int fd, char *buffer, unsigned size) +char *packet_read_line(int fd, int *len_p) { - return packet_read(fd, buffer, size, PACKET_READ_CHOMP_NEWLINE); + int len = packet_read(fd, packet_buffer, sizeof(packet_buffer), + PACKET_READ_CHOMP_NEWLINE); + if (len_p) + *len_p = len; + return len ? packet_buffer : NULL; } int packet_get_line(struct strbuf *out, diff --git a/pkt-line.h b/pkt-line.h index 6927ea521b..fa93e32071 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -54,12 +54,17 @@ int packet_read(int fd, char *buffer, unsigned size, int options); /* * Convenience wrapper for packet_read that is not gentle, and sets the - * CHOMP_NEWLINE option. + * CHOMP_NEWLINE option. The return value is NULL for a flush packet, + * and otherwise points to a static buffer (that may be overwritten by + * subsequent calls). If the size parameter is not NULL, the length of the + * packet is written to it. */ -int packet_read_line(int fd, char *buffer, unsigned size); +char *packet_read_line(int fd, int *size); + #define DEFAULT_PACKET_MAX 1000 #define LARGE_PACKET_MAX 65520 +extern char packet_buffer[LARGE_PACKET_MAX]; int packet_get_line(struct strbuf *out, char **src_buf, size_t *src_len); diff --git a/send-pack.c b/send-pack.c index 8c230bf6c9..7d172ef37f 100644 --- a/send-pack.c +++ b/send-pack.c @@ -106,9 +106,8 @@ static int pack_objects(int fd, struct ref *refs, struct extra_have_objects *ext static int receive_status(int in, struct ref *refs) { struct ref *hint; - char line[1000]; int ret = 0; - int len = packet_read_line(in, line, sizeof(line)); + char *line = packet_read_line(in, NULL); if (prefixcmp(line, "unpack ")) return error("did not receive remote status"); if (strcmp(line, "unpack ok")) { @@ -119,8 +118,8 @@ static int receive_status(int in, struct ref *refs) while (1) { char *refname; char *msg; - len = packet_read_line(in, line, sizeof(line)); - if (!len) + line = packet_read_line(in, NULL); + if (!line) break; if (prefixcmp(line, "ok ") && prefixcmp(line, "ng ")) { error("invalid ref status from remote: %s", line); diff --git a/upload-pack.c b/upload-pack.c index 6e6d166876..98ddb69581 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -408,7 +408,6 @@ static int ok_to_give_up(void) static int get_common_commits(void) { - static char line[1000]; unsigned char sha1[20]; char last_hex[41]; int got_common = 0; @@ -418,10 +417,10 @@ static int get_common_commits(void) save_commit_buffer = 0; for (;;) { - int len = packet_read_line(0, line, sizeof(line)); + char *line = packet_read_line(0, NULL); reset_timeout(); - if (!len) { + if (!line) { if (multi_ack == 2 && got_common && !got_other && ok_to_give_up()) { sent_ready = 1; @@ -567,8 +566,7 @@ error: static void receive_needs(void) { struct object_array shallows = OBJECT_ARRAY_INIT; - static char line[1000]; - int len, depth = 0; + int depth = 0; int has_non_tip = 0; shallow_nr = 0; @@ -576,9 +574,9 @@ static void receive_needs(void) struct object *o; const char *features; unsigned char sha1_buf[20]; - len = packet_read_line(0, line, sizeof(line)); + char *line = packet_read_line(0, NULL); reset_timeout(); - if (!len) + if (!line) break; if (!prefixcmp(line, "shallow ")) { -- cgit v1.3