From a3d6b53e92528d20fdaca504614f260c5b457b29 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:41 -0700 Subject: upload-pack: convert to a builtin In order to allow for code sharing with the server-side of fetch in protocol-v2 convert upload-pack to be a builtin. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- builtin/upload-pack.c | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 builtin/upload-pack.c (limited to 'builtin') diff --git a/builtin/upload-pack.c b/builtin/upload-pack.c new file mode 100644 index 0000000000..2cb5cb35b0 --- /dev/null +++ b/builtin/upload-pack.c @@ -0,0 +1,67 @@ +#include "cache.h" +#include "builtin.h" +#include "exec_cmd.h" +#include "pkt-line.h" +#include "parse-options.h" +#include "protocol.h" +#include "upload-pack.h" + +static const char * const upload_pack_usage[] = { + N_("git upload-pack [] "), + NULL +}; + +int cmd_upload_pack(int argc, const char **argv, const char *prefix) +{ + const char *dir; + int strict = 0; + struct upload_pack_options opts = { 0 }; + struct option options[] = { + OPT_BOOL(0, "stateless-rpc", &opts.stateless_rpc, + N_("quit after a single request/response exchange")), + OPT_BOOL(0, "advertise-refs", &opts.advertise_refs, + N_("exit immediately after initial ref advertisement")), + OPT_BOOL(0, "strict", &strict, + N_("do not try /.git/ if is no Git directory")), + OPT_INTEGER(0, "timeout", &opts.timeout, + N_("interrupt transfer after seconds of inactivity")), + OPT_END() + }; + + packet_trace_identity("upload-pack"); + check_replace_refs = 0; + + argc = parse_options(argc, argv, NULL, options, upload_pack_usage, 0); + + if (argc != 1) + usage_with_options(upload_pack_usage, options); + + if (opts.timeout) + opts.daemon_mode = 1; + + setup_path(); + + dir = argv[0]; + + if (!enter_repo(dir, strict)) + die("'%s' does not appear to be a git repository", dir); + + switch (determine_protocol_version_server()) { + case protocol_v1: + /* + * v1 is just the original protocol with a version string, + * so just fall through after writing the version string. + */ + if (opts.advertise_refs || !opts.stateless_rpc) + packet_write_fmt(1, "version 1\n"); + + /* fallthrough */ + case protocol_v0: + upload_pack(&opts); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } + + return 0; +} -- cgit v1.3-5-g9baa From ad6ac1244fd175d08bcee62060a9a0b7975930fb Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:45 -0700 Subject: connect: discover protocol version outside of get_remote_heads In order to prepare for the addition of protocol_v2 push the protocol version discovery outside of 'get_remote_heads()'. This will allow for keeping the logic for processing the reference advertisement for protocol_v1 and protocol_v0 separate from the logic for protocol_v2. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- builtin/fetch-pack.c | 16 +++++++++++++++- builtin/send-pack.c | 17 +++++++++++++++-- connect.c | 27 ++++++++++----------------- connect.h | 3 +++ remote-curl.c | 20 ++++++++++++++++++-- remote.h | 5 +++-- transport.c | 24 +++++++++++++++++++----- 7 files changed, 83 insertions(+), 29 deletions(-) (limited to 'builtin') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 366b9d13f9..85d4faf76c 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -4,6 +4,7 @@ #include "remote.h" #include "connect.h" #include "sha1-array.h" +#include "protocol.h" static const char fetch_pack_usage[] = "git fetch-pack [--all] [--stdin] [--quiet | -q] [--keep | -k] [--thin] " @@ -52,6 +53,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) struct fetch_pack_args args; struct oid_array shallow = OID_ARRAY_INIT; struct string_list deepen_not = STRING_LIST_INIT_DUP; + struct packet_reader reader; packet_trace_identity("fetch-pack"); @@ -193,7 +195,19 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) if (!conn) return args.diag_url ? 0 : 1; } - get_remote_heads(fd[0], NULL, 0, &ref, 0, NULL, &shallow); + + packet_reader_init(&reader, fd[0], NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + switch (discover_version(&reader)) { + case protocol_v1: + case protocol_v0: + get_remote_heads(&reader, &ref, 0, NULL, &shallow); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } ref = fetch_pack(&args, fd, conn, ref, dest, sought, nr_sought, &shallow, pack_lockfile_ptr); diff --git a/builtin/send-pack.c b/builtin/send-pack.c index fc4f0bb5fb..83cb125a68 100644 --- a/builtin/send-pack.c +++ b/builtin/send-pack.c @@ -14,6 +14,7 @@ #include "sha1-array.h" #include "gpg-interface.h" #include "gettext.h" +#include "protocol.h" static const char * const send_pack_usage[] = { N_("git send-pack [--all | --mirror] [--dry-run] [--force] " @@ -154,6 +155,7 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) int progress = -1; int from_stdin = 0; struct push_cas_option cas = {0}; + struct packet_reader reader; struct option options[] = { OPT__VERBOSITY(&verbose), @@ -256,8 +258,19 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) args.verbose ? CONNECT_VERBOSE : 0); } - get_remote_heads(fd[0], NULL, 0, &remote_refs, REF_NORMAL, - &extra_have, &shallow); + packet_reader_init(&reader, fd[0], NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + switch (discover_version(&reader)) { + case protocol_v1: + case protocol_v0: + get_remote_heads(&reader, &remote_refs, REF_NORMAL, + &extra_have, &shallow); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } transport_verify_remote_names(nr_refspecs, refspecs); diff --git a/connect.c b/connect.c index c82c90b7c3..0b111e62d7 100644 --- a/connect.c +++ b/connect.c @@ -62,7 +62,7 @@ static void die_initial_contact(int unexpected) "and the repository exists.")); } -static enum protocol_version discover_version(struct packet_reader *reader) +enum protocol_version discover_version(struct packet_reader *reader) { enum protocol_version version = protocol_unknown_version; @@ -233,7 +233,7 @@ enum get_remote_heads_state { /* * Read all the refs from the other end */ -struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, +struct ref **get_remote_heads(struct packet_reader *reader, struct ref **list, unsigned int flags, struct oid_array *extra_have, struct oid_array *shallow_points) @@ -241,24 +241,17 @@ struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, struct ref **orig_list = list; int len = 0; enum get_remote_heads_state state = EXPECTING_FIRST_REF; - struct packet_reader reader; const char *arg; - packet_reader_init(&reader, in, src_buf, src_len, - PACKET_READ_CHOMP_NEWLINE | - PACKET_READ_GENTLE_ON_EOF); - - discover_version(&reader); - *list = NULL; while (state != EXPECTING_DONE) { - switch (packet_reader_read(&reader)) { + switch (packet_reader_read(reader)) { case PACKET_READ_EOF: die_initial_contact(1); case PACKET_READ_NORMAL: - len = reader.pktlen; - if (len > 4 && skip_prefix(reader.line, "ERR ", &arg)) + len = reader->pktlen; + if (len > 4 && skip_prefix(reader->line, "ERR ", &arg)) die("remote error: %s", arg); break; case PACKET_READ_FLUSH: @@ -270,22 +263,22 @@ struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, switch (state) { case EXPECTING_FIRST_REF: - process_capabilities(reader.line, &len); - if (process_dummy_ref(reader.line)) { + process_capabilities(reader->line, &len); + if (process_dummy_ref(reader->line)) { state = EXPECTING_SHALLOW; break; } state = EXPECTING_REF; /* fallthrough */ case EXPECTING_REF: - if (process_ref(reader.line, len, &list, flags, extra_have)) + if (process_ref(reader->line, len, &list, flags, extra_have)) break; state = EXPECTING_SHALLOW; /* fallthrough */ case EXPECTING_SHALLOW: - if (process_shallow(reader.line, len, shallow_points)) + if (process_shallow(reader->line, len, shallow_points)) break; - die("protocol error: unexpected '%s'", reader.line); + die("protocol error: unexpected '%s'", reader->line); case EXPECTING_DONE: break; } diff --git a/connect.h b/connect.h index 01f14cdf3f..cdb8979dce 100644 --- a/connect.h +++ b/connect.h @@ -13,4 +13,7 @@ extern int parse_feature_request(const char *features, const char *feature); extern const char *server_feature_value(const char *feature, int *len_ret); extern int url_is_local_not_ssh(const char *url); +struct packet_reader; +extern enum protocol_version discover_version(struct packet_reader *reader); + #endif diff --git a/remote-curl.c b/remote-curl.c index 0053b09549..9f6d07683d 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -1,6 +1,7 @@ #include "cache.h" #include "config.h" #include "remote.h" +#include "connect.h" #include "strbuf.h" #include "walker.h" #include "http.h" @@ -13,6 +14,7 @@ #include "credential.h" #include "sha1-array.h" #include "send-pack.h" +#include "protocol.h" static struct remote *remote; /* always ends with a trailing slash */ @@ -176,8 +178,22 @@ static struct discovery *last_discovery; static struct ref *parse_git_refs(struct discovery *heads, int for_push) { struct ref *list = NULL; - get_remote_heads(-1, heads->buf, heads->len, &list, - for_push ? REF_NORMAL : 0, NULL, &heads->shallow); + struct packet_reader reader; + + packet_reader_init(&reader, -1, heads->buf, heads->len, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + switch (discover_version(&reader)) { + case protocol_v1: + case protocol_v0: + get_remote_heads(&reader, &list, for_push ? REF_NORMAL : 0, + NULL, &heads->shallow); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } + return list; } diff --git a/remote.h b/remote.h index 1f6611be21..2016461df9 100644 --- a/remote.h +++ b/remote.h @@ -150,10 +150,11 @@ int check_ref_type(const struct ref *ref, int flags); void free_refs(struct ref *ref); struct oid_array; -extern struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, +struct packet_reader; +extern struct ref **get_remote_heads(struct packet_reader *reader, struct ref **list, unsigned int flags, struct oid_array *extra_have, - struct oid_array *shallow); + struct oid_array *shallow_points); int resolve_remote_symref(struct ref *ref, struct ref *list); int ref_newer(const struct object_id *new_oid, const struct object_id *old_oid); diff --git a/transport.c b/transport.c index 8e87790962..63c3dbab94 100644 --- a/transport.c +++ b/transport.c @@ -18,6 +18,7 @@ #include "sha1-array.h" #include "sigchain.h" #include "transport-internal.h" +#include "protocol.h" static void set_upstreams(struct transport *transport, struct ref *refs, int pretend) @@ -190,13 +191,26 @@ static int connect_setup(struct transport *transport, int for_push) static struct ref *get_refs_via_connect(struct transport *transport, int for_push) { struct git_transport_data *data = transport->data; - struct ref *refs; + struct ref *refs = NULL; + struct packet_reader reader; connect_setup(transport, for_push); - get_remote_heads(data->fd[0], NULL, 0, &refs, - for_push ? REF_NORMAL : 0, - &data->extra_have, - &data->shallow); + + packet_reader_init(&reader, data->fd[0], NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + switch (discover_version(&reader)) { + case protocol_v1: + case protocol_v0: + get_remote_heads(&reader, &refs, + for_push ? REF_NORMAL : 0, + &data->extra_have, + &data->shallow); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } data->got_remote_heads = 1; return refs; -- cgit v1.3-5-g9baa From 8f6982b4e16c60bba713a3b6592b2ff5c7476974 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 14 Mar 2018 11:31:47 -0700 Subject: protocol: introduce enum protocol_version value protocol_v2 Introduce protocol_v2, a new value for 'enum protocol_version'. Subsequent patches will fill in the implementation of protocol_v2. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- builtin/fetch-pack.c | 2 ++ builtin/receive-pack.c | 6 ++++++ builtin/send-pack.c | 3 +++ builtin/upload-pack.c | 7 +++++++ connect.c | 3 +++ protocol.c | 2 ++ protocol.h | 1 + remote-curl.c | 3 +++ transport.c | 9 +++++++++ 9 files changed, 36 insertions(+) (limited to 'builtin') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 85d4faf76c..b2374ddbbf 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -201,6 +201,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) PACKET_READ_GENTLE_ON_EOF); switch (discover_version(&reader)) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); case protocol_v1: case protocol_v0: get_remote_heads(&reader, &ref, 0, NULL, &shallow); diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index b7ce7c7f52..3656e94fdb 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -1963,6 +1963,12 @@ int cmd_receive_pack(int argc, const char **argv, const char *prefix) unpack_limit = receive_unpack_limit; switch (determine_protocol_version_server()) { + case protocol_v2: + /* + * push support for protocol v2 has not been implemented yet, + * so ignore the request to use v2 and fallback to using v0. + */ + break; case protocol_v1: /* * v1 is just the original protocol with a version string, diff --git a/builtin/send-pack.c b/builtin/send-pack.c index 83cb125a68..b5427f75e3 100644 --- a/builtin/send-pack.c +++ b/builtin/send-pack.c @@ -263,6 +263,9 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) PACKET_READ_GENTLE_ON_EOF); switch (discover_version(&reader)) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; case protocol_v1: case protocol_v0: get_remote_heads(&reader, &remote_refs, REF_NORMAL, diff --git a/builtin/upload-pack.c b/builtin/upload-pack.c index 2cb5cb35b0..8d53e9794b 100644 --- a/builtin/upload-pack.c +++ b/builtin/upload-pack.c @@ -47,6 +47,13 @@ int cmd_upload_pack(int argc, const char **argv, const char *prefix) die("'%s' does not appear to be a git repository", dir); switch (determine_protocol_version_server()) { + case protocol_v2: + /* + * fetch support for protocol v2 has not been implemented yet, + * so ignore the request to use v2 and fallback to using v0. + */ + upload_pack(&opts); + break; case protocol_v1: /* * v1 is just the original protocol with a version string, diff --git a/connect.c b/connect.c index 0b111e62d7..4b89b984c4 100644 --- a/connect.c +++ b/connect.c @@ -83,6 +83,9 @@ enum protocol_version discover_version(struct packet_reader *reader) } switch (version) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; case protocol_v1: /* Read the peeked version line */ packet_reader_read(reader); diff --git a/protocol.c b/protocol.c index 43012b7eb6..5e636785d1 100644 --- a/protocol.c +++ b/protocol.c @@ -8,6 +8,8 @@ static enum protocol_version parse_protocol_version(const char *value) return protocol_v0; else if (!strcmp(value, "1")) return protocol_v1; + else if (!strcmp(value, "2")) + return protocol_v2; else return protocol_unknown_version; } diff --git a/protocol.h b/protocol.h index 1b2bc94a8d..2ad35e433c 100644 --- a/protocol.h +++ b/protocol.h @@ -5,6 +5,7 @@ enum protocol_version { protocol_unknown_version = -1, protocol_v0 = 0, protocol_v1 = 1, + protocol_v2 = 2, }; /* diff --git a/remote-curl.c b/remote-curl.c index 9f6d07683d..dae8a4a48d 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -185,6 +185,9 @@ static struct ref *parse_git_refs(struct discovery *heads, int for_push) PACKET_READ_GENTLE_ON_EOF); switch (discover_version(&reader)) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; case protocol_v1: case protocol_v0: get_remote_heads(&reader, &list, for_push ? REF_NORMAL : 0, diff --git a/transport.c b/transport.c index 2378dcb38c..83d9dd1df6 100644 --- a/transport.c +++ b/transport.c @@ -203,6 +203,9 @@ static struct ref *get_refs_via_connect(struct transport *transport, int for_pus data->version = discover_version(&reader); switch (data->version) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; case protocol_v1: case protocol_v0: get_remote_heads(&reader, &refs, @@ -250,6 +253,9 @@ static int fetch_refs_via_pack(struct transport *transport, refs_tmp = get_refs_via_connect(transport, 0); switch (data->version) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; case protocol_v1: case protocol_v0: refs = fetch_pack(&args, data->fd, data->conn, @@ -585,6 +591,9 @@ static int git_transport_push(struct transport *transport, struct ref *remote_re args.push_cert = SEND_PACK_PUSH_CERT_NEVER; switch (data->version) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; case protocol_v1: case protocol_v0: ret = send_pack(&args, data->fd, data->conn, remote_refs, -- cgit v1.3-5-g9baa From ed10cb952d315ae47d2f7f6f29333a9ee227bf6b Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:19 -0700 Subject: serve: introduce git-serve Introduce git-serve, the base server for protocol version 2. Protocol version 2 is intended to be a replacement for Git's current wire protocol. The intention is that it will be a simpler, less wasteful protocol which can evolve over time. Protocol version 2 improves upon version 1 by eliminating the initial ref advertisement. In its place a server will export a list of capabilities and commands which it supports in a capability advertisement. A client can then request that a particular command be executed by providing a number of capabilities and command specific parameters. At the completion of a command, a client can request that another command be executed or can terminate the connection by sending a flush packet. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- .gitignore | 1 + Documentation/Makefile | 1 + Documentation/technical/protocol-v2.txt | 170 ++++++++++++++++++++++ Makefile | 2 + builtin.h | 1 + builtin/serve.c | 30 ++++ git.c | 1 + serve.c | 247 ++++++++++++++++++++++++++++++++ serve.h | 15 ++ t/t5701-git-serve.sh | 60 ++++++++ 10 files changed, 528 insertions(+) create mode 100644 Documentation/technical/protocol-v2.txt create mode 100644 builtin/serve.c create mode 100644 serve.c create mode 100644 serve.h create mode 100755 t/t5701-git-serve.sh (limited to 'builtin') diff --git a/.gitignore b/.gitignore index 833ef3b0b7..2d0450c262 100644 --- a/.gitignore +++ b/.gitignore @@ -140,6 +140,7 @@ /git-rm /git-send-email /git-send-pack +/git-serve /git-sh-i18n /git-sh-i18n--envsubst /git-sh-setup diff --git a/Documentation/Makefile b/Documentation/Makefile index 4ae9ba5c86..b105775acd 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -77,6 +77,7 @@ TECH_DOCS += technical/pack-heuristics TECH_DOCS += technical/pack-protocol TECH_DOCS += technical/protocol-capabilities TECH_DOCS += technical/protocol-common +TECH_DOCS += technical/protocol-v2 TECH_DOCS += technical/racy-git TECH_DOCS += technical/send-pack-pipeline TECH_DOCS += technical/shallow diff --git a/Documentation/technical/protocol-v2.txt b/Documentation/technical/protocol-v2.txt new file mode 100644 index 0000000000..270b28f364 --- /dev/null +++ b/Documentation/technical/protocol-v2.txt @@ -0,0 +1,170 @@ + Git Wire Protocol, Version 2 +============================== + +This document presents a specification for a version 2 of Git's wire +protocol. Protocol v2 will improve upon v1 in the following ways: + + * Instead of multiple service names, multiple commands will be + supported by a single service + * Easily extendable as capabilities are moved into their own section + of the protocol, no longer being hidden behind a NUL byte and + limited by the size of a pkt-line + * Separate out other information hidden behind NUL bytes (e.g. agent + string as a capability and symrefs can be requested using 'ls-refs') + * Reference advertisement will be omitted unless explicitly requested + * ls-refs command to explicitly request some refs + * Designed with http and stateless-rpc in mind. With clear flush + semantics the http remote helper can simply act as a proxy + +In protocol v2 communication is command oriented. When first contacting a +server a list of capabilities will advertised. Some of these capabilities +will be commands which a client can request be executed. Once a command +has completed, a client can reuse the connection and request that other +commands be executed. + + Packet-Line Framing +--------------------- + +All communication is done using packet-line framing, just as in v1. See +`Documentation/technical/pack-protocol.txt` and +`Documentation/technical/protocol-common.txt` for more information. + +In protocol v2 these special packets will have the following semantics: + + * '0000' Flush Packet (flush-pkt) - indicates the end of a message + * '0001' Delimiter Packet (delim-pkt) - separates sections of a message + + Initial Client Request +------------------------ + +In general a client can request to speak protocol v2 by sending +`version=2` through the respective side-channel for the transport being +used which inevitably sets `GIT_PROTOCOL`. More information can be +found in `pack-protocol.txt` and `http-protocol.txt`. In all cases the +response from the server is the capability advertisement. + + Git Transport +~~~~~~~~~~~~~~~ + +When using the git:// transport, you can request to use protocol v2 by +sending "version=2" as an extra parameter: + + 003egit-upload-pack /project.git\0host=myserver.com\0\0version=2\0 + + SSH and File Transport +~~~~~~~~~~~~~~~~~~~~~~~~ + +When using either the ssh:// or file:// transport, the GIT_PROTOCOL +environment variable must be set explicitly to include "version=2". + + HTTP Transport +~~~~~~~~~~~~~~~~ + +When using the http:// or https:// transport a client makes a "smart" +info/refs request as described in `http-protocol.txt` and requests that +v2 be used by supplying "version=2" in the `Git-Protocol` header. + + C: Git-Protocol: version=2 + C: + C: GET $GIT_URL/info/refs?service=git-upload-pack HTTP/1.0 + +A v2 server would reply: + + S: 200 OK + S: + S: ... + S: + S: 000eversion 2\n + S: + +Subsequent requests are then made directly to the service +`$GIT_URL/git-upload-pack`. (This works the same for git-receive-pack). + + Capability Advertisement +-------------------------- + +A server which decides to communicate (based on a request from a client) +using protocol version 2, notifies the client by sending a version string +in its initial response followed by an advertisement of its capabilities. +Each capability is a key with an optional value. Clients must ignore all +unknown keys. Semantics of unknown values are left to the definition of +each key. Some capabilities will describe commands which can be requested +to be executed by the client. + + capability-advertisement = protocol-version + capability-list + flush-pkt + + protocol-version = PKT-LINE("version 2" LF) + capability-list = *capability + capability = PKT-LINE(key[=value] LF) + + key = 1*(ALPHA | DIGIT | "-_") + value = 1*(ALPHA | DIGIT | " -_.,?\/{}[]()<>!@#$%^&*+=:;") + + Command Request +----------------- + +After receiving the capability advertisement, a client can then issue a +request to select the command it wants with any particular capabilities +or arguments. There is then an optional section where the client can +provide any command specific parameters or queries. Only a single +command can be requested at a time. + + request = empty-request | command-request + empty-request = flush-pkt + command-request = command + capability-list + [command-args] + flush-pkt + command = PKT-LINE("command=" key LF) + command-args = delim-pkt + *command-specific-arg + + command-specific-args are packet line framed arguments defined by + each individual command. + +The server will then check to ensure that the client's request is +comprised of a valid command as well as valid capabilities which were +advertised. If the request is valid the server will then execute the +command. A server MUST wait till it has received the client's entire +request before issuing a response. The format of the response is +determined by the command being executed, but in all cases a flush-pkt +indicates the end of the response. + +When a command has finished, and the client has received the entire +response from the server, a client can either request that another +command be executed or can terminate the connection. A client may +optionally send an empty request consisting of just a flush-pkt to +indicate that no more requests will be made. + + Capabilities +-------------- + +There are two different types of capabilities: normal capabilities, +which can be used to to convey information or alter the behavior of a +request, and commands, which are the core actions that a client wants to +perform (fetch, push, etc). + +Protocol version 2 is stateless by default. This means that all commands +must only last a single round and be stateless from the perspective of the +server side, unless the client has requested a capability indicating that +state should be maintained by the server. Clients MUST NOT require state +management on the server side in order to function correctly. This +permits simple round-robin load-balancing on the server side, without +needing to worry about state management. + + agent +~~~~~~~ + +The server can advertise the `agent` capability with a value `X` (in the +form `agent=X`) to notify the client that the server is running version +`X`. The client may optionally send its own agent string by including +the `agent` capability with a value `Y` (in the form `agent=Y`) in its +request to the server (but it MUST NOT do so if the server did not +advertise the agent capability). The `X` and `Y` strings may contain any +printable ASCII characters except space (i.e., the byte range 32 < x < +127), and are typically of the form "package/version" (e.g., +"git/1.8.3.1"). The agent strings are purely informative for statistics +and debugging purposes, and MUST NOT be used to programmatically assume +the presence or absence of particular features. diff --git a/Makefile b/Makefile index c26fb93b3f..fee1d5c993 100644 --- a/Makefile +++ b/Makefile @@ -876,6 +876,7 @@ LIB_OBJS += revision.o LIB_OBJS += run-command.o LIB_OBJS += send-pack.o LIB_OBJS += sequencer.o +LIB_OBJS += serve.o LIB_OBJS += server-info.o LIB_OBJS += setup.o LIB_OBJS += sha1-array.o @@ -1009,6 +1010,7 @@ BUILTIN_OBJS += builtin/rev-parse.o BUILTIN_OBJS += builtin/revert.o BUILTIN_OBJS += builtin/rm.o BUILTIN_OBJS += builtin/send-pack.o +BUILTIN_OBJS += builtin/serve.o BUILTIN_OBJS += builtin/shortlog.o BUILTIN_OBJS += builtin/show-branch.o BUILTIN_OBJS += builtin/show-ref.o diff --git a/builtin.h b/builtin.h index f332a12574..3f3fdfc281 100644 --- a/builtin.h +++ b/builtin.h @@ -215,6 +215,7 @@ extern int cmd_rev_parse(int argc, const char **argv, const char *prefix); extern int cmd_revert(int argc, const char **argv, const char *prefix); extern int cmd_rm(int argc, const char **argv, const char *prefix); extern int cmd_send_pack(int argc, const char **argv, const char *prefix); +extern int cmd_serve(int argc, const char **argv, const char *prefix); extern int cmd_shortlog(int argc, const char **argv, const char *prefix); extern int cmd_show(int argc, const char **argv, const char *prefix); extern int cmd_show_branch(int argc, const char **argv, const char *prefix); diff --git a/builtin/serve.c b/builtin/serve.c new file mode 100644 index 0000000000..d3fd240bb3 --- /dev/null +++ b/builtin/serve.c @@ -0,0 +1,30 @@ +#include "cache.h" +#include "builtin.h" +#include "parse-options.h" +#include "serve.h" + +static char const * const serve_usage[] = { + N_("git serve []"), + NULL +}; + +int cmd_serve(int argc, const char **argv, const char *prefix) +{ + struct serve_options opts = SERVE_OPTIONS_INIT; + + struct option options[] = { + OPT_BOOL(0, "stateless-rpc", &opts.stateless_rpc, + N_("quit after a single request/response exchange")), + OPT_BOOL(0, "advertise-capabilities", &opts.advertise_capabilities, + N_("exit immediately after advertising capabilities")), + OPT_END() + }; + + /* ignore all unknown cmdline switches for now */ + argc = parse_options(argc, argv, prefix, options, serve_usage, + PARSE_OPT_KEEP_DASHDASH | + PARSE_OPT_KEEP_UNKNOWN); + serve(&opts); + + return 0; +} diff --git a/git.c b/git.c index f71073dc8d..f85d682b62 100644 --- a/git.c +++ b/git.c @@ -461,6 +461,7 @@ static struct cmd_struct commands[] = { { "revert", cmd_revert, RUN_SETUP | NEED_WORK_TREE }, { "rm", cmd_rm, RUN_SETUP }, { "send-pack", cmd_send_pack, RUN_SETUP }, + { "serve", cmd_serve, RUN_SETUP }, { "shortlog", cmd_shortlog, RUN_SETUP_GENTLY | USE_PAGER }, { "show", cmd_show, RUN_SETUP }, { "show-branch", cmd_show_branch, RUN_SETUP }, diff --git a/serve.c b/serve.c new file mode 100644 index 0000000000..7ddcba9003 --- /dev/null +++ b/serve.c @@ -0,0 +1,247 @@ +#include "cache.h" +#include "repository.h" +#include "config.h" +#include "pkt-line.h" +#include "version.h" +#include "argv-array.h" +#include "serve.h" + +static int agent_advertise(struct repository *r, + struct strbuf *value) +{ + if (value) + strbuf_addstr(value, git_user_agent_sanitized()); + return 1; +} + +struct protocol_capability { + /* + * The name of the capability. The server uses this name when + * advertising this capability, and the client uses this name to + * specify this capability. + */ + const char *name; + + /* + * Function queried to see if a capability should be advertised. + * Optionally a value can be specified by adding it to 'value'. + * If a value is added to 'value', the server will advertise this + * capability as "=" instead of "". + */ + int (*advertise)(struct repository *r, struct strbuf *value); + + /* + * Function called when a client requests the capability as a command. + * The function will be provided the capabilities requested via 'keys' + * as well as a struct packet_reader 'request' which the command should + * use to read the command specific part of the request. Every command + * MUST read until a flush packet is seen before sending a response. + * + * This field should be NULL for capabilities which are not commands. + */ + int (*command)(struct repository *r, + struct argv_array *keys, + struct packet_reader *request); +}; + +static struct protocol_capability capabilities[] = { + { "agent", agent_advertise, NULL }, +}; + +static void advertise_capabilities(void) +{ + struct strbuf capability = STRBUF_INIT; + struct strbuf value = STRBUF_INIT; + int i; + + for (i = 0; i < ARRAY_SIZE(capabilities); i++) { + struct protocol_capability *c = &capabilities[i]; + + if (c->advertise(the_repository, &value)) { + strbuf_addstr(&capability, c->name); + + if (value.len) { + strbuf_addch(&capability, '='); + strbuf_addbuf(&capability, &value); + } + + strbuf_addch(&capability, '\n'); + packet_write(1, capability.buf, capability.len); + } + + strbuf_reset(&capability); + strbuf_reset(&value); + } + + packet_flush(1); + strbuf_release(&capability); + strbuf_release(&value); +} + +static struct protocol_capability *get_capability(const char *key) +{ + int i; + + if (!key) + return NULL; + + for (i = 0; i < ARRAY_SIZE(capabilities); i++) { + struct protocol_capability *c = &capabilities[i]; + const char *out; + if (skip_prefix(key, c->name, &out) && (!*out || *out == '=')) + return c; + } + + return NULL; +} + +static int is_valid_capability(const char *key) +{ + const struct protocol_capability *c = get_capability(key); + + return c && c->advertise(the_repository, NULL); +} + +static int is_command(const char *key, struct protocol_capability **command) +{ + const char *out; + + if (skip_prefix(key, "command=", &out)) { + struct protocol_capability *cmd = get_capability(out); + + if (*command) + die("command '%s' requested after already requesting command '%s'", + out, (*command)->name); + if (!cmd || !cmd->advertise(the_repository, NULL) || !cmd->command) + die("invalid command '%s'", out); + + *command = cmd; + return 1; + } + + return 0; +} + +int has_capability(const struct argv_array *keys, const char *capability, + const char **value) +{ + int i; + for (i = 0; i < keys->argc; i++) { + const char *out; + if (skip_prefix(keys->argv[i], capability, &out) && + (!*out || *out == '=')) { + if (value) { + if (*out == '=') + out++; + *value = out; + } + return 1; + } + } + + return 0; +} + +enum request_state { + PROCESS_REQUEST_KEYS, + PROCESS_REQUEST_DONE, +}; + +static int process_request(void) +{ + enum request_state state = PROCESS_REQUEST_KEYS; + struct packet_reader reader; + struct argv_array keys = ARGV_ARRAY_INIT; + struct protocol_capability *command = NULL; + + packet_reader_init(&reader, 0, NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + /* + * Check to see if the client closed their end before sending another + * request. If so we can terminate the connection. + */ + if (packet_reader_peek(&reader) == PACKET_READ_EOF) + return 1; + reader.options = PACKET_READ_CHOMP_NEWLINE; + + while (state != PROCESS_REQUEST_DONE) { + switch (packet_reader_peek(&reader)) { + case PACKET_READ_EOF: + BUG("Should have already died when seeing EOF"); + case PACKET_READ_NORMAL: + /* collect request; a sequence of keys and values */ + if (is_command(reader.line, &command) || + is_valid_capability(reader.line)) + argv_array_push(&keys, reader.line); + else + die("unknown capability '%s'", reader.line); + + /* Consume the peeked line */ + packet_reader_read(&reader); + break; + case PACKET_READ_FLUSH: + /* + * If no command and no keys were given then the client + * wanted to terminate the connection. + */ + if (!keys.argc) + return 1; + + /* + * The flush packet isn't consume here like it is in + * the other parts of this switch statement. This is + * so that the command can read the flush packet and + * see the end of the request in the same way it would + * if command specific arguments were provided after a + * delim packet. + */ + state = PROCESS_REQUEST_DONE; + break; + case PACKET_READ_DELIM: + /* Consume the peeked line */ + packet_reader_read(&reader); + + state = PROCESS_REQUEST_DONE; + break; + } + } + + if (!command) + die("no command requested"); + + command->command(the_repository, &keys, &reader); + + argv_array_clear(&keys); + return 0; +} + +/* Main serve loop for protocol version 2 */ +void serve(struct serve_options *options) +{ + if (options->advertise_capabilities || !options->stateless_rpc) { + /* serve by default supports v2 */ + packet_write_fmt(1, "version 2\n"); + + advertise_capabilities(); + /* + * If only the list of capabilities was requested exit + * immediately after advertising capabilities + */ + if (options->advertise_capabilities) + return; + } + + /* + * If stateless-rpc was requested then exit after + * a single request/response exchange + */ + if (options->stateless_rpc) { + process_request(); + } else { + for (;;) + if (process_request()) + break; + } +} diff --git a/serve.h b/serve.h new file mode 100644 index 0000000000..fe65ba9f46 --- /dev/null +++ b/serve.h @@ -0,0 +1,15 @@ +#ifndef SERVE_H +#define SERVE_H + +struct argv_array; +extern int has_capability(const struct argv_array *keys, const char *capability, + const char **value); + +struct serve_options { + unsigned advertise_capabilities; + unsigned stateless_rpc; +}; +#define SERVE_OPTIONS_INIT { 0 } +extern void serve(struct serve_options *options); + +#endif /* SERVE_H */ diff --git a/t/t5701-git-serve.sh b/t/t5701-git-serve.sh new file mode 100755 index 0000000000..affbad097d --- /dev/null +++ b/t/t5701-git-serve.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +test_description='test git-serve and server commands' + +. ./test-lib.sh + +test_expect_success 'test capability advertisement' ' + cat >expect <<-EOF && + version 2 + agent=git/$(git version | cut -d" " -f3) + 0000 + EOF + + git serve --advertise-capabilities >out && + test-pkt-line unpack actual && + test_cmp actual expect +' + +test_expect_success 'stateless-rpc flag does not list capabilities' ' + # Empty request + test-pkt-line pack >in <<-EOF && + 0000 + EOF + git serve --stateless-rpc >out out && + test_must_be_empty out +' + +test_expect_success 'request invalid capability' ' + test-pkt-line pack >in <<-EOF && + foobar + 0000 + EOF + test_must_fail git serve --stateless-rpc 2>err in <<-EOF && + agent=git/test + 0000 + EOF + test_must_fail git serve --stateless-rpc 2>err in <<-EOF && + command=foo + agent=git/test + 0000 + EOF + test_must_fail git serve --stateless-rpc 2>err Date: Thu, 15 Mar 2018 10:31:21 -0700 Subject: connect: request remote refs using v2 Teach the client to be able to request a remote's refs using protocol v2. This is done by having a client issue a 'ls-refs' request to a v2 server. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- builtin/upload-pack.c | 10 ++-- connect.c | 138 +++++++++++++++++++++++++++++++++++++++++++++++-- connect.h | 2 + remote.h | 6 +++ t/t5702-protocol-v2.sh | 57 ++++++++++++++++++++ transport.c | 2 +- 6 files changed, 204 insertions(+), 11 deletions(-) create mode 100755 t/t5702-protocol-v2.sh (limited to 'builtin') diff --git a/builtin/upload-pack.c b/builtin/upload-pack.c index 8d53e9794b..a757df8da0 100644 --- a/builtin/upload-pack.c +++ b/builtin/upload-pack.c @@ -5,6 +5,7 @@ #include "parse-options.h" #include "protocol.h" #include "upload-pack.h" +#include "serve.h" static const char * const upload_pack_usage[] = { N_("git upload-pack [] "), @@ -16,6 +17,7 @@ int cmd_upload_pack(int argc, const char **argv, const char *prefix) const char *dir; int strict = 0; struct upload_pack_options opts = { 0 }; + struct serve_options serve_opts = SERVE_OPTIONS_INIT; struct option options[] = { OPT_BOOL(0, "stateless-rpc", &opts.stateless_rpc, N_("quit after a single request/response exchange")), @@ -48,11 +50,9 @@ int cmd_upload_pack(int argc, const char **argv, const char *prefix) switch (determine_protocol_version_server()) { case protocol_v2: - /* - * fetch support for protocol v2 has not been implemented yet, - * so ignore the request to use v2 and fallback to using v0. - */ - upload_pack(&opts); + serve_opts.advertise_capabilities = opts.advertise_refs; + serve_opts.stateless_rpc = opts.stateless_rpc; + serve(&serve_opts); break; case protocol_v1: /* diff --git a/connect.c b/connect.c index 4b89b984c4..e42d779f71 100644 --- a/connect.c +++ b/connect.c @@ -12,9 +12,11 @@ #include "sha1-array.h" #include "transport.h" #include "strbuf.h" +#include "version.h" #include "protocol.h" -static char *server_capabilities; +static char *server_capabilities_v1; +static struct argv_array server_capabilities_v2 = ARGV_ARRAY_INIT; static const char *parse_feature_value(const char *, const char *, int *); static int check_ref(const char *name, unsigned int flags) @@ -62,6 +64,33 @@ static void die_initial_contact(int unexpected) "and the repository exists.")); } +/* Checks if the server supports the capability 'c' */ +int server_supports_v2(const char *c, int die_on_error) +{ + int i; + + for (i = 0; i < server_capabilities_v2.argc; i++) { + const char *out; + if (skip_prefix(server_capabilities_v2.argv[i], c, &out) && + (!*out || *out == '=')) + return 1; + } + + if (die_on_error) + die("server doesn't support '%s'", c); + + return 0; +} + +static void process_capabilities_v2(struct packet_reader *reader) +{ + while (packet_reader_read(reader) == PACKET_READ_NORMAL) + argv_array_push(&server_capabilities_v2, reader->line); + + if (reader->status != PACKET_READ_FLUSH) + die("expected flush after capabilities"); +} + enum protocol_version discover_version(struct packet_reader *reader) { enum protocol_version version = protocol_unknown_version; @@ -84,7 +113,7 @@ enum protocol_version discover_version(struct packet_reader *reader) switch (version) { case protocol_v2: - die("support for protocol v2 not implemented yet"); + process_capabilities_v2(reader); break; case protocol_v1: /* Read the peeked version line */ @@ -128,7 +157,7 @@ reject: static void annotate_refs_with_symref_info(struct ref *ref) { struct string_list symref = STRING_LIST_INIT_DUP; - const char *feature_list = server_capabilities; + const char *feature_list = server_capabilities_v1; while (feature_list) { int len; @@ -157,7 +186,7 @@ static void process_capabilities(const char *line, int *len) int nul_location = strlen(line); if (nul_location == *len) return; - server_capabilities = xstrdup(line + nul_location + 1); + server_capabilities_v1 = xstrdup(line + nul_location + 1); *len = nul_location; } @@ -292,6 +321,105 @@ struct ref **get_remote_heads(struct packet_reader *reader, return list; } +/* Returns 1 when a valid ref has been added to `list`, 0 otherwise */ +static int process_ref_v2(const char *line, struct ref ***list) +{ + int ret = 1; + int i = 0; + struct object_id old_oid; + struct ref *ref; + struct string_list line_sections = STRING_LIST_INIT_DUP; + const char *end; + + /* + * Ref lines have a number of fields which are space deliminated. The + * first field is the OID of the ref. The second field is the ref + * name. Subsequent fields (symref-target and peeled) are optional and + * don't have a particular order. + */ + if (string_list_split(&line_sections, line, ' ', -1) < 2) { + ret = 0; + goto out; + } + + if (parse_oid_hex(line_sections.items[i++].string, &old_oid, &end) || + *end) { + ret = 0; + goto out; + } + + ref = alloc_ref(line_sections.items[i++].string); + + oidcpy(&ref->old_oid, &old_oid); + **list = ref; + *list = &ref->next; + + for (; i < line_sections.nr; i++) { + const char *arg = line_sections.items[i].string; + if (skip_prefix(arg, "symref-target:", &arg)) + ref->symref = xstrdup(arg); + + if (skip_prefix(arg, "peeled:", &arg)) { + struct object_id peeled_oid; + char *peeled_name; + struct ref *peeled; + if (parse_oid_hex(arg, &peeled_oid, &end) || *end) { + ret = 0; + goto out; + } + + peeled_name = xstrfmt("%s^{}", ref->name); + peeled = alloc_ref(peeled_name); + + oidcpy(&peeled->old_oid, &peeled_oid); + **list = peeled; + *list = &peeled->next; + + free(peeled_name); + } + } + +out: + string_list_clear(&line_sections, 0); + return ret; +} + +struct ref **get_remote_refs(int fd_out, struct packet_reader *reader, + struct ref **list, int for_push, + const struct argv_array *ref_prefixes) +{ + int i; + *list = NULL; + + if (server_supports_v2("ls-refs", 1)) + packet_write_fmt(fd_out, "command=ls-refs\n"); + + if (server_supports_v2("agent", 0)) + packet_write_fmt(fd_out, "agent=%s", git_user_agent_sanitized()); + + packet_delim(fd_out); + /* When pushing we don't want to request the peeled tags */ + if (!for_push) + packet_write_fmt(fd_out, "peel\n"); + packet_write_fmt(fd_out, "symrefs\n"); + for (i = 0; ref_prefixes && i < ref_prefixes->argc; i++) { + packet_write_fmt(fd_out, "ref-prefix %s\n", + ref_prefixes->argv[i]); + } + packet_flush(fd_out); + + /* Process response from server */ + while (packet_reader_read(reader) == PACKET_READ_NORMAL) { + if (!process_ref_v2(reader->line, &list)) + die("invalid ls-refs response: %s", reader->line); + } + + if (reader->status != PACKET_READ_FLUSH) + die("expected flush after ref listing"); + + return list; +} + static const char *parse_feature_value(const char *feature_list, const char *feature, int *lenp) { int len; @@ -336,7 +464,7 @@ int parse_feature_request(const char *feature_list, const char *feature) const char *server_feature_value(const char *feature, int *len) { - return parse_feature_value(server_capabilities, feature, len); + return parse_feature_value(server_capabilities_v1, feature, len); } int server_supports(const char *feature) diff --git a/connect.h b/connect.h index cdb8979dce..8898d44952 100644 --- a/connect.h +++ b/connect.h @@ -16,4 +16,6 @@ extern int url_is_local_not_ssh(const char *url); struct packet_reader; extern enum protocol_version discover_version(struct packet_reader *reader); +extern int server_supports_v2(const char *c, int die_on_error); + #endif diff --git a/remote.h b/remote.h index 2016461df9..368ba221cc 100644 --- a/remote.h +++ b/remote.h @@ -151,11 +151,17 @@ void free_refs(struct ref *ref); struct oid_array; struct packet_reader; +struct argv_array; extern struct ref **get_remote_heads(struct packet_reader *reader, struct ref **list, unsigned int flags, struct oid_array *extra_have, struct oid_array *shallow_points); +/* Used for protocol v2 in order to retrieve refs from a remote */ +extern struct ref **get_remote_refs(int fd_out, struct packet_reader *reader, + struct ref **list, int for_push, + const struct argv_array *ref_prefixes); + int resolve_remote_symref(struct ref *ref, struct ref *list); int ref_newer(const struct object_id *new_oid, const struct object_id *old_oid); diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh new file mode 100755 index 0000000000..dc5f813beb --- /dev/null +++ b/t/t5702-protocol-v2.sh @@ -0,0 +1,57 @@ +#!/bin/sh + +test_description='test git wire-protocol version 2' + +TEST_NO_CREATE_REPO=1 + +. ./test-lib.sh + +# Test protocol v2 with 'git://' transport +# +. "$TEST_DIRECTORY"/lib-git-daemon.sh +start_git_daemon --export-all --enable=receive-pack +daemon_parent=$GIT_DAEMON_DOCUMENT_ROOT_PATH/parent + +test_expect_success 'create repo to be served by git-daemon' ' + git init "$daemon_parent" && + test_commit -C "$daemon_parent" one +' + +test_expect_success 'list refs with git:// using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \ + ls-remote --symref "$GIT_DAEMON_URL/parent" >actual && + + # Client requested to use protocol v2 + grep "git> .*\\\0\\\0version=2\\\0$" log && + # Server responded using protocol v2 + grep "git< version 2" log && + + git ls-remote --symref "$GIT_DAEMON_URL/parent" >expect && + test_cmp actual expect +' + +stop_git_daemon + +# Test protocol v2 with 'file://' transport +# +test_expect_success 'create repo to be served by file:// transport' ' + git init file_parent && + test_commit -C file_parent one +' + +test_expect_success 'list refs with file:// using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \ + ls-remote --symref "file://$(pwd)/file_parent" >actual && + + # Server responded using protocol v2 + grep "git< version 2" log && + + git ls-remote --symref "file://$(pwd)/file_parent" >expect && + test_cmp actual expect +' + +test_done diff --git a/transport.c b/transport.c index 83d9dd1df6..ffc6b2614f 100644 --- a/transport.c +++ b/transport.c @@ -204,7 +204,7 @@ static struct ref *get_refs_via_connect(struct transport *transport, int for_pus data->version = discover_version(&reader); switch (data->version) { case protocol_v2: - die("support for protocol v2 not implemented yet"); + get_remote_refs(data->fd[1], &reader, &refs, for_push, NULL); break; case protocol_v1: case protocol_v0: -- cgit v1.3-5-g9baa From 1af8ae1cfa44a3f3c3f1efb7f9ebe68bc0ce7578 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:23 -0700 Subject: transport: convert transport_get_remote_refs to take a list of ref prefixes Teach transport_get_remote_refs() to accept a list of ref prefixes, which will be sent to the server for use in filtering when using protocol v2. (This list will be ignored when not using protocol v2.) Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- builtin/clone.c | 2 +- builtin/fetch.c | 4 ++-- builtin/ls-remote.c | 2 +- builtin/remote.c | 2 +- transport.c | 7 +++++-- transport.h | 12 +++++++++++- 6 files changed, 21 insertions(+), 8 deletions(-) (limited to 'builtin') diff --git a/builtin/clone.c b/builtin/clone.c index 2da71db107..4db3079acc 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -1104,7 +1104,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix) if (transport->smart_options && !deepen) transport->smart_options->check_self_contained_and_connected = 1; - refs = transport_get_remote_refs(transport); + refs = transport_get_remote_refs(transport, NULL); if (refs) { mapped_refs = wanted_peer_refs(refs, refspec); diff --git a/builtin/fetch.c b/builtin/fetch.c index 7bbcd26faf..850382f559 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -250,7 +250,7 @@ static void find_non_local_tags(struct transport *transport, struct string_list_item *item = NULL; for_each_ref(add_existing, &existing_refs); - for (ref = transport_get_remote_refs(transport); ref; ref = ref->next) { + for (ref = transport_get_remote_refs(transport, NULL); ref; ref = ref->next) { if (!starts_with(ref->name, "refs/tags/")) continue; @@ -336,7 +336,7 @@ static struct ref *get_ref_map(struct transport *transport, /* opportunistically-updated references: */ struct ref *orefs = NULL, **oref_tail = &orefs; - const struct ref *remote_refs = transport_get_remote_refs(transport); + const struct ref *remote_refs = transport_get_remote_refs(transport, NULL); if (refspec_count) { struct refspec *fetch_refspec; diff --git a/builtin/ls-remote.c b/builtin/ls-remote.c index c4be98ab9e..c6e9847c5c 100644 --- a/builtin/ls-remote.c +++ b/builtin/ls-remote.c @@ -96,7 +96,7 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) if (uploadpack != NULL) transport_set_option(transport, TRANS_OPT_UPLOADPACK, uploadpack); - ref = transport_get_remote_refs(transport); + ref = transport_get_remote_refs(transport, NULL); if (transport_disconnect(transport)) return 1; diff --git a/builtin/remote.c b/builtin/remote.c index d95bf904c3..d0b6ff6e29 100644 --- a/builtin/remote.c +++ b/builtin/remote.c @@ -862,7 +862,7 @@ static int get_remote_ref_states(const char *name, if (query) { transport = transport_get(states->remote, states->remote->url_nr > 0 ? states->remote->url[0] : NULL); - remote_refs = transport_get_remote_refs(transport); + remote_refs = transport_get_remote_refs(transport, NULL); transport_disconnect(transport); states->queried = 1; diff --git a/transport.c b/transport.c index 2e68010dd0..3f130518d2 100644 --- a/transport.c +++ b/transport.c @@ -1138,10 +1138,13 @@ int transport_push(struct transport *transport, return 1; } -const struct ref *transport_get_remote_refs(struct transport *transport) +const struct ref *transport_get_remote_refs(struct transport *transport, + const struct argv_array *ref_prefixes) { if (!transport->got_remote_refs) { - transport->remote_refs = transport->vtable->get_refs_list(transport, 0, NULL); + transport->remote_refs = + transport->vtable->get_refs_list(transport, 0, + ref_prefixes); transport->got_remote_refs = 1; } diff --git a/transport.h b/transport.h index 731c78b679..83992a4257 100644 --- a/transport.h +++ b/transport.h @@ -178,7 +178,17 @@ int transport_push(struct transport *connection, int refspec_nr, const char **refspec, int flags, unsigned int * reject_reasons); -const struct ref *transport_get_remote_refs(struct transport *transport); +/* + * Retrieve refs from a remote. + * + * Optionally a list of ref prefixes can be provided which can be sent to the + * server (when communicating using protocol v2) to enable it to limit the ref + * advertisement. Since ref filtering is done on the server's end (and only + * when using protocol v2), this can return refs which don't match the provided + * ref_prefixes. + */ +const struct ref *transport_get_remote_refs(struct transport *transport, + const struct argv_array *ref_prefixes); int transport_fetch_refs(struct transport *transport, struct ref *refs); void transport_unlock_pack(struct transport *transport); -- cgit v1.3-5-g9baa From b4be74105febef5c6235e1e2f1e468e76166cad8 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:24 -0700 Subject: ls-remote: pass ref prefixes when requesting a remote's refs Construct an argv_array of ref prefixes based on the patterns supplied via the command line and pass them to 'transport_get_remote_refs()' to be used when communicating protocol v2 so that the server can limit the ref advertisement based on those prefixes. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- builtin/ls-remote.c | 15 +++++++++++++-- refs.c | 14 ++++++++++++++ refs.h | 7 +++++++ t/t5702-protocol-v2.sh | 26 ++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) (limited to 'builtin') diff --git a/builtin/ls-remote.c b/builtin/ls-remote.c index c6e9847c5c..4276bf97d5 100644 --- a/builtin/ls-remote.c +++ b/builtin/ls-remote.c @@ -2,6 +2,7 @@ #include "cache.h" #include "transport.h" #include "remote.h" +#include "refs.h" static const char * const ls_remote_usage[] = { N_("git ls-remote [--heads] [--tags] [--refs] [--upload-pack=]\n" @@ -43,6 +44,7 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) int show_symref_target = 0; const char *uploadpack = NULL; const char **pattern = NULL; + struct argv_array ref_prefixes = ARGV_ARRAY_INIT; struct remote *remote; struct transport *transport; @@ -74,8 +76,17 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) if (argc > 1) { int i; pattern = xcalloc(argc, sizeof(const char *)); - for (i = 1; i < argc; i++) + for (i = 1; i < argc; i++) { + const char *glob; pattern[i - 1] = xstrfmt("*/%s", argv[i]); + + glob = strchr(argv[i], '*'); + if (glob) + argv_array_pushf(&ref_prefixes, "%.*s", + (int)(glob - argv[i]), argv[i]); + else + expand_ref_prefix(&ref_prefixes, argv[i]); + } } remote = remote_get(dest); @@ -96,7 +107,7 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) if (uploadpack != NULL) transport_set_option(transport, TRANS_OPT_UPLOADPACK, uploadpack); - ref = transport_get_remote_refs(transport, NULL); + ref = transport_get_remote_refs(transport, &ref_prefixes); if (transport_disconnect(transport)) return 1; diff --git a/refs.c b/refs.c index 20ba82b434..cefbad2076 100644 --- a/refs.c +++ b/refs.c @@ -13,6 +13,7 @@ #include "tag.h" #include "submodule.h" #include "worktree.h" +#include "argv-array.h" /* * List of all available backends @@ -501,6 +502,19 @@ int refname_match(const char *abbrev_name, const char *full_name) return 0; } +/* + * Given a 'prefix' expand it by the rules in 'ref_rev_parse_rules' and add + * the results to 'prefixes' + */ +void expand_ref_prefix(struct argv_array *prefixes, const char *prefix) +{ + const char **p; + int len = strlen(prefix); + + for (p = ref_rev_parse_rules; *p; p++) + argv_array_pushf(prefixes, *p, len, prefix); +} + /* * *string and *len will only be substituted, and *string returned (for * later free()ing) if the string passed in is a magic short-hand form diff --git a/refs.h b/refs.h index 01be5ae32f..93b6dce944 100644 --- a/refs.h +++ b/refs.h @@ -139,6 +139,13 @@ int resolve_gitlink_ref(const char *submodule, const char *refname, */ int refname_match(const char *abbrev_name, const char *full_name); +/* + * Given a 'prefix' expand it by the rules in 'ref_rev_parse_rules' and add + * the results to 'prefixes' + */ +struct argv_array; +void expand_ref_prefix(struct argv_array *prefixes, const char *prefix); + int expand_ref(const char *str, int len, struct object_id *oid, char **ref); int dwim_ref(const char *str, int len, struct object_id *oid, char **ref); int dwim_log(const char *str, int len, struct object_id *oid, char **ref); diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh index dc5f813beb..562610fd25 100755 --- a/t/t5702-protocol-v2.sh +++ b/t/t5702-protocol-v2.sh @@ -32,6 +32,19 @@ test_expect_success 'list refs with git:// using protocol v2' ' test_cmp actual expect ' +test_expect_success 'ref advertisment is filtered with ls-remote using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \ + ls-remote "$GIT_DAEMON_URL/parent" master >actual && + + cat >expect <<-EOF && + $(git -C "$daemon_parent" rev-parse refs/heads/master)$(printf "\t")refs/heads/master + EOF + + test_cmp actual expect +' + stop_git_daemon # Test protocol v2 with 'file://' transport @@ -54,4 +67,17 @@ test_expect_success 'list refs with file:// using protocol v2' ' test_cmp actual expect ' +test_expect_success 'ref advertisment is filtered with ls-remote using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \ + ls-remote "file://$(pwd)/file_parent" master >actual && + + cat >expect <<-EOF && + $(git -C file_parent rev-parse refs/heads/master)$(printf "\t")refs/heads/master + EOF + + test_cmp actual expect +' + test_done -- cgit v1.3-5-g9baa From 230d7dd3917b80d94bf8d522b220d587634f1885 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:25 -0700 Subject: fetch: pass ref prefixes when fetching Construct a list of ref prefixes to be passed to 'transport_get_remote_refs()' from the refspec to be used during the fetch. This list of ref prefixes will be used to allow the server to filter the ref advertisement when communicating using protocol v2. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- builtin/fetch.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'builtin') diff --git a/builtin/fetch.c b/builtin/fetch.c index 850382f559..8258bbf950 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -332,11 +332,28 @@ static struct ref *get_ref_map(struct transport *transport, struct ref *rm; struct ref *ref_map = NULL; struct ref **tail = &ref_map; + struct argv_array ref_prefixes = ARGV_ARRAY_INIT; /* opportunistically-updated references: */ struct ref *orefs = NULL, **oref_tail = &orefs; - const struct ref *remote_refs = transport_get_remote_refs(transport, NULL); + const struct ref *remote_refs; + + for (i = 0; i < refspec_count; i++) { + if (!refspecs[i].exact_sha1) { + const char *glob = strchr(refspecs[i].src, '*'); + if (glob) + argv_array_pushf(&ref_prefixes, "%.*s", + (int)(glob - refspecs[i].src), + refspecs[i].src); + else + expand_ref_prefix(&ref_prefixes, refspecs[i].src); + } + } + + remote_refs = transport_get_remote_refs(transport, &ref_prefixes); + + argv_array_clear(&ref_prefixes); if (refspec_count) { struct refspec *fetch_refspec; -- cgit v1.3-5-g9baa From 685fbd32916f3e94bc89aa14e8fdce835b06f801 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 15 Mar 2018 10:31:28 -0700 Subject: fetch-pack: perform a fetch using v2 When communicating with a v2 server, perform a fetch by requesting the 'fetch' command. Signed-off-by: Brandon Williams Signed-off-by: Junio C Hamano --- Documentation/technical/protocol-v2.txt | 67 +++++++- builtin/fetch-pack.c | 2 +- fetch-pack.c | 270 +++++++++++++++++++++++++++++++- fetch-pack.h | 4 +- serve.c | 2 +- t/t5701-git-serve.sh | 2 +- t/t5702-protocol-v2.sh | 97 ++++++++++++ transport.c | 7 +- upload-pack.c | 141 ++++++++++++----- upload-pack.h | 4 + 10 files changed, 548 insertions(+), 48 deletions(-) (limited to 'builtin') diff --git a/Documentation/technical/protocol-v2.txt b/Documentation/technical/protocol-v2.txt index 307dc1489b..4f7f251569 100644 --- a/Documentation/technical/protocol-v2.txt +++ b/Documentation/technical/protocol-v2.txt @@ -255,12 +255,43 @@ A `fetch` request can take the following arguments: to its base by position in pack rather than by an oid. That is, they can read OBJ_OFS_DELTA (ake type 6) in a packfile. + shallow + A client must notify the server of all commits for which it only + has shallow copies (meaning that it doesn't have the parents of + a commit) by supplying a 'shallow ' line for each such + object so that the server is aware of the limitations of the + client's history. This is so that the server is aware that the + client may not have all objects reachable from such commits. + + deepen + Requests that the fetch/clone should be shallow having a commit + depth of relative to the remote side. + + deepen-relative + Requests that the semantics of the "deepen" command be changed + to indicate that the depth requested is relative to the client's + current shallow boundary, instead of relative to the requested + commits. + + deepen-since + Requests that the shallow clone/fetch should be cut at a + specific time, instead of depth. Internally it's equivalent to + doing "git rev-list --max-age=". Cannot be used with + "deepen". + + deepen-not + Requests that the shallow clone/fetch should be cut at a + specific revision specified by '', instead of a depth. + Internally it's equivalent of doing "git rev-list --not ". + Cannot be used with "deepen", but can be used with + "deepen-since". + The response of `fetch` is broken into a number of sections separated by delimiter packets (0001), with each section beginning with its section header. output = *section - section = (acknowledgments | packfile) + section = (acknowledgments | shallow-info | packfile) (flush-pkt | delim-pkt) acknowledgments = PKT-LINE("acknowledgments" LF) @@ -270,6 +301,11 @@ header. nak = PKT-LINE("NAK" LF) ack = PKT-LINE("ACK" SP obj-id LF) + shallow-info = PKT-LINE("shallow-info" LF) + *PKT-LINE((shallow | unshallow) LF) + shallow = "shallow" SP obj-id + unshallow = "unshallow" SP obj-id + packfile = PKT-LINE("packfile" LF) *PKT-LINE(%x01-03 *%x00-ff) @@ -301,6 +337,35 @@ header. determined the objects it plans to send to the client and no further negotiation is needed. + shallow-info section + If the client has requested a shallow fetch/clone, a shallow + client requests a fetch or the server is shallow then the + server's response may include a shallow-info section. The + shallow-info section will be included if (due to one of the + above conditions) the server needs to inform the client of any + shallow boundaries or adjustments to the clients already + existing shallow boundaries. + + * Always begins with the section header "shallow-info" + + * If a positive depth is requested, the server will compute the + set of commits which are no deeper than the desired depth. + + * The server sends a "shallow obj-id" line for each commit whose + parents will not be sent in the following packfile. + + * The server sends an "unshallow obj-id" line for each commit + which the client has indicated is shallow, but is no longer + shallow as a result of the fetch (due to its parents being + sent in the following packfile). + + * The server MUST NOT send any "unshallow" lines for anything + which the client has not indicated was shallow as a part of + its request. + + * This section is only included if a packfile section is also + included in the response. + packfile section * This section is only included if the client has sent 'want' lines in its request and either requested that no more diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index b2374ddbbf..f9d7d0b5a5 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -212,7 +212,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) } ref = fetch_pack(&args, fd, conn, ref, dest, sought, nr_sought, - &shallow, pack_lockfile_ptr); + &shallow, pack_lockfile_ptr, protocol_v0); if (pack_lockfile) { printf("lock %s\n", pack_lockfile); fflush(stdout); diff --git a/fetch-pack.c b/fetch-pack.c index 9f6b07ad91..dffcfd66a5 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -303,9 +303,9 @@ static void insert_one_alternate_object(struct object *obj) #define PIPESAFE_FLUSH 32 #define LARGE_FLUSH 16384 -static int next_flush(struct fetch_pack_args *args, int count) +static int next_flush(int stateless_rpc, int count) { - if (args->stateless_rpc) { + if (stateless_rpc) { if (count < LARGE_FLUSH) count <<= 1; else @@ -461,7 +461,7 @@ static int find_common(struct fetch_pack_args *args, send_request(args, fd[1], &req_buf); strbuf_setlen(&req_buf, state_len); flushes++; - flush_at = next_flush(args, count); + flush_at = next_flush(args->stateless_rpc, count); /* * We keep one window "ahead" of the other side, and @@ -1008,6 +1008,259 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, return ref; } +static void add_wants(const struct ref *wants, struct strbuf *req_buf) +{ + for ( ; wants ; wants = wants->next) { + const struct object_id *remote = &wants->old_oid; + const char *remote_hex; + struct object *o; + + /* + * If that object is complete (i.e. it is an ancestor of a + * local ref), we tell them we have it but do not have to + * tell them about its ancestors, which they already know + * about. + * + * We use lookup_object here because we are only + * interested in the case we *know* the object is + * reachable and we have already scanned it. + */ + if (((o = lookup_object(remote->hash)) != NULL) && + (o->flags & COMPLETE)) { + continue; + } + + remote_hex = oid_to_hex(remote); + packet_buf_write(req_buf, "want %s\n", remote_hex); + } +} + +static void add_common(struct strbuf *req_buf, struct oidset *common) +{ + struct oidset_iter iter; + const struct object_id *oid; + oidset_iter_init(common, &iter); + + while ((oid = oidset_iter_next(&iter))) { + packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid)); + } +} + +static int add_haves(struct strbuf *req_buf, int *haves_to_send, int *in_vain) +{ + int ret = 0; + int haves_added = 0; + const struct object_id *oid; + + while ((oid = get_rev())) { + packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid)); + if (++haves_added >= *haves_to_send) + break; + } + + *in_vain += haves_added; + if (!haves_added || *in_vain >= MAX_IN_VAIN) { + /* Send Done */ + packet_buf_write(req_buf, "done\n"); + ret = 1; + } + + /* Increase haves to send on next round */ + *haves_to_send = next_flush(1, *haves_to_send); + + return ret; +} + +static int send_fetch_request(int fd_out, const struct fetch_pack_args *args, + const struct ref *wants, struct oidset *common, + int *haves_to_send, int *in_vain) +{ + int ret = 0; + struct strbuf req_buf = STRBUF_INIT; + + if (server_supports_v2("fetch", 1)) + packet_buf_write(&req_buf, "command=fetch"); + if (server_supports_v2("agent", 0)) + packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized()); + + packet_buf_delim(&req_buf); + if (args->use_thin_pack) + packet_buf_write(&req_buf, "thin-pack"); + if (args->no_progress) + packet_buf_write(&req_buf, "no-progress"); + if (args->include_tag) + packet_buf_write(&req_buf, "include-tag"); + if (prefer_ofs_delta) + packet_buf_write(&req_buf, "ofs-delta"); + + /* add wants */ + add_wants(wants, &req_buf); + + /* Add all of the common commits we've found in previous rounds */ + add_common(&req_buf, common); + + /* Add initial haves */ + ret = add_haves(&req_buf, haves_to_send, in_vain); + + /* Send request */ + packet_buf_flush(&req_buf); + write_or_die(fd_out, req_buf.buf, req_buf.len); + + strbuf_release(&req_buf); + return ret; +} + +/* + * Processes a section header in a server's response and checks if it matches + * `section`. If the value of `peek` is 1, the header line will be peeked (and + * not consumed); if 0, the line will be consumed and the function will die if + * the section header doesn't match what was expected. + */ +static int process_section_header(struct packet_reader *reader, + const char *section, int peek) +{ + int ret; + + if (packet_reader_peek(reader) != PACKET_READ_NORMAL) + die("error reading packet"); + + ret = !strcmp(reader->line, section); + + if (!peek) { + if (!ret) + die("expected '%s', received '%s'", + section, reader->line); + packet_reader_read(reader); + } + + return ret; +} + +static int process_acks(struct packet_reader *reader, struct oidset *common) +{ + /* received */ + int received_ready = 0; + int received_ack = 0; + + process_section_header(reader, "acknowledgments", 0); + while (packet_reader_read(reader) == PACKET_READ_NORMAL) { + const char *arg; + + if (!strcmp(reader->line, "NAK")) + continue; + + if (skip_prefix(reader->line, "ACK ", &arg)) { + struct object_id oid; + if (!get_oid_hex(arg, &oid)) { + struct commit *commit; + oidset_insert(common, &oid); + commit = lookup_commit(&oid); + mark_common(commit, 0, 1); + } + continue; + } + + if (!strcmp(reader->line, "ready")) { + clear_prio_queue(&rev_list); + received_ready = 1; + continue; + } + + die("unexpected acknowledgment line: '%s'", reader->line); + } + + if (reader->status != PACKET_READ_FLUSH && + reader->status != PACKET_READ_DELIM) + die("error processing acks: %d", reader->status); + + /* return 0 if no common, 1 if there are common, or 2 if ready */ + return received_ready ? 2 : (received_ack ? 1 : 0); +} + +enum fetch_state { + FETCH_CHECK_LOCAL = 0, + FETCH_SEND_REQUEST, + FETCH_PROCESS_ACKS, + FETCH_GET_PACK, + FETCH_DONE, +}; + +static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, + int fd[2], + const struct ref *orig_ref, + struct ref **sought, int nr_sought, + char **pack_lockfile) +{ + struct ref *ref = copy_ref_list(orig_ref); + enum fetch_state state = FETCH_CHECK_LOCAL; + struct oidset common = OIDSET_INIT; + struct packet_reader reader; + int in_vain = 0; + int haves_to_send = INITIAL_FLUSH; + packet_reader_init(&reader, fd[0], NULL, 0, + PACKET_READ_CHOMP_NEWLINE); + + while (state != FETCH_DONE) { + switch (state) { + case FETCH_CHECK_LOCAL: + sort_ref_list(&ref, ref_compare_name); + QSORT(sought, nr_sought, cmp_ref_by_name); + + /* v2 supports these by default */ + allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1; + use_sideband = 2; + + if (marked) + for_each_ref(clear_marks, NULL); + marked = 1; + + for_each_ref(rev_list_insert_ref_oid, NULL); + for_each_cached_alternate(insert_one_alternate_object); + + /* Filter 'ref' by 'sought' and those that aren't local */ + if (everything_local(args, &ref, sought, nr_sought)) + state = FETCH_DONE; + else + state = FETCH_SEND_REQUEST; + break; + case FETCH_SEND_REQUEST: + if (send_fetch_request(fd[1], args, ref, &common, + &haves_to_send, &in_vain)) + state = FETCH_GET_PACK; + else + state = FETCH_PROCESS_ACKS; + break; + case FETCH_PROCESS_ACKS: + /* Process ACKs/NAKs */ + switch (process_acks(&reader, &common)) { + case 2: + state = FETCH_GET_PACK; + break; + case 1: + in_vain = 0; + /* fallthrough */ + default: + state = FETCH_SEND_REQUEST; + break; + } + break; + case FETCH_GET_PACK: + /* get the pack */ + process_section_header(&reader, "packfile", 0); + if (get_pack(args, fd, pack_lockfile)) + die(_("git fetch-pack: fetch failed.")); + + state = FETCH_DONE; + break; + case FETCH_DONE: + continue; + } + } + + oidset_clear(&common); + return ref; +} + static void fetch_pack_config(void) { git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit); @@ -1153,7 +1406,8 @@ struct ref *fetch_pack(struct fetch_pack_args *args, const char *dest, struct ref **sought, int nr_sought, struct oid_array *shallow, - char **pack_lockfile) + char **pack_lockfile, + enum protocol_version version) { struct ref *ref_cpy; struct shallow_info si; @@ -1167,8 +1421,12 @@ struct ref *fetch_pack(struct fetch_pack_args *args, die(_("no matching remote head")); } prepare_shallow_info(&si, shallow); - ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, - &si, pack_lockfile); + if (version == protocol_v2) + ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought, + pack_lockfile); + else + ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, + &si, pack_lockfile); reprepare_packed_git(); update_shallow(args, sought, nr_sought, &si); clear_shallow_info(&si); diff --git a/fetch-pack.h b/fetch-pack.h index b6aeb43a8e..7afca73056 100644 --- a/fetch-pack.h +++ b/fetch-pack.h @@ -3,6 +3,7 @@ #include "string-list.h" #include "run-command.h" +#include "protocol.h" struct oid_array; @@ -43,7 +44,8 @@ struct ref *fetch_pack(struct fetch_pack_args *args, struct ref **sought, int nr_sought, struct oid_array *shallow, - char **pack_lockfile); + char **pack_lockfile, + enum protocol_version version); /* * Print an appropriate error message for each sought ref that wasn't diff --git a/serve.c b/serve.c index 3692dda801..a5a7b2f7dd 100644 --- a/serve.c +++ b/serve.c @@ -55,7 +55,7 @@ struct protocol_capability { static struct protocol_capability capabilities[] = { { "agent", agent_advertise, NULL }, { "ls-refs", always_advertise, ls_refs }, - { "fetch", always_advertise, upload_pack_v2 }, + { "fetch", upload_pack_advertise, upload_pack_v2 }, }; static void advertise_capabilities(void) diff --git a/t/t5701-git-serve.sh b/t/t5701-git-serve.sh index 888a700443..72d7bc5628 100755 --- a/t/t5701-git-serve.sh +++ b/t/t5701-git-serve.sh @@ -9,7 +9,7 @@ test_expect_success 'test capability advertisement' ' version 2 agent=git/$(git version | cut -d" " -f3) ls-refs - fetch + fetch=shallow 0000 EOF diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh index 562610fd25..4365ac2736 100755 --- a/t/t5702-protocol-v2.sh +++ b/t/t5702-protocol-v2.sh @@ -45,6 +45,56 @@ test_expect_success 'ref advertisment is filtered with ls-remote using protocol test_cmp actual expect ' +test_expect_success 'clone with git:// using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \ + clone "$GIT_DAEMON_URL/parent" daemon_child && + + git -C daemon_child log -1 --format=%s >actual && + git -C "$daemon_parent" log -1 --format=%s >expect && + test_cmp expect actual && + + # Client requested to use protocol v2 + grep "clone> .*\\\0\\\0version=2\\\0$" log && + # Server responded using protocol v2 + grep "clone< version 2" log +' + +test_expect_success 'fetch with git:// using protocol v2' ' + test_when_finished "rm -f log" && + + test_commit -C "$daemon_parent" two && + + GIT_TRACE_PACKET="$(pwd)/log" git -C daemon_child -c protocol.version=2 \ + fetch && + + git -C daemon_child log -1 --format=%s origin/master >actual && + git -C "$daemon_parent" log -1 --format=%s >expect && + test_cmp expect actual && + + # Client requested to use protocol v2 + grep "fetch> .*\\\0\\\0version=2\\\0$" log && + # Server responded using protocol v2 + grep "fetch< version 2" log +' + +test_expect_success 'pull with git:// using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -C daemon_child -c protocol.version=2 \ + pull && + + git -C daemon_child log -1 --format=%s >actual && + git -C "$daemon_parent" log -1 --format=%s >expect && + test_cmp expect actual && + + # Client requested to use protocol v2 + grep "fetch> .*\\\0\\\0version=2\\\0$" log && + # Server responded using protocol v2 + grep "fetch< version 2" log +' + stop_git_daemon # Test protocol v2 with 'file://' transport @@ -80,4 +130,51 @@ test_expect_success 'ref advertisment is filtered with ls-remote using protocol test_cmp actual expect ' +test_expect_success 'clone with file:// using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \ + clone "file://$(pwd)/file_parent" file_child && + + git -C file_child log -1 --format=%s >actual && + git -C file_parent log -1 --format=%s >expect && + test_cmp expect actual && + + # Server responded using protocol v2 + grep "clone< version 2" log +' + +test_expect_success 'fetch with file:// using protocol v2' ' + test_when_finished "rm -f log" && + + test_commit -C file_parent two && + + GIT_TRACE_PACKET="$(pwd)/log" git -C file_child -c protocol.version=2 \ + fetch origin && + + git -C file_child log -1 --format=%s origin/master >actual && + git -C file_parent log -1 --format=%s >expect && + test_cmp expect actual && + + # Server responded using protocol v2 + grep "fetch< version 2" log +' + +test_expect_success 'ref advertisment is filtered during fetch using protocol v2' ' + test_when_finished "rm -f log" && + + test_commit -C file_parent three && + + GIT_TRACE_PACKET="$(pwd)/log" git -C file_child -c protocol.version=2 \ + fetch origin master && + + git -C file_child log -1 --format=%s origin/master >actual && + git -C file_parent log -1 --format=%s >expect && + test_cmp expect actual && + + ! grep "refs/tags/one" log && + ! grep "refs/tags/two" log && + ! grep "refs/tags/three" log +' + test_done diff --git a/transport.c b/transport.c index 57bdbb59bc..9152a45b38 100644 --- a/transport.c +++ b/transport.c @@ -258,14 +258,17 @@ static int fetch_refs_via_pack(struct transport *transport, switch (data->version) { case protocol_v2: - die("support for protocol v2 not implemented yet"); + refs = fetch_pack(&args, data->fd, data->conn, + refs_tmp ? refs_tmp : transport->remote_refs, + dest, to_fetch, nr_heads, &data->shallow, + &transport->pack_lockfile, data->version); break; case protocol_v1: case protocol_v0: refs = fetch_pack(&args, data->fd, data->conn, refs_tmp ? refs_tmp : transport->remote_refs, dest, to_fetch, nr_heads, &data->shallow, - &transport->pack_lockfile); + &transport->pack_lockfile, data->version); break; case protocol_unknown_version: BUG("unknown protocol version"); diff --git a/upload-pack.c b/upload-pack.c index 198e9f046b..4c9428c2db 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -710,7 +710,6 @@ static void deepen(int depth, int deepen_relative, } send_unshallow(shallows); - packet_flush(1); } static void deepen_by_rev_list(int ac, const char **av, @@ -722,7 +721,53 @@ static void deepen_by_rev_list(int ac, const char **av, send_shallow(result); free_commit_list(result); send_unshallow(shallows); - packet_flush(1); +} + +/* Returns 1 if a shallow list is sent or 0 otherwise */ +static int send_shallow_list(int depth, int deepen_rev_list, + timestamp_t deepen_since, + struct string_list *deepen_not, + struct object_array *shallows) +{ + int ret = 0; + + if (depth > 0 && deepen_rev_list) + die("git upload-pack: deepen and deepen-since (or deepen-not) cannot be used together"); + if (depth > 0) { + deepen(depth, deepen_relative, shallows); + ret = 1; + } else if (deepen_rev_list) { + struct argv_array av = ARGV_ARRAY_INIT; + int i; + + argv_array_push(&av, "rev-list"); + if (deepen_since) + argv_array_pushf(&av, "--max-age=%"PRItime, deepen_since); + if (deepen_not->nr) { + argv_array_push(&av, "--not"); + for (i = 0; i < deepen_not->nr; i++) { + struct string_list_item *s = deepen_not->items + i; + argv_array_push(&av, s->string); + } + argv_array_push(&av, "--not"); + } + for (i = 0; i < want_obj.nr; i++) { + struct object *o = want_obj.objects[i].item; + argv_array_push(&av, oid_to_hex(&o->oid)); + } + deepen_by_rev_list(av.argc, av.argv, shallows); + argv_array_clear(&av); + ret = 1; + } else { + if (shallows->nr > 0) { + int i; + for (i = 0; i < shallows->nr; i++) + register_shallow(&shallows->objects[i].item->oid); + } + } + + shallow_nr += shallows->nr; + return ret; } static int process_shallow(const char *line, struct object_array *shallows) @@ -884,40 +929,10 @@ static void receive_needs(void) if (depth == 0 && !deepen_rev_list && shallows.nr == 0) return; - if (depth > 0 && deepen_rev_list) - die("git upload-pack: deepen and deepen-since (or deepen-not) cannot be used together"); - if (depth > 0) - deepen(depth, deepen_relative, &shallows); - else if (deepen_rev_list) { - struct argv_array av = ARGV_ARRAY_INIT; - int i; - argv_array_push(&av, "rev-list"); - if (deepen_since) - argv_array_pushf(&av, "--max-age=%"PRItime, deepen_since); - if (deepen_not.nr) { - argv_array_push(&av, "--not"); - for (i = 0; i < deepen_not.nr; i++) { - struct string_list_item *s = deepen_not.items + i; - argv_array_push(&av, s->string); - } - argv_array_push(&av, "--not"); - } - for (i = 0; i < want_obj.nr; i++) { - struct object *o = want_obj.objects[i].item; - argv_array_push(&av, oid_to_hex(&o->oid)); - } - deepen_by_rev_list(av.argc, av.argv, &shallows); - argv_array_clear(&av); - } - else - if (shallows.nr > 0) { - int i; - for (i = 0; i < shallows.nr; i++) - register_shallow(&shallows.objects[i].item->oid); - } - - shallow_nr += shallows.nr; + if (send_shallow_list(depth, deepen_rev_list, deepen_since, + &deepen_not, &shallows)) + packet_flush(1); object_array_clear(&shallows); } @@ -1071,6 +1086,13 @@ struct upload_pack_data { struct object_array wants; struct oid_array haves; + struct object_array shallows; + struct string_list deepen_not; + int depth; + timestamp_t deepen_since; + int deepen_rev_list; + int deepen_relative; + unsigned stateless_rpc : 1; unsigned use_thin_pack : 1; @@ -1084,16 +1106,22 @@ static void upload_pack_data_init(struct upload_pack_data *data) { struct object_array wants = OBJECT_ARRAY_INIT; struct oid_array haves = OID_ARRAY_INIT; + struct object_array shallows = OBJECT_ARRAY_INIT; + struct string_list deepen_not = STRING_LIST_INIT_DUP; memset(data, 0, sizeof(*data)); data->wants = wants; data->haves = haves; + data->shallows = shallows; + data->deepen_not = deepen_not; } static void upload_pack_data_clear(struct upload_pack_data *data) { object_array_clear(&data->wants); oid_array_clear(&data->haves); + object_array_clear(&data->shallows); + string_list_clear(&data->deepen_not, 0); } static int parse_want(const char *line) @@ -1177,6 +1205,22 @@ static void process_args(struct packet_reader *request, continue; } + /* Shallow related arguments */ + if (process_shallow(arg, &data->shallows)) + continue; + if (process_deepen(arg, &data->depth)) + continue; + if (process_deepen_since(arg, &data->deepen_since, + &data->deepen_rev_list)) + continue; + if (process_deepen_not(arg, &data->deepen_not, + &data->deepen_rev_list)) + continue; + if (!strcmp(arg, "deepen-relative")) { + data->deepen_relative = 1; + continue; + } + /* ignore unknown lines maybe? */ die("unexpect line: '%s'", arg); } @@ -1272,6 +1316,23 @@ static int process_haves_and_send_acks(struct upload_pack_data *data) return ret; } +static void send_shallow_info(struct upload_pack_data *data) +{ + /* No shallow info needs to be sent */ + if (!data->depth && !data->deepen_rev_list && !data->shallows.nr && + !is_repository_shallow()) + return; + + packet_write_fmt(1, "shallow-info\n"); + + if (!send_shallow_list(data->depth, data->deepen_rev_list, + data->deepen_since, &data->deepen_not, + &data->shallows) && is_repository_shallow()) + deepen(INFINITE_DEPTH, data->deepen_relative, &data->shallows); + + packet_delim(1); +} + enum fetch_state { FETCH_PROCESS_ARGS = 0, FETCH_SEND_ACKS, @@ -1319,6 +1380,8 @@ int upload_pack_v2(struct repository *r, struct argv_array *keys, state = FETCH_DONE; break; case FETCH_SEND_PACK: + send_shallow_info(&data); + packet_write_fmt(1, "packfile\n"); create_pack_file(); state = FETCH_DONE; @@ -1331,3 +1394,11 @@ int upload_pack_v2(struct repository *r, struct argv_array *keys, upload_pack_data_clear(&data); return 0; } + +int upload_pack_advertise(struct repository *r, + struct strbuf *value) +{ + if (value) + strbuf_addstr(value, "shallow"); + return 1; +} diff --git a/upload-pack.h b/upload-pack.h index 5bbdb7e445..cab2178796 100644 --- a/upload-pack.h +++ b/upload-pack.h @@ -16,4 +16,8 @@ struct packet_reader; extern int upload_pack_v2(struct repository *r, struct argv_array *keys, struct packet_reader *request); +struct strbuf; +extern int upload_pack_advertise(struct repository *r, + struct strbuf *value); + #endif /* UPLOAD_PACK_H */ -- cgit v1.3-5-g9baa