From 37a8768f83d5932ca66202f9cc4977c20b022e17 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 30 Oct 2009 17:47:26 -0700 Subject: remote-curl: Refactor walker initialization We will need the walker, url and remote in other functions as the code grows larger to support smart HTTP. Extract this out into a set of globals we can easily reference once configured. Signed-off-by: Shawn O. Pearce CC: Daniel Barkalow Signed-off-by: Junio C Hamano --- remote-curl.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) (limited to 'remote-curl.c') diff --git a/remote-curl.c b/remote-curl.c index 2faf1c6344..478f3ea7d3 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -5,7 +5,17 @@ #include "http.h" #include "exec_cmd.h" -static struct ref *get_refs(struct walker *walker, const char *url) +static struct remote *remote; +static const char *url; +static struct walker *walker; + +static void init_walker(void) +{ + if (!walker) + walker = get_http_walker(url, remote); +} + +static struct ref *get_refs(void) { struct strbuf buffer = STRBUF_INIT; char *data, *start, *mid; @@ -21,6 +31,7 @@ static struct ref *get_refs(struct walker *walker, const char *url) refs_url = xmalloc(strlen(url) + 11); sprintf(refs_url, "%s/info/refs", url); + init_walker(); http_ret = http_get_strbuf(refs_url, &buffer, HTTP_NO_CACHE); switch (http_ret) { case HTTP_OK: @@ -78,10 +89,7 @@ static struct ref *get_refs(struct walker *walker, const char *url) int main(int argc, const char **argv) { - struct remote *remote; struct strbuf buf = STRBUF_INIT; - const char *url; - struct walker *walker = NULL; git_extract_argv0_path(argv[0]); setup_git_directory(); @@ -103,8 +111,7 @@ int main(int argc, const char **argv) break; if (!prefixcmp(buf.buf, "fetch ")) { char *obj = buf.buf + strlen("fetch "); - if (!walker) - walker = get_http_walker(url, remote); + init_walker(); walker->get_all = 1; walker->get_tree = 1; walker->get_history = 1; @@ -115,11 +122,8 @@ int main(int argc, const char **argv) printf("\n"); fflush(stdout); } else if (!strcmp(buf.buf, "list")) { - struct ref *refs; + struct ref *refs = get_refs(); struct ref *posn; - if (!walker) - walker = get_http_walker(url, remote); - refs = get_refs(walker, url); for (posn = refs; posn; posn = posn->next) { if (posn->symref) printf("@%s %s\n", posn->symref, posn->name); -- cgit v1.3-5-g9baa From 292ce46b60e2c12450c5c21044acf9c41bd837df Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 30 Oct 2009 17:47:28 -0700 Subject: remote-helpers: Fetch more than one ref in a batch Some network protocols (e.g. native git://) are able to fetch more than one ref at a time and reduce the overall transfer cost by combining the requests into a single exchange. Instead of feeding each fetch request one at a time to the helper, feed all of them at once so the helper can decide whether or not it should batch them. Signed-off-by: Shawn O. Pearce CC: Daniel Barkalow Signed-off-by: Junio C Hamano --- Documentation/git-remote-helpers.txt | 14 ++++-- remote-curl.c | 88 +++++++++++++++++++++++++++++++----- transport-helper.c | 39 +++++++++++----- 3 files changed, 115 insertions(+), 26 deletions(-) (limited to 'remote-curl.c') diff --git a/Documentation/git-remote-helpers.txt b/Documentation/git-remote-helpers.txt index 173ee232f2..e44d821c7b 100644 --- a/Documentation/git-remote-helpers.txt +++ b/Documentation/git-remote-helpers.txt @@ -36,10 +36,16 @@ Commands are given by the caller on the helper's standard input, one per line. complete list, outputs a blank line. 'fetch' :: - Fetches the given object, writing the necessary objects to the - database. Outputs a blank line when the fetch is - complete. Only objects which were reported in the ref list - with a sha1 may be fetched this way. + Fetches the given object, writing the necessary objects + to the database. Fetch commands are sent in a batch, one + per line, and the batch is terminated with a blank line. + Outputs a single blank line when all fetch commands in the + same batch are complete. Only objects which were reported + in the ref list with a sha1 may be fetched this way. ++ +Optionally may output a 'lock ' line indicating a file under +GIT_DIR/objects/pack which is keeping a pack until refs can be +suitably updated. + Supported if the helper has the "fetch" capability. diff --git a/remote-curl.c b/remote-curl.c index 478f3ea7d3..22cd5c5fd2 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -87,6 +87,81 @@ static struct ref *get_refs(void) return refs; } +static int fetch_dumb(int nr_heads, struct ref **to_fetch) +{ + char **targets = xmalloc(nr_heads * sizeof(char*)); + int ret, i; + + for (i = 0; i < nr_heads; i++) + targets[i] = xstrdup(sha1_to_hex(to_fetch[i]->old_sha1)); + + init_walker(); + walker->get_all = 1; + walker->get_tree = 1; + walker->get_history = 1; + walker->get_verbosely = 0; + walker->get_recover = 0; + ret = walker_fetch(walker, nr_heads, targets, NULL, NULL); + + for (i = 0; i < nr_heads; i++) + free(targets[i]); + free(targets); + + return ret ? error("Fetch failed.") : 0; +} + +static void parse_fetch(struct strbuf *buf) +{ + struct ref **to_fetch = NULL; + struct ref *list_head = NULL; + struct ref **list = &list_head; + int alloc_heads = 0, nr_heads = 0; + + do { + if (!prefixcmp(buf->buf, "fetch ")) { + char *p = buf->buf + strlen("fetch "); + char *name; + struct ref *ref; + unsigned char old_sha1[20]; + + if (strlen(p) < 40 || get_sha1_hex(p, old_sha1)) + die("protocol error: expected sha/ref, got %s'", p); + if (p[40] == ' ') + name = p + 41; + else if (!p[40]) + name = ""; + else + die("protocol error: expected sha/ref, got %s'", p); + + ref = alloc_ref(name); + hashcpy(ref->old_sha1, old_sha1); + + *list = ref; + list = &ref->next; + + ALLOC_GROW(to_fetch, nr_heads + 1, alloc_heads); + to_fetch[nr_heads++] = ref; + } + else + die("http transport does not support %s", buf->buf); + + strbuf_reset(buf); + if (strbuf_getline(buf, stdin, '\n') == EOF) + return; + if (!*buf->buf) + break; + } while (1); + + if (fetch_dumb(nr_heads, to_fetch)) + exit(128); /* error already reported */ + free_refs(list_head); + free(to_fetch); + + printf("\n"); + fflush(stdout); + strbuf_reset(buf); +} + int main(int argc, const char **argv) { struct strbuf buf = STRBUF_INIT; @@ -110,17 +185,8 @@ int main(int argc, const char **argv) if (strbuf_getline(&buf, stdin, '\n') == EOF) break; if (!prefixcmp(buf.buf, "fetch ")) { - char *obj = buf.buf + strlen("fetch "); - init_walker(); - walker->get_all = 1; - walker->get_tree = 1; - walker->get_history = 1; - walker->get_verbosely = 0; - walker->get_recover = 0; - if (walker_fetch(walker, 1, &obj, NULL, NULL)) - die("Fetch failed."); - printf("\n"); - fflush(stdout); + parse_fetch(&buf); + } else if (!strcmp(buf.buf, "list")) { struct ref *refs = get_refs(); struct ref *posn; diff --git a/transport-helper.c b/transport-helper.c index f57e84c676..9de3408947 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -10,6 +10,7 @@ struct helper_data { const char *name; struct child_process *helper; + FILE *out; unsigned fetch : 1; }; @@ -18,7 +19,6 @@ static struct child_process *get_helper(struct transport *transport) struct helper_data *data = transport->data; struct strbuf buf = STRBUF_INIT; struct child_process *helper; - FILE *file; if (data->helper) return data->helper; @@ -39,9 +39,9 @@ static struct child_process *get_helper(struct transport *transport) write_str_in_full(helper->in, "capabilities\n"); - file = xfdopen(helper->out, "r"); + data->out = xfdopen(helper->out, "r"); while (1) { - if (strbuf_getline(&buf, file, '\n') == EOF) + if (strbuf_getline(&buf, data->out, '\n') == EOF) exit(128); /* child died, message supplied already */ if (!*buf.buf) @@ -58,6 +58,7 @@ static int disconnect_helper(struct transport *transport) if (data->helper) { write_str_in_full(data->helper->in, "\n"); close(data->helper->in); + fclose(data->out); finish_command(data->helper); free((char *)data->helper->argv[0]); free(data->helper->argv); @@ -70,8 +71,7 @@ static int disconnect_helper(struct transport *transport) static int fetch_with_fetch(struct transport *transport, int nr_heads, const struct ref **to_fetch) { - struct child_process *helper = get_helper(transport); - FILE *file = xfdopen(helper->out, "r"); + struct helper_data *data = transport->data; int i; struct strbuf buf = STRBUF_INIT; @@ -82,12 +82,30 @@ static int fetch_with_fetch(struct transport *transport, strbuf_addf(&buf, "fetch %s %s\n", sha1_to_hex(posn->old_sha1), posn->name); - write_in_full(helper->in, buf.buf, buf.len); - strbuf_reset(&buf); + } - if (strbuf_getline(&buf, file, '\n') == EOF) + strbuf_addch(&buf, '\n'); + if (write_in_full(data->helper->in, buf.buf, buf.len) != buf.len) + die_errno("cannot send fetch to %s", data->name); + + while (1) { + strbuf_reset(&buf); + if (strbuf_getline(&buf, data->out, '\n') == EOF) exit(128); /* child died, message supplied already */ + + if (!prefixcmp(buf.buf, "lock ")) { + const char *name = buf.buf + 5; + if (transport->pack_lockfile) + warning("%s also locked %s", data->name, name); + else + transport->pack_lockfile = xstrdup(name); + } + else if (!buf.len) + break; + else + warning("%s unexpectedly said: '%s'", data->name, buf.buf); } + strbuf_release(&buf); return 0; } @@ -113,21 +131,20 @@ static int fetch(struct transport *transport, static struct ref *get_refs_list(struct transport *transport, int for_push) { + struct helper_data *data = transport->data; struct child_process *helper; struct ref *ret = NULL; struct ref **tail = &ret; struct ref *posn; struct strbuf buf = STRBUF_INIT; - FILE *file; helper = get_helper(transport); write_str_in_full(helper->in, "list\n"); - file = xfdopen(helper->out, "r"); while (1) { char *eov, *eon; - if (strbuf_getline(&buf, file, '\n') == EOF) + if (strbuf_getline(&buf, data->out, '\n') == EOF) exit(128); /* child died, message supplied already */ if (!*buf.buf) -- cgit v1.3-5-g9baa From ef08ef9ea0a271e5be5844408d2496a946d6e8d9 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 30 Oct 2009 17:47:29 -0700 Subject: remote-helpers: Support custom transport options Some transports, like the native pack transport implemented by fetch-pack, support useful features like depth or include tags. These should be exposed if the underlying helper knows how to use them. Signed-off-by: Shawn O. Pearce CC: Daniel Barkalow Signed-off-by: Junio C Hamano --- Documentation/git-remote-helpers.txt | 38 ++++++++++++++++ remote-curl.c | 74 +++++++++++++++++++++++++++++- transport-helper.c | 88 +++++++++++++++++++++++++++++++++++- 3 files changed, 198 insertions(+), 2 deletions(-) (limited to 'remote-curl.c') diff --git a/Documentation/git-remote-helpers.txt b/Documentation/git-remote-helpers.txt index e44d821c7b..1133f047bf 100644 --- a/Documentation/git-remote-helpers.txt +++ b/Documentation/git-remote-helpers.txt @@ -35,6 +35,16 @@ Commands are given by the caller on the helper's standard input, one per line. the name; unrecognized attributes are ignored. After the complete list, outputs a blank line. +'option' :: + Set the transport helper option to . Outputs a + single line containing one of 'ok' (option successfully set), + 'unsupported' (option not recognized) or 'error ' + (option is supported but is not correct + for it). Options should be set before other commands, + and may how those commands behave. ++ +Supported if the helper has the "option" capability. + 'fetch' :: Fetches the given object, writing the necessary objects to the database. Fetch commands are sent in a batch, one @@ -63,11 +73,39 @@ CAPABILITIES 'fetch':: This helper supports the 'fetch' command. +'option':: + This helper supports the option command. + REF LIST ATTRIBUTES ------------------- None are defined yet, but the caller must accept any which are supplied. +OPTIONS +------- +'option verbosity' :: + Change the level of messages displayed by the helper. + When N is 0 the end-user has asked the process to be + quiet, and the helper should produce only error output. + N of 1 is the default level of verbosity, higher values + of N correspond to the number of -v flags passed on the + command line. + +'option progress' \{'true'|'false'\}:: + Enable (or disable) progress messages displayed by the + transport helper during a command. + +'option depth' :: + Deepen the history of a shallow repository. + +'option followtags' \{'true'|'false'\}:: + If enabled the helper should automatically fetch annotated + tag objects if the object the tag points at was transferred + during the fetch command. If the tag is not fetched by + the helper a second fetch command will usually be sent to + ask for the tag specifically. Some helpers may be able to + use this option to avoid a second network connection. + Documentation ------------- Documentation by Daniel Barkalow. diff --git a/remote-curl.c b/remote-curl.c index 22cd5c5fd2..0951f11615 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -9,12 +9,61 @@ static struct remote *remote; static const char *url; static struct walker *walker; +struct options { + int verbosity; + unsigned long depth; + unsigned progress : 1, + followtags : 1; +}; +static struct options options; + static void init_walker(void) { if (!walker) walker = get_http_walker(url, remote); } +static int set_option(const char *name, const char *value) +{ + if (!strcmp(name, "verbosity")) { + char *end; + int v = strtol(value, &end, 10); + if (value == end || *end) + return -1; + options.verbosity = v; + return 0; + } + else if (!strcmp(name, "progress")) { + if (!strcmp(value, "true")) + options.progress = 1; + else if (!strcmp(value, "false")) + options.progress = 0; + else + return -1; + return 1 /* TODO implement later */; + } + else if (!strcmp(name, "depth")) { + char *end; + unsigned long v = strtoul(value, &end, 10); + if (value == end || *end) + return -1; + options.depth = v; + return 1 /* TODO implement later */; + } + else if (!strcmp(name, "followtags")) { + if (!strcmp(value, "true")) + options.followtags = 1; + else if (!strcmp(value, "false")) + options.followtags = 0; + else + return -1; + return 1 /* TODO implement later */; + } + else { + return 1 /* unsupported */; + } +} + static struct ref *get_refs(void) { struct strbuf buffer = STRBUF_INIT; @@ -99,7 +148,7 @@ static int fetch_dumb(int nr_heads, struct ref **to_fetch) walker->get_all = 1; walker->get_tree = 1; walker->get_history = 1; - walker->get_verbosely = 0; + walker->get_verbosely = options.verbosity >= 3; walker->get_recover = 0; ret = walker_fetch(walker, nr_heads, targets, NULL, NULL); @@ -173,6 +222,9 @@ int main(int argc, const char **argv) return 1; } + options.verbosity = 1; + options.progress = !!isatty(2); + remote = remote_get(argv[1]); if (argc > 2) { @@ -198,8 +250,28 @@ int main(int argc, const char **argv) } printf("\n"); fflush(stdout); + } else if (!prefixcmp(buf.buf, "option ")) { + char *name = buf.buf + strlen("option "); + char *value = strchr(name, ' '); + int result; + + if (value) + *value++ = '\0'; + else + value = "true"; + + result = set_option(name, value); + if (!result) + printf("ok\n"); + else if (result < 0) + printf("error invalid value\n"); + else + printf("unsupported\n"); + fflush(stdout); + } else if (!strcmp(buf.buf, "capabilities")) { printf("fetch\n"); + printf("option\n"); printf("\n"); fflush(stdout); } else { diff --git a/transport-helper.c b/transport-helper.c index 9de3408947..577abc638e 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -5,13 +5,15 @@ #include "commit.h" #include "diff.h" #include "revision.h" +#include "quote.h" struct helper_data { const char *name; struct child_process *helper; FILE *out; - unsigned fetch : 1; + unsigned fetch : 1, + option : 1; }; static struct child_process *get_helper(struct transport *transport) @@ -48,6 +50,8 @@ static struct child_process *get_helper(struct transport *transport) break; if (!strcmp(buf.buf, "fetch")) data->fetch = 1; + if (!strcmp(buf.buf, "option")) + data->option = 1; } return data->helper; } @@ -65,9 +69,88 @@ static int disconnect_helper(struct transport *transport) free(data->helper); data->helper = NULL; } + free(data); return 0; } +static const char *unsupported_options[] = { + TRANS_OPT_UPLOADPACK, + TRANS_OPT_RECEIVEPACK, + TRANS_OPT_THIN, + TRANS_OPT_KEEP + }; +static const char *boolean_options[] = { + TRANS_OPT_THIN, + TRANS_OPT_KEEP, + TRANS_OPT_FOLLOWTAGS + }; + +static int set_helper_option(struct transport *transport, + const char *name, const char *value) +{ + struct helper_data *data = transport->data; + struct child_process *helper = get_helper(transport); + struct strbuf buf = STRBUF_INIT; + int i, ret, is_bool = 0; + + if (!data->option) + return 1; + + for (i = 0; i < ARRAY_SIZE(unsupported_options); i++) { + if (!strcmp(name, unsupported_options[i])) + return 1; + } + + for (i = 0; i < ARRAY_SIZE(boolean_options); i++) { + if (!strcmp(name, boolean_options[i])) { + is_bool = 1; + break; + } + } + + strbuf_addf(&buf, "option %s ", name); + if (is_bool) + strbuf_addstr(&buf, value ? "true" : "false"); + else + quote_c_style(value, &buf, NULL, 0); + strbuf_addch(&buf, '\n'); + + if (write_in_full(helper->in, buf.buf, buf.len) != buf.len) + die_errno("cannot send option to %s", data->name); + + strbuf_reset(&buf); + if (strbuf_getline(&buf, data->out, '\n') == EOF) + exit(128); /* child died, message supplied already */ + + if (!strcmp(buf.buf, "ok")) + ret = 0; + else if (!prefixcmp(buf.buf, "error")) { + ret = -1; + } else if (!strcmp(buf.buf, "unsupported")) + ret = 1; + else { + warning("%s unexpectedly said: '%s'", data->name, buf.buf); + ret = 1; + } + strbuf_release(&buf); + return ret; +} + +static void standard_options(struct transport *t) +{ + char buf[16]; + int n; + int v = t->verbose; + int no_progress = v < 0 || (!t->progress && !isatty(1)); + + set_helper_option(t, "progress", !no_progress ? "true" : "false"); + + n = snprintf(buf, sizeof(buf), "%d", v + 1); + if (n >= sizeof(buf)) + die("impossibly large verbosity value"); + set_helper_option(t, "verbosity", buf); +} + static int fetch_with_fetch(struct transport *transport, int nr_heads, const struct ref **to_fetch) { @@ -75,6 +158,8 @@ static int fetch_with_fetch(struct transport *transport, int i; struct strbuf buf = STRBUF_INIT; + standard_options(transport); + for (i = 0; i < nr_heads; i++) { const struct ref *posn = to_fetch[i]; if (posn->status & REF_STATUS_UPTODATE) @@ -178,6 +263,7 @@ int transport_helper_init(struct transport *transport, const char *name) data->name = name; transport->data = data; + transport->set_option = set_helper_option; transport->get_refs_list = get_refs_list; transport->fetch = fetch; transport->disconnect = disconnect_helper; -- cgit v1.3-5-g9baa From ae4efe195752c27cb25fca9451852c0f4eebdb28 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 30 Oct 2009 17:47:30 -0700 Subject: Move WebDAV HTTP push under remote-curl The remote helper interface now supports the push capability, which can be used to ask the implementation to push one or more specs to the remote repository. For remote-curl we implement this by calling the existing WebDAV based git-http-push executable. Internally the helper interface uses the push_refs transport hook so that the complexity of the refspec parsing and matching can be reused between remote implementations. When possible however the helper protocol uses source ref name rather than the source SHA-1, thereby allowing the helper to access this name if it is useful. >From Clemens Buchacher : update http tests according to remote-curl capabilities o Pushing packed refs is now fixed. o The transport helper fails if refs are already up-to-date. Add a test for that. o The transport helper will notice if refs are already up-to-date. We therefore need to update server info in the unpacked-refs test. o The transport helper will purge deleted branches automatically. o Use a variable ($ORIG_HEAD) instead of full SHA-1 name. Signed-off-by: Tay Ray Chuan Signed-off-by: Clemens Buchacher Signed-off-by: Shawn O. Pearce CC: Daniel Barkalow CC: Mike Hommey Signed-off-by: Junio C Hamano --- Documentation/git-remote-helpers.txt | 33 ++++++++- http-push.c | 29 +++++++- remote-curl.c | 97 ++++++++++++++++++++++--- t/t5540-http-push.sh | 14 ++-- transport-helper.c | 137 ++++++++++++++++++++++++++++++++++- transport.c | 31 -------- 6 files changed, 287 insertions(+), 54 deletions(-) (limited to 'remote-curl.c') diff --git a/Documentation/git-remote-helpers.txt b/Documentation/git-remote-helpers.txt index 1133f047bf..8beb42dbb9 100644 --- a/Documentation/git-remote-helpers.txt +++ b/Documentation/git-remote-helpers.txt @@ -34,6 +34,10 @@ Commands are given by the caller on the helper's standard input, one per line. value of the ref. A space-separated list of attributes follows the name; unrecognized attributes are ignored. After the complete list, outputs a blank line. ++ +If 'push' is supported this may be called as 'list for-push' +to obtain the current refs prior to sending one or more 'push' +commands to the helper. 'option' :: Set the transport helper option to . Outputs a @@ -59,6 +63,22 @@ suitably updated. + Supported if the helper has the "fetch" capability. +'push' +::: + Pushes the given commit or branch locally to the + remote branch described by . A batch sequence of + one or more push commands is terminated with a blank line. ++ +Zero or more protocol options may be entered after the last 'push' +command, before the batch's terminating blank line. ++ +When the push is complete, outputs one or more 'ok ' or +'error ?' lines to indicate success or failure of +each pushed ref. The status report output is terminated by +a blank line. The option field may be quoted in a C +style string if it contains an LF. ++ +Supported if the helper has the "push" capability. + If a fatal error occurs, the program writes the error message to stderr and exits. The caller should expect that a suitable error message has been printed if the child closes the connection without @@ -76,10 +96,16 @@ CAPABILITIES 'option':: This helper supports the option command. +'push':: + This helper supports the 'push' command. + REF LIST ATTRIBUTES ------------------- -None are defined yet, but the caller must accept any which are supplied. +'for-push':: + The caller wants to use the ref list to prepare push + commands. A helper might chose to acquire the ref list by + opening a different type of connection to the destination. OPTIONS ------- @@ -106,6 +132,11 @@ OPTIONS ask for the tag specifically. Some helpers may be able to use this option to avoid a second network connection. +'option dry-run' \{'true'|'false'\}: + If true, pretend the operation completed successfully, + but don't actually change any repository data. For most + helpers this only applies to the 'push', if supported. + Documentation ------------- Documentation by Daniel Barkalow. diff --git a/http-push.c b/http-push.c index cc5d4b8c32..f10803a389 100644 --- a/http-push.c +++ b/http-push.c @@ -78,6 +78,7 @@ static int push_verbosely; static int push_all = MATCH_REFS_NONE; static int force_all; static int dry_run; +static int helper_status; static struct object_list *objects; @@ -1813,6 +1814,10 @@ int main(int argc, char **argv) dry_run = 1; continue; } + if (!strcmp(arg, "--helper-status")) { + helper_status = 1; + continue; + } if (!strcmp(arg, "--verbose")) { push_verbosely = 1; http_is_verbose = 1; @@ -1911,9 +1916,12 @@ int main(int argc, char **argv) /* Remove a remote branch if -d or -D was specified */ if (delete_branch) { - if (delete_remote_branch(refspec[0], force_delete) == -1) + if (delete_remote_branch(refspec[0], force_delete) == -1) { fprintf(stderr, "Unable to delete remote branch %s\n", refspec[0]); + if (helper_status) + printf("error %s cannot remove\n", refspec[0]); + } goto cleanup; } @@ -1925,6 +1933,8 @@ int main(int argc, char **argv) } if (!remote_refs) { fprintf(stderr, "No refs in common and none specified; doing nothing.\n"); + if (helper_status) + printf("error null no match\n"); rc = 0; goto cleanup; } @@ -1942,8 +1952,12 @@ int main(int argc, char **argv) if (is_null_sha1(ref->peer_ref->new_sha1)) { if (delete_remote_branch(ref->name, 1) == -1) { error("Could not remove %s", ref->name); + if (helper_status) + printf("error %s cannot remove\n", ref->name); rc = -4; } + else if (helper_status) + printf("ok %s\n", ref->name); new_refs++; continue; } @@ -1951,6 +1965,8 @@ int main(int argc, char **argv) if (!hashcmp(ref->old_sha1, ref->peer_ref->new_sha1)) { if (push_verbosely || 1) fprintf(stderr, "'%s': up-to-date\n", ref->name); + if (helper_status) + printf("ok %s up to date\n", ref->name); continue; } @@ -1974,6 +1990,8 @@ int main(int argc, char **argv) "need to pull first?", ref->name, ref->peer_ref->name); + if (helper_status) + printf("error %s non-fast forward\n", ref->name); rc = -2; continue; } @@ -1987,14 +2005,19 @@ int main(int argc, char **argv) if (strcmp(ref->name, ref->peer_ref->name)) fprintf(stderr, " using '%s'", ref->peer_ref->name); fprintf(stderr, "\n from %s\n to %s\n", old_hex, new_hex); - if (dry_run) + if (dry_run) { + if (helper_status) + printf("ok %s\n", ref->name); continue; + } /* Lock remote branch ref */ ref_lock = lock_remote(ref->name, LOCK_TIME); if (ref_lock == NULL) { fprintf(stderr, "Unable to lock remote branch %s\n", ref->name); + if (helper_status) + printf("error %s lock error\n", ref->name); rc = 1; continue; } @@ -2045,6 +2068,8 @@ int main(int argc, char **argv) if (!rc) fprintf(stderr, " done\n"); + if (helper_status) + printf("%s %s\n", !rc ? "ok" : "error", ref->name); unlock_remote(ref_lock); check_locks(); } diff --git a/remote-curl.c b/remote-curl.c index 0951f11615..5c9dd97d1d 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -4,6 +4,7 @@ #include "walker.h" #include "http.h" #include "exec_cmd.h" +#include "run-command.h" static struct remote *remote; static const char *url; @@ -13,7 +14,8 @@ struct options { int verbosity; unsigned long depth; unsigned progress : 1, - followtags : 1; + followtags : 1, + dry_run : 1; }; static struct options options; @@ -59,6 +61,15 @@ static int set_option(const char *name, const char *value) return -1; return 1 /* TODO implement later */; } + else if (!strcmp(name, "dry-run")) { + if (!strcmp(value, "true")) + options.dry_run = 1; + else if (!strcmp(value, "false")) + options.dry_run = 0; + else + return -1; + return 0; + } else { return 1 /* unsupported */; } @@ -136,6 +147,20 @@ static struct ref *get_refs(void) return refs; } +static void output_refs(struct ref *refs) +{ + struct ref *posn; + for (posn = refs; posn; posn = posn->next) { + if (posn->symref) + printf("@%s %s\n", posn->symref, posn->name); + else + printf("%s %s\n", sha1_to_hex(posn->old_sha1), posn->name); + } + printf("\n"); + fflush(stdout); + free_refs(refs); +} + static int fetch_dumb(int nr_heads, struct ref **to_fetch) { char **targets = xmalloc(nr_heads * sizeof(char*)); @@ -211,6 +236,58 @@ static void parse_fetch(struct strbuf *buf) strbuf_reset(buf); } +static int push_dav(int nr_spec, char **specs) +{ + const char **argv = xmalloc((10 + nr_spec) * sizeof(char*)); + int argc = 0, i; + + argv[argc++] = "http-push"; + argv[argc++] = "--helper-status"; + if (options.dry_run) + argv[argc++] = "--dry-run"; + if (options.verbosity > 1) + argv[argc++] = "--verbose"; + argv[argc++] = url; + for (i = 0; i < nr_spec; i++) + argv[argc++] = specs[i]; + argv[argc++] = NULL; + + if (run_command_v_opt(argv, RUN_GIT_CMD)) + die("git-%s failed", argv[0]); + free(argv); + return 0; +} + +static void parse_push(struct strbuf *buf) +{ + char **specs = NULL; + int alloc_spec = 0, nr_spec = 0, i; + + do { + if (!prefixcmp(buf->buf, "push ")) { + ALLOC_GROW(specs, nr_spec + 1, alloc_spec); + specs[nr_spec++] = xstrdup(buf->buf + 5); + } + else + die("http transport does not support %s", buf->buf); + + strbuf_reset(buf); + if (strbuf_getline(buf, stdin, '\n') == EOF) + return; + if (!*buf->buf) + break; + } while (1); + + if (push_dav(nr_spec, specs)) + exit(128); /* error already reported */ + for (i = 0; i < nr_spec; i++) + free(specs[i]); + free(specs); + + printf("\n"); + fflush(stdout); +} + int main(int argc, const char **argv) { struct strbuf buf = STRBUF_INIT; @@ -239,17 +316,12 @@ int main(int argc, const char **argv) if (!prefixcmp(buf.buf, "fetch ")) { parse_fetch(&buf); - } else if (!strcmp(buf.buf, "list")) { - struct ref *refs = get_refs(); - struct ref *posn; - for (posn = refs; posn; posn = posn->next) { - if (posn->symref) - printf("@%s %s\n", posn->symref, posn->name); - else - printf("%s %s\n", sha1_to_hex(posn->old_sha1), posn->name); - } - printf("\n"); - fflush(stdout); + } else if (!strcmp(buf.buf, "list") || !prefixcmp(buf.buf, "list ")) { + output_refs(get_refs()); + + } else if (!prefixcmp(buf.buf, "push ")) { + parse_push(&buf); + } else if (!prefixcmp(buf.buf, "option ")) { char *name = buf.buf + strlen("option "); char *value = strchr(name, ' '); @@ -272,6 +344,7 @@ int main(int argc, const char **argv) } else if (!strcmp(buf.buf, "capabilities")) { printf("fetch\n"); printf("option\n"); + printf("push\n"); printf("\n"); fflush(stdout); } else { diff --git a/t/t5540-http-push.sh b/t/t5540-http-push.sh index f4a2cf6c17..09edd2387b 100755 --- a/t/t5540-http-push.sh +++ b/t/t5540-http-push.sh @@ -36,6 +36,7 @@ test_expect_success 'setup remote repository' ' cd test_repo.git && git --bare update-server-info && mv hooks/post-update.sample hooks/post-update && + ORIG_HEAD=$(git rev-parse --verify HEAD) && cd - && mv test_repo.git "$HTTPD_DOCUMENT_ROOT_PATH" ' @@ -45,7 +46,7 @@ test_expect_success 'clone remote repository' ' git clone $HTTPD_URL/test_repo.git test_repo_clone ' -test_expect_failure 'push to remote repository with packed refs' ' +test_expect_success 'push to remote repository with packed refs' ' cd "$ROOT_PATH"/test_repo_clone && : >path2 && git add path2 && @@ -57,11 +58,15 @@ test_expect_failure 'push to remote repository with packed refs' ' test $HEAD = $(git rev-parse --verify HEAD)) ' -test_expect_success ' push to remote repository with unpacked refs' ' +test_expect_failure 'push already up-to-date' ' + git push +' + +test_expect_success 'push to remote repository with unpacked refs' ' (cd "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git && rm packed-refs && - git update-ref refs/heads/master \ - 0c973ae9bd51902a28466f3850b543fa66a6aaf4) && + git update-ref refs/heads/master $ORIG_HEAD && + git --bare update-server-info) && git push && (cd "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git && test $HEAD = $(git rev-parse --verify HEAD)) @@ -113,7 +118,6 @@ test_expect_success 'create and delete remote branch' ' git push origin dev && git fetch && git push origin :dev && - git branch -d -r origin/dev && git fetch && test_must_fail git show-ref --verify refs/remotes/origin/dev ' diff --git a/transport-helper.c b/transport-helper.c index 577abc638e..16c6641f03 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -1,6 +1,6 @@ #include "cache.h" #include "transport.h" - +#include "quote.h" #include "run-command.h" #include "commit.h" #include "diff.h" @@ -13,7 +13,8 @@ struct helper_data struct child_process *helper; FILE *out; unsigned fetch : 1, - option : 1; + option : 1, + push : 1; }; static struct child_process *get_helper(struct transport *transport) @@ -52,6 +53,8 @@ static struct child_process *get_helper(struct transport *transport) data->fetch = 1; if (!strcmp(buf.buf, "option")) data->option = 1; + if (!strcmp(buf.buf, "push")) + data->push = 1; } return data->helper; } @@ -214,6 +217,130 @@ static int fetch(struct transport *transport, return -1; } +static int push_refs(struct transport *transport, + struct ref *remote_refs, int flags) +{ + int force_all = flags & TRANSPORT_PUSH_FORCE; + int mirror = flags & TRANSPORT_PUSH_MIRROR; + struct helper_data *data = transport->data; + struct strbuf buf = STRBUF_INIT; + struct child_process *helper; + struct ref *ref; + + if (!remote_refs) + return 0; + + helper = get_helper(transport); + if (!data->push) + return 1; + + for (ref = remote_refs; ref; ref = ref->next) { + if (ref->peer_ref) + hashcpy(ref->new_sha1, ref->peer_ref->new_sha1); + else if (!mirror) + continue; + + ref->deletion = is_null_sha1(ref->new_sha1); + if (!ref->deletion && + !hashcmp(ref->old_sha1, ref->new_sha1)) { + ref->status = REF_STATUS_UPTODATE; + continue; + } + + if (force_all) + ref->force = 1; + + strbuf_addstr(&buf, "push "); + if (!ref->deletion) { + if (ref->force) + strbuf_addch(&buf, '+'); + if (ref->peer_ref) + strbuf_addstr(&buf, ref->peer_ref->name); + else + strbuf_addstr(&buf, sha1_to_hex(ref->new_sha1)); + } + strbuf_addch(&buf, ':'); + strbuf_addstr(&buf, ref->name); + strbuf_addch(&buf, '\n'); + } + + transport->verbose = flags & TRANSPORT_PUSH_VERBOSE ? 1 : 0; + standard_options(transport); + + if (flags & TRANSPORT_PUSH_DRY_RUN) { + if (set_helper_option(transport, "dry-run", "true") != 0) + die("helper %s does not support dry-run", data->name); + } + + strbuf_addch(&buf, '\n'); + if (write_in_full(helper->in, buf.buf, buf.len) != buf.len) + exit(128); + + ref = remote_refs; + while (1) { + char *refname, *msg; + int status; + + strbuf_reset(&buf); + if (strbuf_getline(&buf, data->out, '\n') == EOF) + exit(128); /* child died, message supplied already */ + if (!buf.len) + break; + + if (!prefixcmp(buf.buf, "ok ")) { + status = REF_STATUS_OK; + refname = buf.buf + 3; + } else if (!prefixcmp(buf.buf, "error ")) { + status = REF_STATUS_REMOTE_REJECT; + refname = buf.buf + 6; + } else + die("expected ok/error, helper said '%s'\n", buf.buf); + + msg = strchr(refname, ' '); + if (msg) { + struct strbuf msg_buf = STRBUF_INIT; + const char *end; + + *msg++ = '\0'; + if (!unquote_c_style(&msg_buf, msg, &end)) + msg = strbuf_detach(&msg_buf, NULL); + else + msg = xstrdup(msg); + strbuf_release(&msg_buf); + + if (!strcmp(msg, "no match")) { + status = REF_STATUS_NONE; + free(msg); + msg = NULL; + } + else if (!strcmp(msg, "up to date")) { + status = REF_STATUS_UPTODATE; + free(msg); + msg = NULL; + } + else if (!strcmp(msg, "non-fast forward")) { + status = REF_STATUS_REJECT_NONFASTFORWARD; + free(msg); + msg = NULL; + } + } + + if (ref) + ref = find_ref_by_name(ref, refname); + if (!ref) + ref = find_ref_by_name(remote_refs, refname); + if (!ref) { + warning("helper reported unexpected status of %s", refname); + continue; + } + + ref->status = status; + ref->remote_status = msg; + } + strbuf_release(&buf); + return 0; +} + static struct ref *get_refs_list(struct transport *transport, int for_push) { struct helper_data *data = transport->data; @@ -225,7 +352,10 @@ static struct ref *get_refs_list(struct transport *transport, int for_push) helper = get_helper(transport); - write_str_in_full(helper->in, "list\n"); + if (data->push && for_push) + write_str_in_full(helper->in, "list for-push\n"); + else + write_str_in_full(helper->in, "list\n"); while (1) { char *eov, *eon; @@ -266,6 +396,7 @@ int transport_helper_init(struct transport *transport, const char *name) transport->set_option = set_helper_option; transport->get_refs_list = get_refs_list; transport->fetch = fetch; + transport->push_refs = push_refs; transport->disconnect = disconnect_helper; return 0; } diff --git a/transport.c b/transport.c index 644a30a0b2..6d9652db94 100644 --- a/transport.c +++ b/transport.c @@ -349,35 +349,6 @@ static int rsync_transport_push(struct transport *transport, return result; } -#ifndef NO_CURL -static int curl_transport_push(struct transport *transport, int refspec_nr, const char **refspec, int flags) -{ - const char **argv; - int argc; - - if (flags & TRANSPORT_PUSH_MIRROR) - return error("http transport does not support mirror mode"); - - argv = xmalloc((refspec_nr + 12) * sizeof(char *)); - argv[0] = "http-push"; - argc = 1; - if (flags & TRANSPORT_PUSH_ALL) - argv[argc++] = "--all"; - if (flags & TRANSPORT_PUSH_FORCE) - argv[argc++] = "--force"; - if (flags & TRANSPORT_PUSH_DRY_RUN) - argv[argc++] = "--dry-run"; - if (flags & TRANSPORT_PUSH_VERBOSE) - argv[argc++] = "--verbose"; - argv[argc++] = transport->url; - while (refspec_nr--) - argv[argc++] = *refspec++; - argv[argc] = NULL; - return !!run_command_v_opt(argv, RUN_GIT_CMD); -} - -#endif - struct bundle_transport_data { int fd; struct bundle_header header; @@ -826,8 +797,6 @@ struct transport *transport_get(struct remote *remote, const char *url) transport_helper_init(ret, "curl"); #ifdef NO_CURL error("git was compiled without libcurl support."); -#else - ret->push = curl_transport_push; #endif } else if (is_local(url) && is_file(url)) { -- cgit v1.3-5-g9baa From 97cc7bc45c2e076184b84749952b216036a6ecd6 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 30 Oct 2009 17:47:40 -0700 Subject: Discover refs via smart HTTP server when available Instead of loading the cached info/refs, try to use the smart HTTP version when the server supports it. Since the smart variant is actually the pkt-line stream from the start of either upload-pack or receive-pack we need to parse these through get_remote_heads, which requires a background thread to feed its pipe. Signed-off-by: Shawn O. Pearce CC: Daniel Barkalow Signed-off-by: Junio C Hamano --- remote-curl.c | 148 +++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 131 insertions(+), 17 deletions(-) (limited to 'remote-curl.c') diff --git a/remote-curl.c b/remote-curl.c index 5c9dd97d1d..3917d45deb 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -5,6 +5,7 @@ #include "http.h" #include "exec_cmd.h" #include "run-command.h" +#include "pkt-line.h" static struct remote *remote; static const char *url; @@ -75,21 +76,46 @@ static int set_option(const char *name, const char *value) } } -static struct ref *get_refs(void) +struct discovery { + const char *service; + char *buf_alloc; + char *buf; + size_t len; + unsigned proto_git : 1; +}; +static struct discovery *last_discovery; + +static void free_discovery(struct discovery *d) +{ + if (d) { + if (d == last_discovery) + last_discovery = NULL; + free(d->buf_alloc); + free(d); + } +} + +static struct discovery* discover_refs(const char *service) { struct strbuf buffer = STRBUF_INIT; - char *data, *start, *mid; - char *ref_name; + struct discovery *last = last_discovery; char *refs_url; - int i = 0; - int http_ret; + int http_ret, is_http = 0; - struct ref *refs = NULL; - struct ref *ref = NULL; - struct ref *last_ref = NULL; + if (last && !strcmp(service, last->service)) + return last; + free_discovery(last); - refs_url = xmalloc(strlen(url) + 11); - sprintf(refs_url, "%s/info/refs", url); + strbuf_addf(&buffer, "%s/info/refs", url); + if (!prefixcmp(url, "http://") || !prefixcmp(url, "https://")) { + is_http = 1; + if (!strchr(url, '?')) + strbuf_addch(&buffer, '?'); + else + strbuf_addch(&buffer, '&'); + strbuf_addf(&buffer, "service=%s", service); + } + refs_url = strbuf_detach(&buffer, NULL); init_walker(); http_ret = http_get_strbuf(refs_url, &buffer, HTTP_NO_CACHE); @@ -104,10 +130,86 @@ static struct ref *get_refs(void) die("HTTP request failed"); } - data = buffer.buf; + last= xcalloc(1, sizeof(*last_discovery)); + last->service = service; + last->buf_alloc = strbuf_detach(&buffer, &last->len); + last->buf = last->buf_alloc; + + if (is_http && 5 <= last->len && last->buf[4] == '#') { + /* smart HTTP response; validate that the service + * pkt-line matches our request. + */ + struct strbuf exp = STRBUF_INIT; + + if (packet_get_line(&buffer, &last->buf, &last->len) <= 0) + die("%s has invalid packet header", refs_url); + if (buffer.len && buffer.buf[buffer.len - 1] == '\n') + strbuf_setlen(&buffer, buffer.len - 1); + + strbuf_addf(&exp, "# service=%s", service); + if (strbuf_cmp(&exp, &buffer)) + die("invalid server response; got '%s'", buffer.buf); + strbuf_release(&exp); + + /* The header can include additional metadata lines, up + * until a packet flush marker. Ignore these now, but + * in the future we might start to scan them. + */ + strbuf_reset(&buffer); + while (packet_get_line(&buffer, &last->buf, &last->len) > 0) + strbuf_reset(&buffer); + + last->proto_git = 1; + } + + free(refs_url); + strbuf_release(&buffer); + last_discovery = last; + return last; +} + +static int write_discovery(int fd, void *data) +{ + struct discovery *heads = data; + int err = 0; + if (write_in_full(fd, heads->buf, heads->len) != heads->len) + err = 1; + close(fd); + return err; +} + +static struct ref *parse_git_refs(struct discovery *heads) +{ + struct ref *list = NULL; + struct async async; + + memset(&async, 0, sizeof(async)); + async.proc = write_discovery; + async.data = heads; + + if (start_async(&async)) + die("cannot start thread to parse advertised refs"); + get_remote_heads(async.out, &list, 0, NULL, 0, NULL); + close(async.out); + if (finish_async(&async)) + die("ref parsing thread failed"); + return list; +} + +static struct ref *parse_info_refs(struct discovery *heads) +{ + char *data, *start, *mid; + char *ref_name; + int i = 0; + + struct ref *refs = NULL; + struct ref *ref = NULL; + struct ref *last_ref = NULL; + + data = heads->buf; start = NULL; mid = data; - while (i < buffer.len) { + while (i < heads->len) { if (!start) { start = &data[i]; } @@ -131,8 +233,7 @@ static struct ref *get_refs(void) i++; } - strbuf_release(&buffer); - + init_walker(); ref = alloc_ref("HEAD"); if (!walker->fetch_ref(walker, ref) && !resolve_remote_symref(ref, refs)) { @@ -142,11 +243,23 @@ static struct ref *get_refs(void) free(ref); } - strbuf_release(&buffer); - free(refs_url); return refs; } +static struct ref *get_refs(int for_push) +{ + struct discovery *heads; + + if (for_push) + heads = discover_refs("git-receive-pack"); + else + heads = discover_refs("git-upload-pack"); + + if (heads->proto_git) + return parse_git_refs(heads); + return parse_info_refs(heads); +} + static void output_refs(struct ref *refs) { struct ref *posn; @@ -317,7 +430,8 @@ int main(int argc, const char **argv) parse_fetch(&buf); } else if (!strcmp(buf.buf, "list") || !prefixcmp(buf.buf, "list ")) { - output_refs(get_refs()); + int for_push = !!strstr(buf.buf + 4, "for-push"); + output_refs(get_refs(for_push)); } else if (!prefixcmp(buf.buf, "push ")) { parse_push(&buf); -- cgit v1.3-5-g9baa From de1a2fdd38b138c4e4fed6412783dcb74d63d2da Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 30 Oct 2009 17:47:41 -0700 Subject: Smart push over HTTP: client side The git-remote-curl backend detects if the remote server supports the git-receive-pack service, and if so, runs git-send-pack in a pipe to dump the command and pack data as a single POST request. The advertisements from the server that were obtained during the discovery are passed into git-send-pack before the POST request starts. This permits git-send-pack to operate largely unmodified. For smaller packs (those under 1 MiB) a HTTP/1.0 POST with a Content-Length is used, permitting interaction with any server. The 1 MiB limit is arbitrary, but is sufficent to fit most deltas created by human authors against text sources with the occasional small binary file (e.g. few KiB icon image). The configuration option http.postBuffer can be used to increase (or shink) this buffer if the default is not sufficient. For larger packs which cannot be spooled entirely into the helper's memory space (due to http.postBuffer being too small), the POST request requires HTTP/1.1 and sets "Transfer-Encoding: chunked". This permits the client to upload an unknown amount of data in one HTTP transaction without needing to pregenerate the entire pack file locally. Signed-off-by: Shawn O. Pearce CC: Daniel Barkalow Signed-off-by: Junio C Hamano --- Documentation/config.txt | 8 ++ builtin-send-pack.c | 116 +++++++++++++++++++++-- http.c | 13 ++- http.h | 2 + remote-curl.c | 235 ++++++++++++++++++++++++++++++++++++++++++++++- send-pack.h | 3 +- sideband.c | 11 ++- transport.c | 1 + 8 files changed, 374 insertions(+), 15 deletions(-) (limited to 'remote-curl.c') diff --git a/Documentation/config.txt b/Documentation/config.txt index cd1781498e..7130d0723a 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -1089,6 +1089,14 @@ http.maxRequests:: How many HTTP requests to launch in parallel. Can be overridden by the 'GIT_HTTP_MAX_REQUESTS' environment variable. Default is 5. +http.postBuffer:: + Maximum size in bytes of the buffer used by smart HTTP + transports when POSTing data to the remote system. + For requests larger than this buffer size, HTTP/1.1 and + Transfer-Encoding: chunked is used to avoid creating a + massive pack file locally. Default is 1 MiB, which is + sufficient for most requests. + http.lowSpeedLimit, http.lowSpeedTime:: If the HTTP transfer speed is less than 'http.lowSpeedLimit' for longer than 'http.lowSpeedTime' seconds, the transfer is aborted. diff --git a/builtin-send-pack.c b/builtin-send-pack.c index 37e528e283..a0fbad0f61 100644 --- a/builtin-send-pack.c +++ b/builtin-send-pack.c @@ -2,9 +2,11 @@ #include "commit.h" #include "refs.h" #include "pkt-line.h" +#include "sideband.h" #include "run-command.h" #include "remote.h" #include "send-pack.h" +#include "quote.h" static const char send_pack_usage[] = "git send-pack [--all | --mirror] [--dry-run] [--force] [--receive-pack=] [--verbose] [--thin] [:] [...]\n" @@ -59,7 +61,7 @@ static int pack_objects(int fd, struct ref *refs, struct extra_have_objects *ext memset(&po, 0, sizeof(po)); po.argv = argv; po.in = -1; - po.out = fd; + po.out = args->stateless_rpc ? -1 : fd; po.git_cmd = 1; if (start_command(&po)) die_errno("git pack-objects failed"); @@ -83,6 +85,20 @@ static int pack_objects(int fd, struct ref *refs, struct extra_have_objects *ext } close(po.in); + + if (args->stateless_rpc) { + char *buf = xmalloc(LARGE_PACKET_MAX); + while (1) { + ssize_t n = xread(po.out, buf, LARGE_PACKET_MAX); + if (n <= 0) + break; + send_sideband(fd, -1, buf, n, LARGE_PACKET_MAX); + } + free(buf); + close(po.out); + po.out = -1; + } + if (finish_command(&po)) return error("pack-objects died with strange error"); return 0; @@ -303,6 +319,59 @@ static int refs_pushed(struct ref *ref) return 0; } +static void print_helper_status(struct ref *ref) +{ + struct strbuf buf = STRBUF_INIT; + + for (; ref; ref = ref->next) { + const char *msg = NULL; + const char *res; + + switch(ref->status) { + case REF_STATUS_NONE: + res = "error"; + msg = "no match"; + break; + + case REF_STATUS_OK: + res = "ok"; + break; + + case REF_STATUS_UPTODATE: + res = "ok"; + msg = "up to date"; + break; + + case REF_STATUS_REJECT_NONFASTFORWARD: + res = "error"; + msg = "non-fast forward"; + break; + + case REF_STATUS_REJECT_NODELETE: + case REF_STATUS_REMOTE_REJECT: + res = "error"; + break; + + case REF_STATUS_EXPECTING_REPORT: + default: + continue; + } + + strbuf_reset(&buf); + strbuf_addf(&buf, "%s %s", res, ref->name); + if (ref->remote_status) + msg = ref->remote_status; + if (msg) { + strbuf_addch(&buf, ' '); + quote_two_c_style(&buf, "", msg, 0); + } + strbuf_addch(&buf, '\n'); + + safe_write(1, buf.buf, buf.len); + } + strbuf_release(&buf); +} + int send_pack(struct send_pack_args *args, int fd[], struct child_process *conn, struct ref *remote_refs, @@ -310,6 +379,7 @@ int send_pack(struct send_pack_args *args, { int in = fd[0]; int out = fd[1]; + struct strbuf req_buf = STRBUF_INIT; struct ref *ref; int new_refs; int ask_for_status_report = 0; @@ -391,14 +461,14 @@ int send_pack(struct send_pack_args *args, char *new_hex = sha1_to_hex(ref->new_sha1); if (ask_for_status_report) { - packet_write(out, "%s %s %s%c%s", + packet_buf_write(&req_buf, "%s %s %s%c%s", old_hex, new_hex, ref->name, 0, "report-status"); ask_for_status_report = 0; expect_status_report = 1; } else - packet_write(out, "%s %s %s", + packet_buf_write(&req_buf, "%s %s %s", old_hex, new_hex, ref->name); } ref->status = expect_status_report ? @@ -406,7 +476,17 @@ int send_pack(struct send_pack_args *args, REF_STATUS_OK; } - packet_flush(out); + if (args->stateless_rpc) { + if (!args->dry_run) { + packet_buf_flush(&req_buf); + send_sideband(out, -1, req_buf.buf, req_buf.len, LARGE_PACKET_MAX); + } + } else { + safe_write(out, req_buf.buf, req_buf.len); + packet_flush(out); + } + strbuf_release(&req_buf); + if (new_refs && !args->dry_run) { if (pack_objects(out, remote_refs, extra_have, args) < 0) { for (ref = remote_refs; ref; ref = ref->next) @@ -414,11 +494,15 @@ int send_pack(struct send_pack_args *args, return -1; } } + if (args->stateless_rpc && !args->dry_run) + packet_flush(out); if (expect_status_report) ret = receive_status(in, remote_refs); else ret = 0; + if (args->stateless_rpc) + packet_flush(out); if (ret < 0) return ret; @@ -478,6 +562,7 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) struct extra_have_objects extra_have; struct ref *remote_refs, *local_refs; int ret; + int helper_status = 0; int send_all = 0; const char *receivepack = "git-receive-pack"; int flags; @@ -523,6 +608,14 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) args.use_thin_pack = 1; continue; } + if (!strcmp(arg, "--stateless-rpc")) { + args.stateless_rpc = 1; + continue; + } + if (!strcmp(arg, "--helper-status")) { + helper_status = 1; + continue; + } usage(send_pack_usage); } if (!dest) { @@ -551,7 +644,14 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) } } - conn = git_connect(fd, dest, receivepack, args.verbose ? CONNECT_VERBOSE : 0); + if (args.stateless_rpc) { + conn = NULL; + fd[0] = 0; + fd[1] = 1; + } else { + conn = git_connect(fd, dest, receivepack, + args.verbose ? CONNECT_VERBOSE : 0); + } memset(&extra_have, 0, sizeof(extra_have)); @@ -575,12 +675,16 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) ret = send_pack(&args, fd, conn, remote_refs, &extra_have); + if (helper_status) + print_helper_status(remote_refs); + close(fd[1]); close(fd[0]); ret |= finish_connect(conn); - print_push_status(dest, remote_refs); + if (!helper_status) + print_push_status(dest, remote_refs); if (!args.dry_run && remote) { struct ref *ref; diff --git a/http.c b/http.c index 23b2a1932c..ed6414a2aa 100644 --- a/http.c +++ b/http.c @@ -1,9 +1,11 @@ #include "http.h" #include "pack.h" +#include "sideband.h" int data_received; int active_requests; int http_is_verbose; +size_t http_post_buffer = 16 * LARGE_PACKET_MAX; #ifdef USE_CURL_MULTI static int max_requests = -1; @@ -97,8 +99,6 @@ size_t fwrite_null(const void *ptr, size_t eltsize, size_t nmemb, void *strbuf) return eltsize * nmemb; } -static void finish_active_slot(struct active_request_slot *slot); - #ifdef USE_CURL_MULTI static void process_curl_messages(void) { @@ -174,6 +174,13 @@ static int http_options(const char *var, const char *value, void *cb) if (!strcmp("http.proxy", var)) return git_config_string(&curl_http_proxy, var, value); + if (!strcmp("http.postbuffer", var)) { + http_post_buffer = git_config_int(var, value); + if (http_post_buffer < LARGE_PACKET_MAX) + http_post_buffer = LARGE_PACKET_MAX; + return 0; + } + /* Fall back on the default ones */ return git_default_config(var, value, cb); } @@ -638,7 +645,7 @@ void release_active_slot(struct active_request_slot *slot) #endif } -static void finish_active_slot(struct active_request_slot *slot) +void finish_active_slot(struct active_request_slot *slot) { closedown_active_slot(slot); curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code); diff --git a/http.h b/http.h index 4c4e99c2f6..f828e1d806 100644 --- a/http.h +++ b/http.h @@ -79,6 +79,7 @@ extern curlioerr ioctl_buffer(CURL *handle, int cmd, void *clientp); extern struct active_request_slot *get_active_slot(void); extern int start_active_slot(struct active_request_slot *slot); extern void run_active_slot(struct active_request_slot *slot); +extern void finish_active_slot(struct active_request_slot *slot); extern void finish_all_active_slots(void); extern void release_active_slot(struct active_request_slot *slot); @@ -94,6 +95,7 @@ extern void http_cleanup(void); extern int data_received; extern int active_requests; extern int http_is_verbose; +extern size_t http_post_buffer; extern char curl_errorstr[CURL_ERROR_SIZE]; diff --git a/remote-curl.c b/remote-curl.c index 3917d45deb..f1206cb572 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -6,6 +6,7 @@ #include "exec_cmd.h" #include "run-command.h" #include "pkt-line.h" +#include "sideband.h" static struct remote *remote; static const char *url; @@ -16,7 +17,8 @@ struct options { unsigned long depth; unsigned progress : 1, followtags : 1, - dry_run : 1; + dry_run : 1, + thin : 1; }; static struct options options; @@ -274,6 +276,188 @@ static void output_refs(struct ref *refs) free_refs(refs); } +struct rpc_state { + const char *service_name; + const char **argv; + char *service_url; + char *hdr_content_type; + char *hdr_accept; + char *buf; + size_t alloc; + size_t len; + size_t pos; + int in; + int out; + struct strbuf result; +}; + +static size_t rpc_out(void *ptr, size_t eltsize, + size_t nmemb, void *buffer_) +{ + size_t max = eltsize * nmemb; + struct rpc_state *rpc = buffer_; + size_t avail = rpc->len - rpc->pos; + + if (!avail) { + avail = packet_read_line(rpc->out, rpc->buf, rpc->alloc); + if (!avail) + return 0; + rpc->pos = 0; + rpc->len = avail; + } + + if (max < avail); + avail = max; + memcpy(ptr, rpc->buf + rpc->pos, avail); + rpc->pos += avail; + return avail; +} + +static size_t rpc_in(const void *ptr, size_t eltsize, + size_t nmemb, void *buffer_) +{ + size_t size = eltsize * nmemb; + struct rpc_state *rpc = buffer_; + write_or_die(rpc->in, ptr, size); + return size; +} + +static int post_rpc(struct rpc_state *rpc) +{ + struct active_request_slot *slot; + struct slot_results results; + struct curl_slist *headers = NULL; + int err = 0, large_request = 0; + + /* Try to load the entire request, if we can fit it into the + * allocated buffer space we can use HTTP/1.0 and avoid the + * chunked encoding mess. + */ + while (1) { + size_t left = rpc->alloc - rpc->len; + char *buf = rpc->buf + rpc->len; + int n; + + if (left < LARGE_PACKET_MAX) { + large_request = 1; + break; + } + + n = packet_read_line(rpc->out, buf, left); + if (!n) + break; + rpc->len += n; + } + + slot = get_active_slot(); + slot->results = &results; + + curl_easy_setopt(slot->curl, CURLOPT_POST, 1); + curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0); + curl_easy_setopt(slot->curl, CURLOPT_URL, rpc->service_url); + + headers = curl_slist_append(headers, rpc->hdr_content_type); + headers = curl_slist_append(headers, rpc->hdr_accept); + + if (large_request) { + /* The request body is large and the size cannot be predicted. + * We must use chunked encoding to send it. + */ + headers = curl_slist_append(headers, "Expect: 100-continue"); + headers = curl_slist_append(headers, "Transfer-Encoding: chunked"); + curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, rpc_out); + curl_easy_setopt(slot->curl, CURLOPT_INFILE, rpc); + if (options.verbosity > 1) { + fprintf(stderr, "POST %s (chunked)\n", rpc->service_name); + fflush(stderr); + } + + } else { + /* We know the complete request size in advance, use the + * more normal Content-Length approach. + */ + curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, rpc->buf); + curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, rpc->len); + if (options.verbosity > 1) { + fprintf(stderr, "POST %s (%lu bytes)\n", + rpc->service_name, (unsigned long)rpc->len); + fflush(stderr); + } + } + + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, rpc_in); + curl_easy_setopt(slot->curl, CURLOPT_FILE, rpc); + + slot->curl_result = curl_easy_perform(slot->curl); + finish_active_slot(slot); + + if (results.curl_result != CURLE_OK) { + err |= error("RPC failed; result=%d, HTTP code = %ld", + results.curl_result, results.http_code); + } + + curl_slist_free_all(headers); + return err; +} + +static int rpc_service(struct rpc_state *rpc, struct discovery *heads) +{ + const char *svc = rpc->service_name; + struct strbuf buf = STRBUF_INIT; + struct child_process client; + int err = 0; + + init_walker(); + memset(&client, 0, sizeof(client)); + client.in = -1; + client.out = -1; + client.git_cmd = 1; + client.argv = rpc->argv; + if (start_command(&client)) + exit(1); + if (heads) + write_or_die(client.in, heads->buf, heads->len); + + rpc->alloc = http_post_buffer; + rpc->buf = xmalloc(rpc->alloc); + rpc->in = client.in; + rpc->out = client.out; + strbuf_init(&rpc->result, 0); + + strbuf_addf(&buf, "%s/%s", url, svc); + rpc->service_url = strbuf_detach(&buf, NULL); + + strbuf_addf(&buf, "Content-Type: application/x-%s-request", svc); + rpc->hdr_content_type = strbuf_detach(&buf, NULL); + + strbuf_addf(&buf, "Accept: application/x-%s-response", svc); + rpc->hdr_accept = strbuf_detach(&buf, NULL); + + while (!err) { + int n = packet_read_line(rpc->out, rpc->buf, rpc->alloc); + if (!n) + break; + rpc->pos = 0; + rpc->len = n; + err |= post_rpc(rpc); + } + strbuf_read(&rpc->result, client.out, 0); + + close(client.in); + close(client.out); + client.in = -1; + client.out = -1; + + err |= finish_command(&client); + free(rpc->service_url); + free(rpc->hdr_content_type); + free(rpc->hdr_accept); + free(rpc->buf); + strbuf_release(&buf); + return err; +} + static int fetch_dumb(int nr_heads, struct ref **to_fetch) { char **targets = xmalloc(nr_heads * sizeof(char*)); @@ -371,6 +555,52 @@ static int push_dav(int nr_spec, char **specs) return 0; } +static int push_git(struct discovery *heads, int nr_spec, char **specs) +{ + struct rpc_state rpc; + const char **argv; + int argc = 0, i, err; + + argv = xmalloc((10 + nr_spec) * sizeof(char*)); + argv[argc++] = "send-pack"; + argv[argc++] = "--stateless-rpc"; + argv[argc++] = "--helper-status"; + if (options.thin) + argv[argc++] = "--thin"; + if (options.dry_run) + argv[argc++] = "--dry-run"; + if (options.verbosity > 1) + argv[argc++] = "--verbose"; + argv[argc++] = url; + for (i = 0; i < nr_spec; i++) + argv[argc++] = specs[i]; + argv[argc++] = NULL; + + memset(&rpc, 0, sizeof(rpc)); + rpc.service_name = "git-receive-pack", + rpc.argv = argv; + + err = rpc_service(&rpc, heads); + if (rpc.result.len) + safe_write(1, rpc.result.buf, rpc.result.len); + strbuf_release(&rpc.result); + free(argv); + return err; +} + +static int push(int nr_spec, char **specs) +{ + struct discovery *heads = discover_refs("git-receive-pack"); + int ret; + + if (heads->proto_git) + ret = push_git(heads, nr_spec, specs); + else + ret = push_dav(nr_spec, specs); + free_discovery(heads); + return ret; +} + static void parse_push(struct strbuf *buf) { char **specs = NULL; @@ -391,7 +621,7 @@ static void parse_push(struct strbuf *buf) break; } while (1); - if (push_dav(nr_spec, specs)) + if (push(nr_spec, specs)) exit(128); /* error already reported */ for (i = 0; i < nr_spec; i++) free(specs[i]); @@ -414,6 +644,7 @@ int main(int argc, const char **argv) options.verbosity = 1; options.progress = !!isatty(2); + options.thin = 1; remote = remote_get(argv[1]); diff --git a/send-pack.h b/send-pack.h index 8b3cf028ed..28141ac913 100644 --- a/send-pack.h +++ b/send-pack.h @@ -8,7 +8,8 @@ struct send_pack_args { force_update:1, use_thin_pack:1, use_ofs_delta:1, - dry_run:1; + dry_run:1, + stateless_rpc:1; }; int send_pack(struct send_pack_args *args, diff --git a/sideband.c b/sideband.c index 899b1ff366..d5ffa1c891 100644 --- a/sideband.c +++ b/sideband.c @@ -135,9 +135,14 @@ ssize_t send_sideband(int fd, int band, const char *data, ssize_t sz, int packet n = sz; if (packet_max - 5 < n) n = packet_max - 5; - sprintf(hdr, "%04x", n + 5); - hdr[4] = band; - safe_write(fd, hdr, 5); + if (0 <= band) { + sprintf(hdr, "%04x", n + 5); + hdr[4] = band; + safe_write(fd, hdr, 5); + } else { + sprintf(hdr, "%04x", n + 4); + safe_write(fd, hdr, 4); + } safe_write(fd, p, n); p += n; sz -= n; diff --git a/transport.c b/transport.c index 6d9652db94..2ff16503be 100644 --- a/transport.c +++ b/transport.c @@ -731,6 +731,7 @@ static int git_transport_push(struct transport *transport, struct ref *remote_re NULL); } + memset(&args, 0, sizeof(args)); args.send_mirror = !!(flags & TRANSPORT_PUSH_MIRROR); args.force_update = !!(flags & TRANSPORT_PUSH_FORCE); args.use_thin_pack = data->thin; -- cgit v1.3-5-g9baa From 249b2004d8c9c58ed1ea1665dfd376af0312ed7e Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 30 Oct 2009 17:47:42 -0700 Subject: Smart fetch over HTTP: client side The git-remote-curl backend detects if the remote server supports the git-upload-pack service, and if so, runs git-fetch-pack locally in a pipe to generate the want/have commands. The advertisements from the server that were obtained during the discovery are passed into git-fetch-pack before the POST request starts, permitting server capability discovery and enablement. Common objects that are discovered are appended onto the request as have lines and are sent again on the next request. This allows the remote side to reinitialize its in-memory list of common objects during the next request. Because all requests are relatively short, below git-remote-curl's 1 MiB buffer limit, requests will use the standard Content-Length header and be valid HTTP/1.0 POST requests. This makes the fetch client more tolerant of proxy servers which don't support HTTP/1.1 or the chunked transfer encoding. Signed-off-by: Shawn O. Pearce CC: Daniel Barkalow Signed-off-by: Junio C Hamano --- builtin-fetch-pack.c | 110 +++++++++++++++++++++++++++++++++++++++++++-------- fetch-pack.h | 3 +- remote-curl.c | 69 ++++++++++++++++++++++++++++++-- 3 files changed, 160 insertions(+), 22 deletions(-) (limited to 'remote-curl.c') diff --git a/builtin-fetch-pack.c b/builtin-fetch-pack.c index 615f54974e..8ed4a6feaa 100644 --- a/builtin-fetch-pack.c +++ b/builtin-fetch-pack.c @@ -165,6 +165,24 @@ enum ack_type { ACK_ready }; +static void consume_shallow_list(int fd) +{ + if (args.stateless_rpc && args.depth > 0) { + /* If we sent a depth we will get back "duplicate" + * shallow and unshallow commands every time there + * is a block of have lines exchanged. + */ + char line[1000]; + while (packet_read_line(fd, line, sizeof(line))) { + if (!prefixcmp(line, "shallow ")) + continue; + if (!prefixcmp(line, "unshallow ")) + continue; + die("git fetch-pack: expected shallow list"); + } + } +} + static enum ack_type get_ack(int fd, unsigned char *result_sha1) { static char line[1000]; @@ -190,6 +208,15 @@ static enum ack_type get_ack(int fd, unsigned char *result_sha1) die("git fetch_pack: expected ACK/NAK, got '%s'", line); } +static void send_request(int fd, struct strbuf *buf) +{ + if (args.stateless_rpc) { + send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX); + packet_flush(fd); + } else + safe_write(fd, buf->buf, buf->len); +} + static int find_common(int fd[2], unsigned char *result_sha1, struct ref *refs) { @@ -199,7 +226,10 @@ static int find_common(int fd[2], unsigned char *result_sha1, unsigned in_vain = 0; int got_continue = 0; struct strbuf req_buf = STRBUF_INIT; + size_t state_len = 0; + if (args.stateless_rpc && multi_ack == 1) + die("--stateless-rpc requires multi_ack_detailed"); if (marked) for_each_ref(clear_marks, NULL); marked = 1; @@ -256,13 +286,13 @@ static int find_common(int fd[2], unsigned char *result_sha1, if (args.depth > 0) packet_buf_write(&req_buf, "deepen %d", args.depth); packet_buf_flush(&req_buf); - - safe_write(fd[1], req_buf.buf, req_buf.len); + state_len = req_buf.len; if (args.depth > 0) { char line[1024]; unsigned char sha1[20]; + send_request(fd[1], &req_buf); while (packet_read_line(fd[0], line, sizeof(line))) { if (!prefixcmp(line, "shallow ")) { if (get_sha1_hex(line + 8, sha1)) @@ -284,28 +314,40 @@ static int find_common(int fd[2], unsigned char *result_sha1, } die("expected shallow/unshallow, got %s", line); } + } else if (!args.stateless_rpc) + send_request(fd[1], &req_buf); + + if (!args.stateless_rpc) { + /* If we aren't using the stateless-rpc interface + * we don't need to retain the headers. + */ + strbuf_setlen(&req_buf, 0); + state_len = 0; } flushes = 0; retval = -1; while ((sha1 = get_rev())) { - packet_write(fd[1], "have %s\n", sha1_to_hex(sha1)); + packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1)); if (args.verbose) fprintf(stderr, "have %s\n", sha1_to_hex(sha1)); in_vain++; if (!(31 & ++count)) { int ack; - packet_flush(fd[1]); + packet_buf_flush(&req_buf); + send_request(fd[1], &req_buf); + strbuf_setlen(&req_buf, state_len); flushes++; /* * We keep one window "ahead" of the other side, and * will wait for an ACK only on the next one */ - if (count == 32) + if (!args.stateless_rpc && count == 32) continue; + consume_shallow_list(fd[0]); do { ack = get_ack(fd[0], result_sha1); if (args.verbose && ack) @@ -322,6 +364,17 @@ static int find_common(int fd[2], unsigned char *result_sha1, case ACK_continue: { struct commit *commit = lookup_commit(result_sha1); + if (args.stateless_rpc + && ack == ACK_common + && !(commit->object.flags & COMMON)) { + /* We need to replay the have for this object + * on the next RPC request so the peer knows + * it is in common with us. + */ + const char *hex = sha1_to_hex(result_sha1); + packet_buf_write(&req_buf, "have %s\n", hex); + state_len = req_buf.len; + } mark_common(commit, 0, 1); retval = 0; in_vain = 0; @@ -339,7 +392,8 @@ static int find_common(int fd[2], unsigned char *result_sha1, } } done: - packet_write(fd[1], "done\n"); + packet_buf_write(&req_buf, "done\n"); + send_request(fd[1], &req_buf); if (args.verbose) fprintf(stderr, "done\n"); if (retval != 0) { @@ -348,6 +402,7 @@ done: } strbuf_release(&req_buf); + consume_shallow_list(fd[0]); while (flushes || multi_ack) { int ack = get_ack(fd[0], result_sha1); if (ack) { @@ -672,6 +727,8 @@ static struct ref *do_fetch_pack(int fd[2], */ warning("no common commits"); + if (args.stateless_rpc) + packet_flush(fd[1]); if (get_pack(fd, pack_lockfile)) die("git fetch-pack: fetch failed."); @@ -742,6 +799,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) struct ref *ref = NULL; char *dest = NULL, **heads; int fd[2]; + char *pack_lockfile = NULL; + char **pack_lockfile_ptr = NULL; struct child_process *conn; nr_heads = 0; @@ -791,6 +850,15 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) args.no_progress = 1; continue; } + if (!strcmp("--stateless-rpc", arg)) { + args.stateless_rpc = 1; + continue; + } + if (!strcmp("--lock-pack", arg)) { + args.lock_pack = 1; + pack_lockfile_ptr = &pack_lockfile; + continue; + } usage(fetch_pack_usage); } dest = (char *)arg; @@ -801,19 +869,27 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) if (!dest) usage(fetch_pack_usage); - conn = git_connect(fd, (char *)dest, args.uploadpack, - args.verbose ? CONNECT_VERBOSE : 0); - if (conn) { - get_remote_heads(fd[0], &ref, 0, NULL, 0, NULL); - - ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL); - close(fd[0]); - close(fd[1]); - if (finish_connect(conn)) - ref = NULL; + if (args.stateless_rpc) { + conn = NULL; + fd[0] = 0; + fd[1] = 1; } else { - ref = NULL; + conn = git_connect(fd, (char *)dest, args.uploadpack, + args.verbose ? CONNECT_VERBOSE : 0); + } + + get_remote_heads(fd[0], &ref, 0, NULL, 0, NULL); + + ref = fetch_pack(&args, fd, conn, ref, dest, + nr_heads, heads, pack_lockfile_ptr); + if (pack_lockfile) { + printf("lock %s\n", pack_lockfile); + fflush(stdout); } + close(fd[0]); + close(fd[1]); + if (finish_connect(conn)) + ref = NULL; ret = !ref; if (!ret && nr_heads) { diff --git a/fetch-pack.h b/fetch-pack.h index 8bd9c32561..fbe85ac05f 100644 --- a/fetch-pack.h +++ b/fetch-pack.h @@ -13,7 +13,8 @@ struct fetch_pack_args fetch_all:1, verbose:1, no_progress:1, - include_tag:1; + include_tag:1, + stateless_rpc:1; }; struct ref *fetch_pack(struct fetch_pack_args *args, diff --git a/remote-curl.c b/remote-curl.c index f1206cb572..0eb6fc48fb 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -45,7 +45,7 @@ static int set_option(const char *name, const char *value) options.progress = 0; else return -1; - return 1 /* TODO implement later */; + return 0; } else if (!strcmp(name, "depth")) { char *end; @@ -53,7 +53,7 @@ static int set_option(const char *name, const char *value) if (value == end || *end) return -1; options.depth = v; - return 1 /* TODO implement later */; + return 0; } else if (!strcmp(name, "followtags")) { if (!strcmp(value, "true")) @@ -62,7 +62,7 @@ static int set_option(const char *name, const char *value) options.followtags = 0; else return -1; - return 1 /* TODO implement later */; + return 0; } else if (!strcmp(name, "dry-run")) { if (!strcmp(value, "true")) @@ -463,6 +463,8 @@ static int fetch_dumb(int nr_heads, struct ref **to_fetch) char **targets = xmalloc(nr_heads * sizeof(char*)); int ret, i; + if (options.depth) + die("dumb http transport does not support --depth"); for (i = 0; i < nr_heads; i++) targets[i] = xstrdup(sha1_to_hex(to_fetch[i]->old_sha1)); @@ -481,6 +483,65 @@ static int fetch_dumb(int nr_heads, struct ref **to_fetch) return ret ? error("Fetch failed.") : 0; } +static int fetch_git(struct discovery *heads, + int nr_heads, struct ref **to_fetch) +{ + struct rpc_state rpc; + char *depth_arg = NULL; + const char **argv; + int argc = 0, i, err; + + argv = xmalloc((15 + nr_heads) * sizeof(char*)); + argv[argc++] = "fetch-pack"; + argv[argc++] = "--stateless-rpc"; + argv[argc++] = "--lock-pack"; + if (options.followtags) + argv[argc++] = "--include-tag"; + if (options.thin) + argv[argc++] = "--thin"; + if (options.verbosity >= 3) { + argv[argc++] = "-v"; + argv[argc++] = "-v"; + } + if (!options.progress) + argv[argc++] = "--no-progress"; + if (options.depth) { + struct strbuf buf = STRBUF_INIT; + strbuf_addf(&buf, "--depth=%lu", options.depth); + depth_arg = strbuf_detach(&buf, NULL); + argv[argc++] = depth_arg; + } + argv[argc++] = url; + for (i = 0; i < nr_heads; i++) { + struct ref *ref = to_fetch[i]; + if (!ref->name || !*ref->name) + die("cannot fetch by sha1 over smart http"); + argv[argc++] = ref->name; + } + argv[argc++] = NULL; + + memset(&rpc, 0, sizeof(rpc)); + rpc.service_name = "git-upload-pack", + rpc.argv = argv; + + err = rpc_service(&rpc, heads); + if (rpc.result.len) + safe_write(1, rpc.result.buf, rpc.result.len); + strbuf_release(&rpc.result); + free(argv); + free(depth_arg); + return err; +} + +static int fetch(int nr_heads, struct ref **to_fetch) +{ + struct discovery *d = discover_refs("git-upload-pack"); + if (d->proto_git) + return fetch_git(d, nr_heads, to_fetch); + else + return fetch_dumb(nr_heads, to_fetch); +} + static void parse_fetch(struct strbuf *buf) { struct ref **to_fetch = NULL; @@ -523,7 +584,7 @@ static void parse_fetch(struct strbuf *buf) break; } while (1); - if (fetch_dumb(nr_heads, to_fetch)) + if (fetch(nr_heads, to_fetch)) exit(128); /* error already reported */ free_refs(list_head); free(to_fetch); -- cgit v1.3-5-g9baa From b8538603a34b13ccb1a3a096f461b8a35d7ea878 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 30 Oct 2009 17:47:43 -0700 Subject: Smart HTTP fetch: gzip requests The upload-pack requests are mostly plain text and they compress rather well. Deflating them with Content-Encoding: gzip can easily drop the size of the request by 50%, reducing the amount of data to transfer as we negotiate the common commits. Signed-off-by: Shawn O. Pearce CC: Daniel Barkalow Signed-off-by: Junio C Hamano --- remote-curl.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) (limited to 'remote-curl.c') diff --git a/remote-curl.c b/remote-curl.c index 0eb6fc48fb..0d7cf16e9c 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -289,6 +289,7 @@ struct rpc_state { int in; int out; struct strbuf result; + unsigned gzip_request : 1; }; static size_t rpc_out(void *ptr, size_t eltsize, @@ -327,6 +328,8 @@ static int post_rpc(struct rpc_state *rpc) struct active_request_slot *slot; struct slot_results results; struct curl_slist *headers = NULL; + int use_gzip = rpc->gzip_request; + char *gzip_body = NULL; int err = 0, large_request = 0; /* Try to load the entire request, if we can fit it into the @@ -340,6 +343,7 @@ static int post_rpc(struct rpc_state *rpc) if (left < LARGE_PACKET_MAX) { large_request = 1; + use_gzip = 0; break; } @@ -355,6 +359,7 @@ static int post_rpc(struct rpc_state *rpc) curl_easy_setopt(slot->curl, CURLOPT_POST, 1); curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0); curl_easy_setopt(slot->curl, CURLOPT_URL, rpc->service_url); + curl_easy_setopt(slot->curl, CURLOPT_ENCODING, ""); headers = curl_slist_append(headers, rpc->hdr_content_type); headers = curl_slist_append(headers, rpc->hdr_accept); @@ -372,6 +377,49 @@ static int post_rpc(struct rpc_state *rpc) fflush(stderr); } + } else if (use_gzip && 1024 < rpc->len) { + /* The client backend isn't giving us compressed data so + * we can try to deflate it ourselves, this may save on. + * the transfer time. + */ + size_t size; + z_stream stream; + int ret; + + memset(&stream, 0, sizeof(stream)); + ret = deflateInit2(&stream, Z_BEST_COMPRESSION, + Z_DEFLATED, (15 + 16), + 8, Z_DEFAULT_STRATEGY); + if (ret != Z_OK) + die("cannot deflate request; zlib init error %d", ret); + size = deflateBound(&stream, rpc->len); + gzip_body = xmalloc(size); + + stream.next_in = (unsigned char *)rpc->buf; + stream.avail_in = rpc->len; + stream.next_out = (unsigned char *)gzip_body; + stream.avail_out = size; + + ret = deflate(&stream, Z_FINISH); + if (ret != Z_STREAM_END) + die("cannot deflate request; zlib deflate error %d", ret); + + ret = deflateEnd(&stream); + if (ret != Z_OK) + die("cannot deflate request; zlib end error %d", ret); + + size = stream.total_out; + + headers = curl_slist_append(headers, "Content-Encoding: gzip"); + curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, gzip_body); + curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, size); + + if (options.verbosity > 1) { + fprintf(stderr, "POST %s (gzip %lu to %lu bytes)\n", + rpc->service_name, + (unsigned long)rpc->len, (unsigned long)size); + fflush(stderr); + } } else { /* We know the complete request size in advance, use the * more normal Content-Length approach. @@ -398,6 +446,7 @@ static int post_rpc(struct rpc_state *rpc) } curl_slist_free_all(headers); + free(gzip_body); return err; } @@ -523,6 +572,7 @@ static int fetch_git(struct discovery *heads, memset(&rpc, 0, sizeof(rpc)); rpc.service_name = "git-upload-pack", rpc.argv = argv; + rpc.gzip_request = 1; err = rpc_service(&rpc, heads); if (rpc.result.len) -- cgit v1.3-5-g9baa