From a1743343f410290578fbd6e0ada50b8cdf1e7df8 Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Fri, 8 Dec 2017 15:58:43 +0000 Subject: fetch: refactor calculation of remote list Separate out the calculation of remotes to be fetched from and the actual fetching. This will allow us to include an additional step before the actual fetching in a subsequent commit. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano --- builtin/fetch.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'builtin/fetch.c') diff --git a/builtin/fetch.c b/builtin/fetch.c index 225c734924..1b1f03923d 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -1322,7 +1322,7 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) { int i; struct string_list list = STRING_LIST_INIT_DUP; - struct remote *remote; + struct remote *remote = NULL; int result = 0; struct argv_array argv_gc_auto = ARGV_ARRAY_INIT; @@ -1367,17 +1367,14 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) else if (argc > 1) die(_("fetch --all does not make sense with refspecs")); (void) for_each_remote(get_one_remote_for_fetch, &list); - result = fetch_multiple(&list); } else if (argc == 0) { /* No arguments -- use default remote */ remote = remote_get(NULL); - result = fetch_one(remote, argc, argv); } else if (multiple) { /* All arguments are assumed to be remotes or groups */ for (i = 0; i < argc; i++) if (!add_remote_or_group(argv[i], &list)) die(_("No such remote or remote group: %s"), argv[i]); - result = fetch_multiple(&list); } else { /* Single remote or group */ (void) add_remote_or_group(argv[0], &list); @@ -1385,14 +1382,19 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) /* More than one remote */ if (argc > 1) die(_("Fetching a group and specifying refspecs does not make sense")); - result = fetch_multiple(&list); } else { /* Zero or one remotes */ remote = remote_get(argv[0]); - result = fetch_one(remote, argc-1, argv+1); + argc--; + argv++; } } + if (remote) + result = fetch_one(remote, argc, argv); + else + result = fetch_multiple(&list); + if (!result && (recurse_submodules != RECURSE_SUBMODULES_OFF)) { struct argv_array options = ARGV_ARRAY_INIT; -- cgit v1.3-5-g9baa From acb0c57260aa78fc99939de2a27c48b5a3fb4f21 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 8 Dec 2017 15:58:44 +0000 Subject: fetch: support filters Teach fetch to support filters. This is only allowed for the remote configured in extensions.partialcloneremote. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano --- builtin/fetch.c | 23 +++++++++++++++++++++-- connected.c | 2 ++ remote-curl.c | 6 ++++++ t/t5500-fetch-pack.sh | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 2 deletions(-) (limited to 'builtin/fetch.c') diff --git a/builtin/fetch.c b/builtin/fetch.c index 1b1f03923d..14aab71c2c 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -18,6 +18,7 @@ #include "argv-array.h" #include "utf8.h" #include "packfile.h" +#include "list-objects-filter-options.h" static const char * const builtin_fetch_usage[] = { N_("git fetch [] [ [...]]"), @@ -55,6 +56,7 @@ static int recurse_submodules_default = RECURSE_SUBMODULES_ON_DEMAND; static int shown_url = 0; static int refmap_alloc, refmap_nr; static const char **refmap_array; +static struct list_objects_filter_options filter_options; static int git_fetch_config(const char *k, const char *v, void *cb) { @@ -160,6 +162,7 @@ static struct option builtin_fetch_options[] = { TRANSPORT_FAMILY_IPV4), OPT_SET_INT('6', "ipv6", &family, N_("use IPv6 addresses only"), TRANSPORT_FAMILY_IPV6), + OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options), OPT_END() }; @@ -1044,6 +1047,11 @@ static struct transport *prepare_transport(struct remote *remote, int deepen) set_option(transport, TRANS_OPT_DEEPEN_RELATIVE, "yes"); if (update_shallow) set_option(transport, TRANS_OPT_UPDATE_SHALLOW, "yes"); + if (filter_options.choice) { + set_option(transport, TRANS_OPT_LIST_OBJECTS_FILTER, + filter_options.filter_spec); + set_option(transport, TRANS_OPT_FROM_PROMISOR, "1"); + } return transport; } @@ -1328,6 +1336,8 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) packet_trace_identity("fetch"); + fetch_if_missing = 0; + /* Record the command line for the reflog */ strbuf_addstr(&default_rla, "fetch"); for (i = 1; i < argc; i++) @@ -1361,6 +1371,9 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) if (depth || deepen_since || deepen_not.nr) deepen = 1; + if (filter_options.choice && !repository_format_partial_clone) + die("--filter can only be used when extensions.partialClone is set"); + if (all) { if (argc == 1) die(_("fetch --all does not take a repository argument")); @@ -1390,10 +1403,16 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) } } - if (remote) + if (remote) { + if (filter_options.choice && + strcmp(remote->name, repository_format_partial_clone)) + die(_("--filter can only be used with the remote configured in core.partialClone")); result = fetch_one(remote, argc, argv); - else + } else { + if (filter_options.choice) + die(_("--filter can only be used with the remote configured in core.partialClone")); result = fetch_multiple(&list); + } if (!result && (recurse_submodules != RECURSE_SUBMODULES_OFF)) { struct argv_array options = ARGV_ARRAY_INIT; diff --git a/connected.c b/connected.c index f416b05051..3a5bd67944 100644 --- a/connected.c +++ b/connected.c @@ -56,6 +56,8 @@ int check_connected(sha1_iterate_fn fn, void *cb_data, argv_array_push(&rev_list.args,"rev-list"); argv_array_push(&rev_list.args, "--objects"); argv_array_push(&rev_list.args, "--stdin"); + if (repository_format_partial_clone) + argv_array_push(&rev_list.args, "--exclude-promisor-objects"); argv_array_push(&rev_list.args, "--not"); argv_array_push(&rev_list.args, "--all"); argv_array_push(&rev_list.args, "--quiet"); diff --git a/remote-curl.c b/remote-curl.c index 431839111a..6ec5352435 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -24,6 +24,7 @@ struct options { char *deepen_since; struct string_list deepen_not; struct string_list push_options; + char *filter; unsigned progress : 1, check_self_contained_and_connected : 1, cloning : 1, @@ -165,6 +166,9 @@ static int set_option(const char *name, const char *value) } else if (!strcmp(name, "no-dependents")) { options.no_dependents = 1; return 0; + } else if (!strcmp(name, "filter")) { + options.filter = xstrdup(value);; + return 0; } else { return 1 /* unsupported */; } @@ -834,6 +838,8 @@ static int fetch_git(struct discovery *heads, argv_array_push(&args, "--from-promisor"); if (options.no_dependents) argv_array_push(&args, "--no-dependents"); + if (options.filter) + argv_array_pushf(&args, "--filter=%s", options.filter); argv_array_push(&args, url.buf); for (i = 0; i < nr_heads; i++) { diff --git a/t/t5500-fetch-pack.sh b/t/t5500-fetch-pack.sh index c57916b64a..ec9ba9bf6e 100755 --- a/t/t5500-fetch-pack.sh +++ b/t/t5500-fetch-pack.sh @@ -782,4 +782,40 @@ test_expect_success 'filtering by size has no effect if support for it is not ad test_i18ngrep "filtering not recognized by server" err ' +fetch_filter_blob_limit_zero () { + SERVER="$1" + URL="$2" + + rm -rf "$SERVER" client && + test_create_repo "$SERVER" && + test_commit -C "$SERVER" one && + test_config -C "$SERVER" uploadpack.allowfilter 1 && + + git clone "$URL" client && + test_config -C client extensions.partialclone origin && + + test_commit -C "$SERVER" two && + + git -C client fetch --filter=blob:limit=0 origin HEAD:somewhere && + + # Ensure that commit is fetched, but blob is not + test_config -C client extensions.partialclone "arbitrary string" && + git -C client cat-file -e $(git -C "$SERVER" rev-parse two) && + test_must_fail git -C client cat-file -e $(git hash-object "$SERVER/two.t") +} + +test_expect_success 'fetch with --filter=blob:limit=0' ' + fetch_filter_blob_limit_zero server server +' + +. "$TEST_DIRECTORY"/lib-httpd.sh +start_httpd + +test_expect_success 'fetch with --filter=blob:limit=0 and HTTP' ' + fetch_filter_blob_limit_zero "$HTTPD_DOCUMENT_ROOT_PATH/server" "$HTTPD_URL/smart/server" +' + +stop_httpd + + test_done -- cgit v1.3-5-g9baa From aa57b871dad3cca07abead9a8d1fefceffe7578d Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 8 Dec 2017 15:58:50 +0000 Subject: fetch: inherit filter-spec from partial clone Teach (partial) fetch to inherit the filter-spec used by the partial clone. Extend --no-filter to override this inheritance. Signed-off-by: Jeff Hostetler Signed-off-by: Junio C Hamano --- builtin/fetch-pack.c | 2 +- builtin/fetch.c | 56 ++++++++++++++++++++++++++++++++++++++++--- builtin/rev-list.c | 2 +- list-objects-filter-options.c | 2 +- list-objects-filter-options.h | 12 ++++++++++ t/t5616-partial-clone.sh | 22 ++++++++++++++++- 6 files changed, 89 insertions(+), 7 deletions(-) (limited to 'builtin/fetch.c') diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index cbf503537a..a7bc1366ab 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -158,7 +158,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) continue; } if (!strcmp(arg, ("--no-" CL_ARG__FILTER))) { - list_objects_filter_release(&args.filter_options); + list_objects_filter_set_no_filter(&args.filter_options); continue; } usage(fetch_pack_usage); diff --git a/builtin/fetch.c b/builtin/fetch.c index 14aab71c2c..79c866c225 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -1275,6 +1275,56 @@ static int fetch_multiple(struct string_list *list) return result; } +/* + * Fetching from the promisor remote should use the given filter-spec + * or inherit the default filter-spec from the config. + */ +static inline void fetch_one_setup_partial(struct remote *remote) +{ + /* + * Explicit --no-filter argument overrides everything, regardless + * of any prior partial clones and fetches. + */ + if (filter_options.no_filter) + return; + + /* + * If no prior partial clone/fetch and the current fetch DID NOT + * request a partial-fetch, do a normal fetch. + */ + if (!repository_format_partial_clone && !filter_options.choice) + return; + + /* + * If this is the FIRST partial-fetch request, we enable partial + * on this repo and remember the given filter-spec as the default + * for subsequent fetches to this remote. + */ + if (!repository_format_partial_clone && filter_options.choice) { + partial_clone_register(remote->name, &filter_options); + return; + } + + /* + * We are currently limited to only ONE promisor remote and only + * allow partial-fetches from the promisor remote. + */ + if (strcmp(remote->name, repository_format_partial_clone)) { + if (filter_options.choice) + die(_("--filter can only be used with the remote configured in core.partialClone")); + return; + } + + /* + * Do a partial-fetch from the promisor remote using either the + * explicitly given filter-spec or inherit the filter-spec from + * the config. + */ + if (!filter_options.choice) + partial_clone_get_default_filter_spec(&filter_options); + return; +} + static int fetch_one(struct remote *remote, int argc, const char **argv) { static const char **refs = NULL; @@ -1404,13 +1454,13 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) } if (remote) { - if (filter_options.choice && - strcmp(remote->name, repository_format_partial_clone)) - die(_("--filter can only be used with the remote configured in core.partialClone")); + if (filter_options.choice || repository_format_partial_clone) + fetch_one_setup_partial(remote); result = fetch_one(remote, argc, argv); } else { if (filter_options.choice) die(_("--filter can only be used with the remote configured in core.partialClone")); + /* TODO should this also die if we have a previous partial-clone? */ result = fetch_multiple(&list); } diff --git a/builtin/rev-list.c b/builtin/rev-list.c index 48f922d709..8503dea42d 100644 --- a/builtin/rev-list.c +++ b/builtin/rev-list.c @@ -460,7 +460,7 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix) continue; } if (!strcmp(arg, ("--no-" CL_ARG__FILTER))) { - list_objects_filter_release(&filter_options); + list_objects_filter_set_no_filter(&filter_options); continue; } if (!strcmp(arg, "--filter-print-omitted")) { diff --git a/list-objects-filter-options.c b/list-objects-filter-options.c index 5c47e2b002..6a3cc985c4 100644 --- a/list-objects-filter-options.c +++ b/list-objects-filter-options.c @@ -94,7 +94,7 @@ int opt_parse_list_objects_filter(const struct option *opt, struct list_objects_filter_options *filter_options = opt->value; if (unset || !arg) { - list_objects_filter_release(filter_options); + list_objects_filter_set_no_filter(filter_options); return 0; } diff --git a/list-objects-filter-options.h b/list-objects-filter-options.h index 11435392c2..0000a61f82 100644 --- a/list-objects-filter-options.h +++ b/list-objects-filter-options.h @@ -30,6 +30,11 @@ struct list_objects_filter_options { */ enum list_objects_filter_choice choice; + /* + * Choice is LOFC_DISABLED because "--no-filter" was requested. + */ + unsigned int no_filter : 1; + /* * Parsed values (fields) from within the filter-spec. These are * choice-specific; not all values will be defined for any given @@ -58,6 +63,13 @@ int opt_parse_list_objects_filter(const struct option *opt, void list_objects_filter_release( struct list_objects_filter_options *filter_options); +static inline void list_objects_filter_set_no_filter( + struct list_objects_filter_options *filter_options) +{ + list_objects_filter_release(filter_options); + filter_options->no_filter = 1; +} + void partial_clone_register( const char *remote, const struct list_objects_filter_options *filter_options); diff --git a/t/t5616-partial-clone.sh b/t/t5616-partial-clone.sh index 3b8cc0bba2..433e07e0cb 100755 --- a/t/t5616-partial-clone.sh +++ b/t/t5616-partial-clone.sh @@ -59,7 +59,7 @@ test_expect_success 'push new commits to server' ' git -C src remote add srv "file://$(pwd)/srv.bare" && for x in a b c d e do - echo "Mod $x" >>src/file.1.txt + echo "Mod file.1.txt $x" >>src/file.1.txt git -C src add file.1.txt git -C src commit -m "mod $x" done && @@ -93,4 +93,24 @@ test_expect_success 'verify blame causes dynamic object fetch' ' test_line_count = 0 observed ' +# create new commits in "src" repo to establish a history on file.2.txt +# and push to "srv.bare". +test_expect_success 'push new commits to server for file.2.txt' ' + for x in a b c d e f + do + echo "Mod file.2.txt $x" >>src/file.2.txt + git -C src add file.2.txt + git -C src commit -m "mod $x" + done && + git -C src push -u srv master +' + +# Do FULL fetch by disabling filter-spec using --no-filter. +# Verify we have all the new blobs. +test_expect_success 'override inherited filter-spec using --no-filter' ' + git -C pc1 fetch --no-filter origin && + git -C pc1 rev-list master..origin/master --quiet --objects --missing=print >observed && + test_line_count = 0 observed +' + test_done -- cgit v1.3-5-g9baa