From 8c6b5491182510f2344dae71784fcb5c3992d005 Mon Sep 17 00:00:00 2001 From: Stefan Beller Date: Mon, 29 Feb 2016 18:07:14 -0800 Subject: submodule update: direct error message to stderr Reroute the error message for specified but initialized submodules to stderr instead of stdout. Reviewed-by: Jonathan Nieder Signed-off-by: Stefan Beller Signed-off-by: Junio C Hamano --- git-submodule.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'git-submodule.sh') diff --git a/git-submodule.sh b/git-submodule.sh index 9bc5c5f94d..9ee86d4f5b 100755 --- a/git-submodule.sh +++ b/git-submodule.sh @@ -693,7 +693,7 @@ cmd_update() if test "$update_module" = "none" then - echo "Skipping submodule '$displaypath'" + echo >&2 "Skipping submodule '$displaypath'" continue fi @@ -702,7 +702,7 @@ cmd_update() # Only mention uninitialized submodules when its # path have been specified test "$#" != "0" && - say "$(eval_gettext "Submodule path '\$displaypath' not initialized + say >&2 "$(eval_gettext "Submodule path '\$displaypath' not initialized Maybe you want to use 'update --init'?")" continue fi -- cgit v1.3 From 48308681b072a1d32e1361c255347324a8ad151e Mon Sep 17 00:00:00 2001 From: Stefan Beller Date: Mon, 29 Feb 2016 18:07:17 -0800 Subject: git submodule update: have a dedicated helper for cloning This introduces a new helper function in git submodule--helper which takes care of cloning all submodules, which we want to parallelize eventually. Some tests (such as empty URL, update_mode=none) are required in the helper to make the decision for cloning. These checks have been moved into the C function as well (no need to repeat them in the shell script). Reviewed-by: Jonathan Nieder Signed-off-by: Stefan Beller Signed-off-by: Junio C Hamano --- builtin/submodule--helper.c | 246 ++++++++++++++++++++++++++++++++++++++++++++ git-submodule.sh | 47 +++------ 2 files changed, 259 insertions(+), 34 deletions(-) (limited to 'git-submodule.sh') diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index f4c3eff179..fd2b1684a7 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -255,6 +255,251 @@ static int module_clone(int argc, const char **argv, const char *prefix) return 0; } +struct submodule_update_clone { + /* index into 'list', the list of submodules to look into for cloning */ + int current; + struct module_list list; + unsigned warn_if_uninitialized : 1; + + /* update parameter passed via commandline */ + struct submodule_update_strategy update; + + /* configuration parameters which are passed on to the children */ + int quiet; + const char *reference; + const char *depth; + const char *recursive_prefix; + const char *prefix; + + /* Machine-readable status lines to be consumed by git-submodule.sh */ + struct string_list projectlines; + + /* If we want to stop as fast as possible and return an error */ + unsigned quickstop : 1; +}; +#define SUBMODULE_UPDATE_CLONE_INIT {0, MODULE_LIST_INIT, 0, \ + SUBMODULE_UPDATE_STRATEGY_INIT, 0, NULL, NULL, NULL, NULL, \ + STRING_LIST_INIT_DUP, 0} + +/** + * Determine whether 'ce' needs to be cloned. If so, prepare the 'child' to + * run the clone. Returns 1 if 'ce' needs to be cloned, 0 otherwise. + */ +static int prepare_to_clone_next_submodule(const struct cache_entry *ce, + struct child_process *child, + struct submodule_update_clone *suc, + struct strbuf *out) +{ + const struct submodule *sub = NULL; + struct strbuf displaypath_sb = STRBUF_INIT; + struct strbuf sb = STRBUF_INIT; + const char *displaypath = NULL; + char *url = NULL; + int needs_cloning = 0; + + if (ce_stage(ce)) { + if (suc->recursive_prefix) + strbuf_addf(&sb, "%s/%s", suc->recursive_prefix, ce->name); + else + strbuf_addf(&sb, "%s", ce->name); + strbuf_addf(out, _("Skipping unmerged submodule %s"), sb.buf); + strbuf_addch(out, '\n'); + goto cleanup; + } + + sub = submodule_from_path(null_sha1, ce->name); + + if (suc->recursive_prefix) + displaypath = relative_path(suc->recursive_prefix, + ce->name, &displaypath_sb); + else + displaypath = ce->name; + + if (suc->update.type == SM_UPDATE_NONE + || (suc->update.type == SM_UPDATE_UNSPECIFIED + && sub->update_strategy.type == SM_UPDATE_NONE)) { + strbuf_addf(out, _("Skipping submodule '%s'"), displaypath); + strbuf_addch(out, '\n'); + goto cleanup; + } + + /* + * Looking up the url in .git/config. + * We must not fall back to .gitmodules as we only want + * to process configured submodules. + */ + strbuf_reset(&sb); + strbuf_addf(&sb, "submodule.%s.url", sub->name); + git_config_get_string(sb.buf, &url); + if (!url) { + /* + * Only mention uninitialized submodules when their + * path have been specified + */ + if (suc->warn_if_uninitialized) { + strbuf_addf(out, + _("Submodule path '%s' not initialized"), + displaypath); + strbuf_addch(out, '\n'); + strbuf_addstr(out, + _("Maybe you want to use 'update --init'?")); + strbuf_addch(out, '\n'); + } + goto cleanup; + } + + strbuf_reset(&sb); + strbuf_addf(&sb, "%s/.git", ce->name); + needs_cloning = !file_exists(sb.buf); + + strbuf_reset(&sb); + strbuf_addf(&sb, "%06o %s %d %d\t%s\n", ce->ce_mode, + sha1_to_hex(ce->sha1), ce_stage(ce), + needs_cloning, ce->name); + string_list_append(&suc->projectlines, sb.buf); + + if (!needs_cloning) + goto cleanup; + + child->git_cmd = 1; + child->no_stdin = 1; + child->stdout_to_stderr = 1; + child->err = -1; + argv_array_push(&child->args, "submodule--helper"); + argv_array_push(&child->args, "clone"); + if (suc->quiet) + argv_array_push(&child->args, "--quiet"); + if (suc->prefix) + argv_array_pushl(&child->args, "--prefix", suc->prefix, NULL); + argv_array_pushl(&child->args, "--path", sub->path, NULL); + argv_array_pushl(&child->args, "--name", sub->name, NULL); + argv_array_pushl(&child->args, "--url", url, NULL); + if (suc->reference) + argv_array_push(&child->args, suc->reference); + if (suc->depth) + argv_array_push(&child->args, suc->depth); + +cleanup: + free(url); + strbuf_reset(&displaypath_sb); + strbuf_reset(&sb); + + return needs_cloning; +} + +static int update_clone_get_next_task(struct child_process *child, + struct strbuf *err, + void *suc_cb, + void **void_task_cb) +{ + struct submodule_update_clone *suc = suc_cb; + + for (; suc->current < suc->list.nr; suc->current++) { + const struct cache_entry *ce = suc->list.entries[suc->current]; + if (prepare_to_clone_next_submodule(ce, child, suc, err)) { + suc->current++; + return 1; + } + } + return 0; +} + +static int update_clone_start_failure(struct strbuf *err, + void *suc_cb, + void *void_task_cb) +{ + struct submodule_update_clone *suc = suc_cb; + suc->quickstop = 1; + return 1; +} + +static int update_clone_task_finished(int result, + struct strbuf *err, + void *suc_cb, + void *void_task_cb) +{ + struct submodule_update_clone *suc = suc_cb; + + if (!result) + return 0; + + suc->quickstop = 1; + return 1; +} + +static int update_clone(int argc, const char **argv, const char *prefix) +{ + const char *update = NULL; + struct string_list_item *item; + struct pathspec pathspec; + struct submodule_update_clone suc = SUBMODULE_UPDATE_CLONE_INIT; + + struct option module_update_clone_options[] = { + OPT_STRING(0, "prefix", &prefix, + N_("path"), + N_("path into the working tree")), + OPT_STRING(0, "recursive-prefix", &suc.recursive_prefix, + N_("path"), + N_("path into the working tree, across nested " + "submodule boundaries")), + OPT_STRING(0, "update", &update, + N_("string"), + N_("rebase, merge, checkout or none")), + OPT_STRING(0, "reference", &suc.reference, N_("repo"), + N_("reference repository")), + OPT_STRING(0, "depth", &suc.depth, "", + N_("Create a shallow clone truncated to the " + "specified number of revisions")), + OPT__QUIET(&suc.quiet, N_("don't print cloning progress")), + OPT_END() + }; + + const char *const git_submodule_helper_usage[] = { + N_("git submodule--helper update_clone [--prefix=] [...]"), + NULL + }; + suc.prefix = prefix; + + argc = parse_options(argc, argv, prefix, module_update_clone_options, + git_submodule_helper_usage, 0); + + if (update) + if (parse_submodule_update_strategy(update, &suc.update) < 0) + die(_("bad value for update parameter")); + + if (module_list_compute(argc, argv, prefix, &pathspec, &suc.list) < 0) + return 1; + + if (pathspec.nr) + suc.warn_if_uninitialized = 1; + + /* Overlay the parsed .gitmodules file with .git/config */ + gitmodules_config(); + git_config(submodule_config, NULL); + + run_processes_parallel(1, + update_clone_get_next_task, + update_clone_start_failure, + update_clone_task_finished, + &suc); + + /* + * We saved the output and put it out all at once now. + * That means: + * - the listener does not have to interleave their (checkout) + * work with our fetching. The writes involved in a + * checkout involve more straightforward sequential I/O. + * - the listener can avoid doing any work if fetching failed. + */ + if (suc.quickstop) + return 1; + + for_each_string_list_item(item, &suc.projectlines) + utf8_fprintf(stdout, "%s", item->string); + + return 0; +} + struct cmd_struct { const char *cmd; int (*fn)(int, const char **, const char *); @@ -264,6 +509,7 @@ static struct cmd_struct commands[] = { {"list", module_list}, {"name", module_name}, {"clone", module_clone}, + {"update-clone", update_clone} }; int cmd_submodule__helper(int argc, const char **argv, const char *prefix) diff --git a/git-submodule.sh b/git-submodule.sh index 9ee86d4f5b..a6a82d25aa 100755 --- a/git-submodule.sh +++ b/git-submodule.sh @@ -664,17 +664,20 @@ cmd_update() cmd_init "--" "$@" || return fi - cloned_modules= - git submodule--helper list --prefix "$wt_prefix" "$@" | { + { + git submodule--helper update-clone ${GIT_QUIET:+--quiet} \ + ${wt_prefix:+--prefix "$wt_prefix"} \ + ${prefix:+--recursive-prefix "$prefix"} \ + ${update:+--update "$update"} \ + ${reference:+--reference "$reference"} \ + ${depth:+--depth "$depth"} \ + "$@" || echo "#unmatched" + } | { err= - while read mode sha1 stage sm_path + while read mode sha1 stage just_cloned sm_path do die_if_unmatched "$mode" - if test "$stage" = U - then - echo >&2 "Skipping unmerged submodule $prefix$sm_path" - continue - fi + name=$(git submodule--helper name "$sm_path") || exit url=$(git config submodule."$name".url) branch=$(get_submodule_config "$name" branch master) @@ -691,27 +694,10 @@ cmd_update() displaypath=$(relative_path "$prefix$sm_path") - if test "$update_module" = "none" - then - echo >&2 "Skipping submodule '$displaypath'" - continue - fi - - if test -z "$url" - then - # Only mention uninitialized submodules when its - # path have been specified - test "$#" != "0" && - say >&2 "$(eval_gettext "Submodule path '\$displaypath' not initialized -Maybe you want to use 'update --init'?")" - continue - fi - - if ! test -d "$sm_path"/.git && ! test -f "$sm_path"/.git + if test $just_cloned -eq 1 then - git submodule--helper clone ${GIT_QUIET:+--quiet} --prefix "$prefix" --path "$sm_path" --name "$name" --url "$url" "$reference" "$depth" || exit - cloned_modules="$cloned_modules;$name" subsha1= + update_module=checkout else subsha1=$(clear_local_git_env; cd "$sm_path" && git rev-parse --verify HEAD) || @@ -751,13 +737,6 @@ Maybe you want to use 'update --init'?")" die "$(eval_gettext "Unable to fetch in submodule path '\$displaypath'")" fi - # Is this something we just cloned? - case ";$cloned_modules;" in - *";$name;"*) - # then there is no local change to integrate - update_module=checkout ;; - esac - must_die_on_failure= case "$update_module" in checkout) -- cgit v1.3 From 2335b870fa7942d9ca9e73e96e35171fda741376 Mon Sep 17 00:00:00 2001 From: Stefan Beller Date: Mon, 29 Feb 2016 18:07:19 -0800 Subject: submodule update: expose parallelism to the user Expose possible parallelism either via the "--jobs" CLI parameter or the "submodule.fetchJobs" setting. By having the variable initialized to -1, we make sure 0 can be passed into the parallel processing machine, which will then pick as many parallel workers as there are CPUs. Reviewed-by: Jonathan Nieder Signed-off-by: Stefan Beller Signed-off-by: Junio C Hamano --- Documentation/git-submodule.txt | 7 ++++++- builtin/submodule--helper.c | 8 +++++++- git-submodule.sh | 9 +++++++++ t/t7406-submodule-update.sh | 12 ++++++++++++ 4 files changed, 34 insertions(+), 2 deletions(-) (limited to 'git-submodule.sh') diff --git a/Documentation/git-submodule.txt b/Documentation/git-submodule.txt index 1572f058f5..13adebf7b7 100644 --- a/Documentation/git-submodule.txt +++ b/Documentation/git-submodule.txt @@ -16,7 +16,7 @@ SYNOPSIS 'git submodule' [--quiet] deinit [-f|--force] [--] ... 'git submodule' [--quiet] update [--init] [--remote] [-N|--no-fetch] [-f|--force] [--rebase|--merge] [--reference ] - [--depth ] [--recursive] [--] [...] + [--depth ] [--recursive] [--jobs ] [--] [...] 'git submodule' [--quiet] summary [--cached|--files] [(-n|--summary-limit) ] [commit] [--] [...] 'git submodule' [--quiet] foreach [--recursive] @@ -377,6 +377,11 @@ for linkgit:git-clone[1]'s `--reference` and `--shared` options carefully. clone with a history truncated to the specified number of revisions. See linkgit:git-clone[1] +-j :: +--jobs :: + This option is only valid for the update command. + Clone new submodules in parallel with as many jobs. + Defaults to the `submodule.fetchJobs` option. ...:: Paths to submodule(s). When specified this will restrict the command diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index 9e8109e542..a484945d37 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -430,6 +430,7 @@ static int update_clone_task_finished(int result, static int update_clone(int argc, const char **argv, const char *prefix) { const char *update = NULL; + int max_jobs = -1; struct string_list_item *item; struct pathspec pathspec; struct submodule_update_clone suc = SUBMODULE_UPDATE_CLONE_INIT; @@ -450,6 +451,8 @@ static int update_clone(int argc, const char **argv, const char *prefix) OPT_STRING(0, "depth", &suc.depth, "", N_("Create a shallow clone truncated to the " "specified number of revisions")), + OPT_INTEGER('j', "jobs", &max_jobs, + N_("parallel jobs")), OPT__QUIET(&suc.quiet, N_("don't print cloning progress")), OPT_END() }; @@ -477,7 +480,10 @@ static int update_clone(int argc, const char **argv, const char *prefix) gitmodules_config(); git_config(submodule_config, NULL); - run_processes_parallel(1, + if (max_jobs < 0) + max_jobs = parallel_submodules(); + + run_processes_parallel(max_jobs, update_clone_get_next_task, update_clone_start_failure, update_clone_task_finished, diff --git a/git-submodule.sh b/git-submodule.sh index a6a82d25aa..86018ee9c5 100755 --- a/git-submodule.sh +++ b/git-submodule.sh @@ -645,6 +645,14 @@ cmd_update() --depth=*) depth=$1 ;; + -j|--jobs) + case "$2" in '') usage ;; esac + jobs="--jobs=$2" + shift + ;; + --jobs=*) + jobs=$1 + ;; --) shift break @@ -671,6 +679,7 @@ cmd_update() ${update:+--update "$update"} \ ${reference:+--reference "$reference"} \ ${depth:+--depth "$depth"} \ + ${jobs:+$jobs} \ "$@" || echo "#unmatched" } | { err= diff --git a/t/t7406-submodule-update.sh b/t/t7406-submodule-update.sh index dda3929d99..7fd5142d6c 100755 --- a/t/t7406-submodule-update.sh +++ b/t/t7406-submodule-update.sh @@ -774,4 +774,16 @@ test_expect_success 'submodule update --recursive drops module name before recur test_i18ngrep "Submodule path .deeper/submodule/subsubmodule.: checked out" actual ) ' + +test_expect_success 'submodule update can be run in parallel' ' + (cd super2 && + GIT_TRACE=$(pwd)/trace.out git submodule update --jobs 7 && + grep "7 tasks" trace.out && + git config submodule.fetchJobs 8 && + GIT_TRACE=$(pwd)/trace.out git submodule update && + grep "8 tasks" trace.out && + GIT_TRACE=$(pwd)/trace.out git submodule update --jobs 9 && + grep "9 tasks" trace.out + ) +' test_done -- cgit v1.3