[PATCH 8/8] submodules: allow parallel fetching, add tests and documentation

[Stefan Beller]

This enables the work of the previous patches.

Signed-off-by: Stefan Beller 
Signed-off-by: Junio C Hamano 
---
 Documentation/fetch-options.txt |  7 +++
 builtin/fetch.c |  6 +-
 builtin/pull.c  |  6 ++
 submodule.c |  3 +--
 submodule.h |  2 +-
 t/t5526-fetch-submodules.sh | 20 
 6 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/Documentation/fetch-options.txt b/Documentation/fetch-options.txt
index 45583d8..6b109f6 100644
--- a/Documentation/fetch-options.txt
+++ b/Documentation/fetch-options.txt
@@ -100,6 +100,13 @@ ifndef::git-pull[]
reference to a commit that isn't already in the local submodule
clone.
 
+-j::
+--jobs=::
+   Number of parallel children to be used for fetching submodules.
+   Each will fetch from different submodules, such that fetching many
+   submodules will be faster. By default submodules will be fetched
+   one at a time.
+
 --no-recurse-submodules::
Disable recursive fetching of submodules (this has the same effect as
using the '--recurse-submodules=no' option).
diff --git a/builtin/fetch.c b/builtin/fetch.c
index c85f347..586840d 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -37,6 +37,7 @@ static int prune = -1; /* unspecified */
 static int all, append, dry_run, force, keep, multiple, update_head_ok, 
verbosity;
 static int progress = -1, recurse_submodules = RECURSE_SUBMODULES_DEFAULT;
 static int tags = TAGS_DEFAULT, unshallow, update_shallow;
+static int max_children = 1;
 static const char *depth;
 static const char *upload_pack;
 static struct strbuf default_rla = STRBUF_INIT;
@@ -99,6 +100,8 @@ static struct option builtin_fetch_options[] = {
N_("fetch all tags and associated objects"), TAGS_SET),
OPT_SET_INT('n', NULL, ,
N_("do not fetch all tags (--no-tags)"), TAGS_UNSET),
+   OPT_INTEGER('j', "jobs", _children,
+   N_("number of submodules fetched in parallel")),
OPT_BOOL('p', "prune", ,
 N_("prune remote-tracking branches no longer on remote")),
{ OPTION_CALLBACK, 0, "recurse-submodules", NULL, N_("on-demand"),
@@ -1213,7 +1216,8 @@ int cmd_fetch(int argc, const char **argv, const char 
*prefix)
result = fetch_populated_submodules(,
submodule_prefix,
recurse_submodules,
-   verbosity < 0);
+   verbosity < 0,
+   max_children);
argv_array_clear();
}
 
diff --git a/builtin/pull.c b/builtin/pull.c
index 5145fc6..9e3c738 100644
--- a/builtin/pull.c
+++ b/builtin/pull.c
@@ -95,6 +95,7 @@ static int opt_force;
 static char *opt_tags;
 static char *opt_prune;
 static char *opt_recurse_submodules;
+static char *max_children;
 static int opt_dry_run;
 static char *opt_keep;
 static char *opt_depth;
@@ -178,6 +179,9 @@ static struct option pull_options[] = {
N_("on-demand"),
N_("control recursive fetching of submodules"),
PARSE_OPT_OPTARG),
+   OPT_PASSTHRU('j', "jobs", _children, N_("n"),
+   N_("number of submodules pulled in parallel"),
+   PARSE_OPT_OPTARG),
OPT_BOOL(0, "dry-run", _dry_run,
N_("dry run")),
OPT_PASSTHRU('k', "keep", _keep, NULL,
@@ -525,6 +529,8 @@ static int run_fetch(const char *repo, const char 
**refspecs)
argv_array_push(, opt_prune);
if (opt_recurse_submodules)
argv_array_push(, opt_recurse_submodules);
+   if (max_children)
+   argv_array_push(, max_children);
if (opt_dry_run)
argv_array_push(, "--dry-run");
if (opt_keep)
diff --git a/submodule.c b/submodule.c
index 6a2d786..0b48734 100644
--- a/submodule.c
+++ b/submodule.c
@@ -729,10 +729,9 @@ static int fetch_finish(int retvalue, struct child_process 
*cp,
 
 int fetch_populated_submodules(const struct argv_array *options,
   const char *prefix, int command_line_option,
-  int quiet)
+  int quiet, int max_parallel_jobs)
 {
int i;
-   int max_parallel_jobs = 1;
struct submodule_parallel_fetch spf = SPF_INIT;
 
spf.work_tree = get_git_work_tree();
diff --git a/submodule.h b/submodule.h
index 5507c3d..cbc0003 100644
--- a/submodule.h
+++ b/submodule.h
@@ -31,7 +31,7 @@ void set_config_fetch_recurse_submodules(int value);
 void check_for_new_submodule_commits(unsigned char new_sha1[20]);
 int fetch_populated_submodules(const struct argv_array *options,
   const char *prefix, 

[PATCH 8/8] submodules: allow parallel fetching, add tests and documentation

[Stefan Beller]

This enables the work of the previous patches.

Signed-off-by: Stefan Beller 
Signed-off-by: Junio C Hamano 
---
 Documentation/fetch-options.txt |  7 +++
 builtin/fetch.c |  6 +-
 builtin/pull.c  |  6 ++
 submodule.c |  3 +--
 submodule.h |  2 +-
 t/t5526-fetch-submodules.sh | 19 +++
 6 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/Documentation/fetch-options.txt b/Documentation/fetch-options.txt
index 45583d8..6b109f6 100644
--- a/Documentation/fetch-options.txt
+++ b/Documentation/fetch-options.txt
@@ -100,6 +100,13 @@ ifndef::git-pull[]
reference to a commit that isn't already in the local submodule
clone.
 
+-j::
+--jobs=::
+   Number of parallel children to be used for fetching submodules.
+   Each will fetch from different submodules, such that fetching many
+   submodules will be faster. By default submodules will be fetched
+   one at a time.
+
 --no-recurse-submodules::
Disable recursive fetching of submodules (this has the same effect as
using the '--recurse-submodules=no' option).
diff --git a/builtin/fetch.c b/builtin/fetch.c
index ee1f1a9..f28eac6 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -37,6 +37,7 @@ static int prune = -1; /* unspecified */
 static int all, append, dry_run, force, keep, multiple, update_head_ok, 
verbosity;
 static int progress = -1, recurse_submodules = RECURSE_SUBMODULES_DEFAULT;
 static int tags = TAGS_DEFAULT, unshallow, update_shallow;
+static int max_children = 1;
 static const char *depth;
 static const char *upload_pack;
 static struct strbuf default_rla = STRBUF_INIT;
@@ -99,6 +100,8 @@ static struct option builtin_fetch_options[] = {
N_("fetch all tags and associated objects"), TAGS_SET),
OPT_SET_INT('n', NULL, ,
N_("do not fetch all tags (--no-tags)"), TAGS_UNSET),
+   OPT_INTEGER('j', "jobs", _children,
+   N_("number of submodules fetched in parallel")),
OPT_BOOL('p', "prune", ,
 N_("prune remote-tracking branches no longer on remote")),
{ OPTION_CALLBACK, 0, "recurse-submodules", NULL, N_("on-demand"),
@@ -1217,7 +1220,8 @@ int cmd_fetch(int argc, const char **argv, const char 
*prefix)
result = fetch_populated_submodules(,
submodule_prefix,
recurse_submodules,
-   verbosity < 0);
+   verbosity < 0,
+   max_children);
argv_array_clear();
}
 
diff --git a/builtin/pull.c b/builtin/pull.c
index 722a83c..f0af196 100644
--- a/builtin/pull.c
+++ b/builtin/pull.c
@@ -94,6 +94,7 @@ static int opt_force;
 static char *opt_tags;
 static char *opt_prune;
 static char *opt_recurse_submodules;
+static char *max_children;
 static int opt_dry_run;
 static char *opt_keep;
 static char *opt_depth;
@@ -177,6 +178,9 @@ static struct option pull_options[] = {
N_("on-demand"),
N_("control recursive fetching of submodules"),
PARSE_OPT_OPTARG),
+   OPT_PASSTHRU('j', "jobs", _children, N_("n"),
+   N_("number of submodules pulled in parallel"),
+   PARSE_OPT_OPTARG),
OPT_BOOL(0, "dry-run", _dry_run,
N_("dry run")),
OPT_PASSTHRU('k', "keep", _keep, NULL,
@@ -524,6 +528,8 @@ static int run_fetch(const char *repo, const char 
**refspecs)
argv_array_push(, opt_prune);
if (opt_recurse_submodules)
argv_array_push(, opt_recurse_submodules);
+   if (max_children)
+   argv_array_push(, max_children);
if (opt_dry_run)
argv_array_push(, "--dry-run");
if (opt_keep)
diff --git a/submodule.c b/submodule.c
index 8134fe7..7ab89f4 100644
--- a/submodule.c
+++ b/submodule.c
@@ -653,10 +653,9 @@ static int fetch_finish(void *data, struct child_process 
*cp,
 
 int fetch_populated_submodules(const struct argv_array *options,
   const char *prefix, int command_line_option,
-  int quiet)
+  int quiet, int max_parallel_jobs)
 {
int i;
-   int max_parallel_jobs = 1;
struct submodule_parallel_fetch spf = SPF_INIT;
 
spf.work_tree = get_git_work_tree();
diff --git a/submodule.h b/submodule.h
index 5507c3d..cbc0003 100644
--- a/submodule.h
+++ b/submodule.h
@@ -31,7 +31,7 @@ void set_config_fetch_recurse_submodules(int value);
 void check_for_new_submodule_commits(unsigned char new_sha1[20]);
 int fetch_populated_submodules(const struct argv_array *options,
   const char *prefix, int