This should improve speed a lot:
- Increasing the number of fetch jobs speeds up a lot the
downloads when resuming from a previously interrupted
download session as checking for new commits can have
a big latency, so checking many as repositories as
possible at the same time speeds up things a lot.
With Replicant 4.2 0004, using "-j 100" worked well for
me, however it didn't seem to have reached 100 parallel
download jobs, so using more would not improve
performance in my case.
- As for the number of compression threads, one has to consider
it carefully: xz -9e uses quite a big amount of RAM when compressing,
so using too much threads can result in xz being killed because
it used too much RAM. In another hand using 1 thread per core
seems to divide the amount of time spent compressing by the number
of threads.
Signed-off-by: Denis 'GNUtoo' Carikli <gnu...@cyberdimension.org>
---
make_source_tarball.sh | 44 +++++++++++++++++++++++++++++++++++-------
1 file changed, 37 insertions(+), 7 deletions(-)
diff --git a/make_source_tarball.sh b/make_source_tarball.sh
index 41b8195..7939144 100755
--- a/make_source_tarball.sh
+++ b/make_source_tarball.sh
@@ -19,10 +19,44 @@ set -e
usage()
{
- echo "$0 <tag> <output directory>"
+ echo "$0 <tag> <output directory> [-j N] [-T N]"
+ echo " -j N use at most N download jobs to run in parallel;"
+ echo " the default is 4;"
+ echo " -T N use at most N threads to compress the tarball;"
+ echo " the default is 1; set to 0 to use as many threads
as there are processor cores"
+
exit 1
}
+jobs=4
+threads=1
+
+get_option()
+{
+ option="$1"
+ value="$2"
+
+ if [ "${option}" = "-T" ] ; then
+ threads="${value}"
+ elif [ "${option}" = "-j" ] ; then
+ jobs="${value}"
+ else
+ usage
+ fi
+}
+
+if [ $# -ne 2 -a $# -ne 4 -a $# -ne 6 ] ; then
+ usage
+elif [ $# -eq 4 ] ; then
+ get_option "$3" "$4"
+elif [ $# -eq 6 ] ; then
+ if [ "$3" = "$5" ] ; then
+ usage
+ fi
+ get_option "$3" "$4"
+ get_option "$5" "$6"
+fi
+
tag="$1"
outdir="$2"
@@ -33,10 +67,6 @@ version="$(echo ${tag} | sed 's/replicant-//')"
major_version="$(echo ${version} | sed 's/-.*//')"
minor_version="$(echo ${version} | sed 's/.*-//')"
-if [ $# -ne 2 ] ; then
- usage
-fi
-
mkdir -p "${replicant_dir}"
cd "${replicant_dir}"
if [ ! -d .repo ] ; then
@@ -57,7 +87,7 @@ if [ $(echo ${major_version} | sed 's/\..*//') -lt 6 ] ; then
.repo/manifests/default.xml
fi
-repo sync
+repo sync -j "${jobs}"
if [ $(echo $major_version | sed 's/\..*//') -lt 6 ] ; then
git -C .repo/manifests reset --hard
@@ -71,4 +101,4 @@ fi
# TODO: Make the tarball reproducible by removing metadata
-xz -9e --verbose "${tarball}"
+xz -T "${threads}" -9e --verbose "${tarball}"
--
2.30.2
_______________________________________________
Replicant mailing list
Replicant@osuosl.org
https://lists.osuosl.org/mailman/listinfo/replicant
