CustomTarget_html.mk | 64 ++++++++++++++++++++++++---------------------------
1 file changed, 31 insertions(+), 33 deletions(-)
New commits:
commit 2cdc1c10b56d99405bd46531497aa5dfaa17956d
Author: Christian Lohmaier <cl...@documentfoundation.org>
AuthorDate: Sat Dec 17 13:14:38 2022 +0100
Commit: Christian Lohmaier <lohmaier+libreoff...@googlemail.com>
CommitDate: Mon Dec 19 11:29:29 2022 +0000
reduce number of process calls when building html help
For each language times the 2500+ xhp files the build did run
dirname, mkdir, and pwd (twice, and on windows also with additional
cygpath) - replacing those by static values provided my make saves
4-5 minutes on a fast m1 machine and over 25 on windows with
cygwin (despite building with 32 threads...)
Change-Id: Ib3ca9e2ffdea622a4b5f578f2580152eb0b3b674
Reviewed-on: https://gerrit.libreoffice.org/c/help/+/144393
Tested-by: Jenkins
Reviewed-by: Christian Lohmaier <lohmaier+libreoff...@googlemail.com>
(cherry picked from commit 88c2d7943241daed9cc850e9518555ef9056634b)
Reviewed-on: https://gerrit.libreoffice.org/c/help/+/144372
diff --git a/CustomTarget_html.mk b/CustomTarget_html.mk
index 7ba88de11a..f3ff31ca9e 100644
--- a/CustomTarget_html.mk
+++ b/CustomTarget_html.mk
@@ -14,6 +14,8 @@ html_TREE_MODULES := swriter scalc simpress sdraw sdatabase
smath schart sbasic
html_TEXT_MODULES := $(html_TREE_MODULES)
html_BMARK_MODULES := swriter:WRITER scalc:CALC simpress:IMPRESS sdraw:DRAW
sdatabase:BASE smath:MATH schart:CHART sbasic:BASIC shared:SHARED
+gb_html_allhelpfiles = $(foreach
module,$(html_TEXT_MODULES),$(gb_AllLangHelp_$(module)_HELPFILES))
+
# In case someone has a product name containing quotes, use Unicode
# code points for ' (27) and " (22) in JS, CSS and entities for HTML.
gb_PRODUCTNAME_JS := $(subst ',\\\u{27},$(subst ",\\\u{22},$(PRODUCTNAME)))
@@ -47,7 +49,7 @@ $(call
gb_CustomTarget_get_workdir,helpcontent2/help3xsl)/hid2file.js : \
$(SRCDIR)/helpcontent2/CustomTarget_html.mk
$(call gb_Output_announce,$(subst $(WORKDIR)/,,$@),$(true),XSL,1)
( \
- RESPONSEFILE=$(call gb_var2file,$(shell $(gb_MKTEMP)),$(subst
helpcontent2/source/text/,,$(foreach
module,$(html_TEXT_MODULES),$(gb_AllLangHelp_$(module)_HELPFILES))$(if $(filter
WNT,$(OS)), ))) && \
+ RESPONSEFILE=$(call gb_var2file,$(shell $(gb_MKTEMP)),$(subst
helpcontent2/source/text/,,$(gb_html_allhelpfiles)$(if $(filter WNT,$(OS)), )))
&& \
echo 'var hid2fileMap = {' \
&& cd $(SRCDIR)/helpcontent2/source/text && $(call
gb_ExternalExecutable_get_command,xsltproc,xargs) $< <$$RESPONSEFILE || { rm
$$RESPONSEFILE; exit 1 ; } \
&& rm "$$RESPONSEFILE" \
@@ -208,31 +210,27 @@ endef
$(eval $(foreach lang,$(gb_HELP_LANGS),$(call html_gen_html_dep,$(lang))))
$(call gb_CustomTarget_get_workdir,helpcontent2/help3xsl)/%/html.text : \
- $(call
gb_CustomTarget_get_workdir,helpcontent2/help3xsl/links.txt.xsl) \
- $(call gb_ExternalExecutable_get_dependencies,xsltproc)
+ $(call
gb_CustomTarget_get_workdir,helpcontent2/help3xsl/links.txt.xsl) \
+ $(call gb_ExternalExecutable_get_dependencies,xsltproc)
$(call gb_Output_announce,$(subst $(WORKDIR)/,,$@),$(true),XSL,1)
- $(call gb_Helper_abbreviate_dirs,\
- cd $(if $(filter en-US,$*),$(SRCDIR),$(call
gb_HelpTranslatePartTarget_get_workdir,$*))/helpcontent2/source \
- && rm -rf $(dir $@)text \
- && RESPONSEFILE=$(call gb_var2file,$(shell
$(gb_MKTEMP)),$(foreach module,$(html_TEXT_MODULES),$(patsubst
helpcontent2/source/%,%,$(gb_AllLangHelp_$(module)_HELPFILES)))) \
- && <"$$RESPONSEFILE" $(if $(filter WNT,$(OS)),tr -d '\r' | env
-i PATH="$$PATH") xargs -n 1 printf '%s\n' \
- | while read xhp; do \
- mkdir -p $$(dirname $(dir $@)$$xhp) && \
- $(call gb_ExternalExecutable_get_command,xsltproc) \
- --stringparam Language $* \
- --stringparam local $(if
$(HELP_ONLINE),'no','yes') \
- --stringparam root $(if $(filter
WNT,$(OS)),$$(cygpath -m `pwd`),`pwd`)/ \
- --stringparam productname
"$(gb_PRODUCTNAME_HTML)" \
- --stringparam productversion
"$(PRODUCTVERSION)" \
- --stringparam xapian $(if $(filter TRUE,
$(HELP_OMINDEX_PAGE)),'yes','no') \
- -o $(dir $@)$${xhp%.xhp}.html \
- $(call
gb_CustomTarget_get_workdir,helpcontent2/help3xsl/online_transform.xsl) \
- $(if $(filter WNT,$(OS)),$$(cygpath -m
`pwd`),`pwd`)/$$xhp \
- || exit \
- ; done \
- && rm "$$RESPONSEFILE" \
- && touch $@ \
- )
+ rm -rf $(dir $@)text && mkdir -p $(dir $@)text && cd $(dir $@)text &&
mkdir -p $(sort $(subst helpcontent2/source/text/,,$(dir
$(gb_html_allhelpfiles)))) \
+ && cd $(if $(filter en-US,$*),$(SRCDIR),$(call
gb_HelpTranslatePartTarget_get_workdir,$*)) \
+ && RESPONSEFILE=$(call gb_var2file,$(shell $(gb_MKTEMP)),$(addsuffix
$(NEWLINE),$(subst helpcontent2/source/,,$(gb_html_allhelpfiles)))) \
+ && while read xhp; do \
+ $(call gb_ExternalExecutable_get_command,xsltproc) \
+ --stringparam Language $* \
+ --stringparam local $(if $(HELP_ONLINE),'no','yes') \
+ --stringparam root $(if $(filter en-US,$*),$(SRCDIR),$(call
gb_HelpTranslatePartTarget_get_workdir,$*))/helpcontent2/source/ \
+ --stringparam productname "$(gb_PRODUCTNAME_HTML)" \
+ --stringparam productversion "$(PRODUCTVERSION)" \
+ --stringparam xapian $(if $(filter TRUE,
$(HELP_OMINDEX_PAGE)),'yes','no') \
+ -o $(dir $@)$${xhp%.xhp}.html \
+ $(call
gb_CustomTarget_get_workdir,helpcontent2/help3xsl/online_transform.xsl) \
+ helpcontent2/source/$$xhp \
+ || exit \
+ ; done <"$$RESPONSEFILE" \
+ && rm "$$RESPONSEFILE" \
+ && touch $@
$(call gb_CustomTarget_get_workdir,helpcontent2/help3xsl)/%/bookmarks.js :
$(call gb_Output_announce,$(subst $(WORKDIR)/,,$@),$(true),CAT,2)
commit 0d390035db0a9115aaf061bc090d95a00ea95451
Author: Christian Lohmaier <lohmaier+libreoff...@googlemail.com>
AuthorDate: Thu Dec 15 16:35:31 2022 +0100
Commit: Christian Lohmaier <lohmaier+libreoff...@googlemail.com>
CommitDate: Mon Dec 19 11:29:20 2022 +0000
cd to helpcontent2/source/text & drop it from filenames for larger batches
On the Jenkins Windows builder only 40~45 of the ~2500 helpfiles can be
processed at once when not stripping that prefix. With the shortened
filenames the build needs less than half of the xsltproc invocations.
Also prevent a trailing carriage return to be interpreted as part of the
filename on windows and drop the abbreviate_dir wrapper around the
command since there's nothing left to shorten anymore.
Change-Id: Ifaff26f73f262bd1d7cb4f813bf081782661ba17
Reviewed-on: https://gerrit.libreoffice.org/c/help/+/144236
Tested-by: Jenkins
Reviewed-by: Christian Lohmaier <lohmaier+libreoff...@googlemail.com>
(cherry picked from commit 5daaa3e440b17505d644273a7305f073d6d5834d)
Reviewed-on: https://gerrit.libreoffice.org/c/help/+/144264
diff --git a/CustomTarget_html.mk b/CustomTarget_html.mk
index 8db68cf8c1..7ba88de11a 100644
--- a/CustomTarget_html.mk
+++ b/CustomTarget_html.mk
@@ -38,21 +38,21 @@ $(eval $(call
gb_CustomTarget_register_targets,helpcontent2/help3xsl,\
) \
))
+# trailing space for Windows so that xargs doesn't interpret the final CR (that
+# win-make adds as the newline character) as part of the last filename
$(call gb_CustomTarget_get_workdir,helpcontent2/help3xsl)/hid2file.js : \
$(SRCDIR)/helpcontent2/help3xsl/generate_hid2file.xsl \
$(call gb_ExternalExecutable_get_dependencies,xsltproc) \
$(foreach module,$(html_TEXT_MODULES),$(call
gb_AllLangHelp_get_helpfiles_target,$(module))) \
$(SRCDIR)/helpcontent2/CustomTarget_html.mk
$(call gb_Output_announce,$(subst $(WORKDIR)/,,$@),$(true),XSL,1)
- $(call gb_Helper_abbreviate_dirs,\
- ( \
- RESPONSEFILE=$(call gb_var2file,$(shell
$(gb_MKTEMP)),$(foreach
module,$(html_TEXT_MODULES),$(gb_AllLangHelp_$(module)_HELPFILES))) && \
- echo 'var hid2fileMap = {' \
- && cd $(SRCDIR) && $(call
gb_ExternalExecutable_get_command,xsltproc,xargs) $< <$$RESPONSEFILE || { rm
$$RESPONSEFILE; exit 1 ; } \
- && rm "$$RESPONSEFILE" \
- && echo '};' \
- ) > $@ \
- )
+ ( \
+ RESPONSEFILE=$(call gb_var2file,$(shell $(gb_MKTEMP)),$(subst
helpcontent2/source/text/,,$(foreach
module,$(html_TEXT_MODULES),$(gb_AllLangHelp_$(module)_HELPFILES))$(if $(filter
WNT,$(OS)), ))) && \
+ echo 'var hid2fileMap = {' \
+ && cd $(SRCDIR)/helpcontent2/source/text && $(call
gb_ExternalExecutable_get_command,xsltproc,xargs) $< <$$RESPONSEFILE || { rm
$$RESPONSEFILE; exit 1 ; } \
+ && rm "$$RESPONSEFILE" \
+ && echo '};' \
+ ) > $@
# Xapian localized templates
