ArielGlenn has uploaded a new change for review. https://gerrit.wikimedia.org/r/263661
Change subject: dumps: add nocreate versions of stages files for huge wikis ...................................................................... dumps: add nocreate versions of stages files for huge wikis also run 8 createdir processes at once for small/big wikis instead of 1 and fix up name of createdirs stage file for huge wikis to be consistent with the other stage files for huge wikis Change-Id: I71161ba044c55990b75d37d550f2b6132c322581 --- M modules/snapshot/manifests/dumps/stagesconfig.pp M modules/snapshot/templates/dumpstages.erb 2 files changed, 14 insertions(+), 6 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/61/263661/1 diff --git a/modules/snapshot/manifests/dumps/stagesconfig.pp b/modules/snapshot/manifests/dumps/stagesconfig.pp index 407e62a..86d8a92 100644 --- a/modules/snapshot/manifests/dumps/stagesconfig.pp +++ b/modules/snapshot/manifests/dumps/stagesconfig.pp @@ -73,11 +73,19 @@ stagestype => 'partial_huge', stages => $stages, } + snapshot::dumps::stagesconf { 'stages_normal_nocreate_hugewikis': + stagestype => 'normal_nocreate_huge', + stages => $stages, + } + snapshot::dumps::stagesconf { 'stages_partial_nocreate_hugewikis': + stagestype => 'partial_nocreate_huge', + stages => $stages, + } snapshot::dumps::stagesconf { 'stages_create': stagestype => 'create', stages => $stages, } - snapshot::dumps::stagesconf { 'stages_create_huge': + snapshot::dumps::stagesconf { 'stages_create_hugewikis': stagestype => 'create_huge', stages => $stages, } diff --git a/modules/snapshot/templates/dumpstages.erb b/modules/snapshot/templates/dumpstages.erb index 786c479..b6b0ab1 100644 --- a/modules/snapshot/templates/dumpstages.erb +++ b/modules/snapshot/templates/dumpstages.erb @@ -9,7 +9,7 @@ <% if @stagestype == 'normal' or @stagestype == 'partial' or @stagestype == 'create' %> # mark the start of the run for all small, big wikis -8 1 continue none <%= @stages['smallwikis']['firststage'] -%> --job createdirs --sleep 5; <%= @stages['bigwikis']['firststage'] %> --job createdirs --sleep 5 +8 8 continue none <%= @stages['smallwikis']['firststage'] -%> --job createdirs --sleep 5; <%= @stages['bigwikis']['firststage'] %> --job createdirs --sleep 5 <% end -%> @@ -48,8 +48,8 @@ <% end -%> -<% if @stagestype == 'normal_huge' or @stagestype == 'partial_huge' %> -# stubs first +<% if @stagestype == 'normal_huge' or @stagestype == 'partial_huge' or @stagestype == 'normal_huge_nocreate' or @stagestype == 'partial_huge_nocreate' %> +# stubs 27 1 continue none <%= @stages['hugewikis']['rest'] -%> --job xmlstubsdump # tables next so inconsistencies between stubs and tables aren't too huge 1 1 continue none <%= @stages['hugewikis']['rest'] -%> --job tables @@ -69,12 +69,12 @@ # recombine articles plus meta pages for huge wikis 1 1 continue none <%= @stages['hugewikis']['rest'] -%> --job metacurrentdumprecombine -<% if @stagestype == 'normal_huge' %> +<% if @stagestype == 'normal_huge' or @stagestype == 'normal_huge_nocreate' %> # all remaining jobs 27 1 continue none <%= @stages['hugewikis']['rest'] %> <% end -%> -<% if @stagestype == 'partial_huge' %> +<% if @stagestype == 'partial_huge' or @stagestype == 'partial_huge_nocreate' %> # all remaining jobs except for the history revs 27 1 continue none <%= @stages['hugewikis']['rest'] %> <%= @stages['skipjob_args'] %> <% end -%> -- To view, visit https://gerrit.wikimedia.org/r/263661 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I71161ba044c55990b75d37d550f2b6132c322581 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: ArielGlenn <ar...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits