ArielGlenn has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/386388 )
Change subject: generate one config fule for xml/sql dumps for wikis ...................................................................... generate one config fule for xml/sql dumps for wikis We had separate files for: big wikis, enwiki, wiidatawiki, the rest. This commit combines them all into one. No more wondering which onfig file to use with a dumps run. This will help to facilitate the move of xml/sql dumps to the dumpsdata hosts. Bug: T178893 Change-Id: I6fc98e3152693b073591da1c582aa9cb9c9b1411 --- M modules/snapshot/manifests/dumps/configs.pp M modules/snapshot/manifests/dumps/wikiconf.pp M modules/snapshot/templates/dumps/wikidump.conf.erb 3 files changed, 125 insertions(+), 156 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/88/386388/1 diff --git a/modules/snapshot/manifests/dumps/configs.pp b/modules/snapshot/manifests/dumps/configs.pp index a6ffb47..48d3608 100644 --- a/modules/snapshot/manifests/dumps/configs.pp +++ b/modules/snapshot/manifests/dumps/configs.pp @@ -12,131 +12,99 @@ $wikidatachunkhistory2 = '2587436,951696,942913,837759,1568292,1293747,2018593,1461235,1797642,1487121,2012246,874850,1486799' $config = { - smallwikis => { - dblist => "${apachedir}/dblists/all.dblist", - skipdblist => "${dblistsdir}/skip.dblist", - keep => '10', - chunksEnabled => '0', - }, - bigwikis => { - dblist => "${dblistsdir}/bigwikis.dblist", - skipdblist => "${dblistsdir}/skipnone.dblist", - keep => '8', - chunksEnabled => '1', - recombineHistory => '0', - revsPerJob => '1500000', - retryWait => '30', - maxRetries => '3', - revsMargin => '100', - wikis => { - ruwiki => { - pagesPerChunkHistory => '311181,1142420,1627923,3122803', - pagesPerChunkAbstract => '1200000', - chunksForAbstract => '4', - checkpointTime => '720', - }, - eswiki => { - pagesPerChunkHistory => '229078,854374,2324057,3860337', - pagesPerChunkAbstract => '1500000', - chunksForAbstract => '4', - checkpointTime => '720', - }, - dewiki => { - pagesPerChunkHistory => '425449,1451596,3087456,4274631', - pagesPerChunkAbstract => '2000000', - chunksForAbstract => '4', - checkpointTime => '720', - }, - ptwiki => { - pagesPerChunkHistory => '158756,789502,1361027,2669897', - pagesPerChunkAbstract => '1000000', - chunksForAbstract => '4', - }, - plwiki => { - pagesPerChunkHistory => '235373,514490,957079,1923498', - pagesPerChunkAbstract => '800000', - chunksForAbstract => '4', - }, - nlwiki => { - pagesPerChunkHistory => '224145,552515,1098243,2570924', - pagesPerChunkAbstract => '1000000', - chunksForAbstract => '4', - }, - frwiki => { - pagesPerChunkHistory => '412303,1235589,2771967,5313378', - pagesPerChunkAbstract => '1900000', - chunksForAbstract => '4', - checkpointTime => '720', - }, - itwiki => { - pagesPerChunkHistory => '442893,1049883,1381705,2929437', - pagesPerChunkAbstract => '1200000', - chunksForAbstract => '4', - checkpointTime => '720', - }, - jawiki => { - pagesPerChunkHistory => '168815,480631,943865,1760565', - pagesPerChunkAbstract => '800000', - chunksForAbstract => '4', - checkpointTime => '720', - }, - commonswiki => { - pagesPerChunkHistory => '6457504,9672260,12929298,17539944', - pagesPerChunkAbstract => '11000000', - chunksForAbstract => '4', - checkpointTime => '720', - }, - zhwiki => { - pagesPerChunkHistory => '231819,564192,1300322,3112369', - pagesPerChunkAbstract => '1300000', - chunksForAbstract => '4', - }, - metawiki => { - pagesPerChunkHistory => '823386,2594392,3242670,3284506', - pagesPerChunkAbstract => '2500000', - chunksForAbstract => '4', + allwikis => { + global => { + dblist => "${apachedir}/dblists/all.dblist", + skipdblist => "${dblistsdir}/skip.dblist", + keep => '10', + chunksEnabled => '0', + }, + big => { + dblist => "${dblistsdir}/bigwikis.dblist", + skipdblist => "${dblistsdir}/skipnone.dblist", + keep => '8', + chunksEnabled => '1', + chunksForAbstract => '4', + checkpointTime => '720', + recombineHistory => '0', + revsPerJob => '1500000', + retryWait => '30', + maxRetries => '3', + revsMargin => '100', + wikis => { + ruwiki => { + pagesPerChunkHistory => '311181,1142420,1627923,3122803', + pagesPerChunkAbstract => '1200000', + }, + eswiki => { + pagesPerChunkHistory => '229078,854374,2324057,3860337', + pagesPerChunkAbstract => '1500000', + }, + dewiki => { + pagesPerChunkHistory => '425449,1451596,3087456,4274631', + pagesPerChunkAbstract => '2000000', + }, + ptwiki => { + pagesPerChunkHistory => '158756,789502,1361027,2669897', + pagesPerChunkAbstract => '1000000', + }, + plwiki => { + pagesPerChunkHistory => '235373,514490,957079,1923498', + pagesPerChunkAbstract => '800000', + }, + nlwiki => { + pagesPerChunkHistory => '224145,552515,1098243,2570924', + pagesPerChunkAbstract => '1000000', + }, + frwiki => { + pagesPerChunkHistory => '412303,1235589,2771967,5313378', + pagesPerChunkAbstract => '1900000', + }, + itwiki => { + pagesPerChunkHistory => '442893,1049883,1381705,2929437', + pagesPerChunkAbstract => '1200000', + }, + jawiki => { + pagesPerChunkHistory => '168815,480631,943865,1760565', + pagesPerChunkAbstract => '800000', + }, + commonswiki => { + pagesPerChunkHistory => '6457504,9672260,12929298,17539944', + pagesPerChunkAbstract => '11000000', + }, + zhwiki => { + pagesPerChunkHistory => '231819,564192,1300322,3112369', + pagesPerChunkAbstract => '1300000', + }, + metawiki => { + pagesPerChunkHistory => '823386,2594392,3242670,3284506', + pagesPerChunkAbstract => '2500000', + }, }, }, - }, - wikidatawiki => { - dblist => "${dblistsdir}/wikidatawiki.dblist", - skipdblist => "${dblistsdir}/skipnone.dblist", - keep => '7', - chunksEnabled => '1', - recombineHistory => '0', - checkpointTime => '720', - revsPerJob => '1500000', - retryWait => '30', - maxRetries => '3', - revsMargin => '100', - wikis => { + huge => { + skipdblist => "${dblistsdir}/skipnone.dblist", + jobsperbatch => 'xmlstubsdump=9,abstractsdump=9', + keep => '7', + chunksEnabled => '1', + chunksForAbstract => 27, + recombineHistory => '0', + checkpointTime => '720', + revsPerJob => '1500000', + retryWait => '30', + maxRetries => '3', + revsMargin => '100', + maxrevs => '20000', + wikis => { wikidatawiki => { - jobsperbatch => 'xmlstubsdump=9,abstractsdump=9', + dblist => "${dblistsdir}/wikidatawiki.dblist", pagesPerChunkHistory => "${wikidatachunkhistory1},${$wikidatachunkhistory2}", - chunksForAbstract => 27, orderrevs => '1', minpages => '10', - maxrevs => '20000', }, - }, - }, - enwiki => { - dblist => "${dblistsdir}/enwiki.dblist", - skipdblist => "${dblistsdir}/skipnone.dblist", - keep => '7', - chunksEnabled => '1', - recombineHistory => '0', - checkpointTime => '720', - revsPerJob => '1500000', - retryWait => '30', - maxRetries => '3', - revsMargin => '100', - wikis => { enwiki => { - jobsperbatch => 'xmlstubsdump=9,abstractsdump=9', + dblist => "${dblistsdir}/enwiki.dblist", pagesPerChunkHistory => "${enchunkhistory1},${enchunkhistory2}", - chunksForAbstract => 27, - maxrevs => '20000', }, }, }, @@ -154,20 +122,8 @@ }, } - snapshot::dumps::wikiconf { 'wikidump.conf': - configtype => 'smallwikis', - config => $config, - } - snapshot::dumps::wikiconf { 'wikidump.conf.bigwikis': - configtype => 'bigwikis', - config => $config, - } - snapshot::dumps::wikiconf { 'wikidump.conf.enwiki': - configtype => 'enwiki', - config => $config, - } - snapshot::dumps::wikiconf { 'wikidump.conf.wikidatawiki': - configtype => 'wikidatawiki', + snapshot::dumps::wikiconf { 'wikidump.conf.allwikis': + configtype => 'allwikis', config => $config, } snapshot::dumps::wikiconf { 'wikidump.conf.monitor': diff --git a/modules/snapshot/manifests/dumps/wikiconf.pp b/modules/snapshot/manifests/dumps/wikiconf.pp index 2e7af1d..ed8f894 100644 --- a/modules/snapshot/manifests/dumps/wikiconf.pp +++ b/modules/snapshot/manifests/dumps/wikiconf.pp @@ -1,5 +1,5 @@ define snapshot::dumps::wikiconf( - $configtype = 'smallwikis', + $configtype = 'allwikis', $config = undef, ) { diff --git a/modules/snapshot/templates/dumps/wikidump.conf.erb b/modules/snapshot/templates/dumps/wikidump.conf.erb index 6e580d7..3aeef3d 100644 --- a/modules/snapshot/templates/dumps/wikidump.conf.erb +++ b/modules/snapshot/templates/dumps/wikidump.conf.erb @@ -4,10 +4,10 @@ ############################################################# [wiki] -dblist=<%= @config[@configtype]['dblist'] %> +dblist=<%= @config[@configtype]['global']['dblist'] %> privatelist=<%= scope.lookupvar('snapshot::dumps::dirs::apachedir') -%>/dblists/private.dblist closedlist=<%= scope.lookupvar('snapshot::dumps::dirs::apachedir') -%>/dblists/closed.dblist -skipdblist=<%= @config[@configtype]['skipdblist'] %> +skipdblist=<%= @config[@configtype]['global']['skipdblist'] %> flowlist=<%= scope.lookupvar('snapshot::dumps::dirs::apachedir') -%>/dblists/flow.dblist dir=<%= scope.lookupvar('snapshot::dumps::dirs::apachedir') %> adminsettings=private/PrivateSettings.php @@ -46,28 +46,28 @@ recompressxml=/usr/local/bin/recompressxml [cleanup] -keep=<%= @config[@configtype]['keep'] %> +keep=<%= @config[@configtype]['global']['keep'] %> [chunks] -<% if @config[@configtype].has_key?('chunksEnabled') -%> -chunksEnabled=<%= @config[@configtype]['chunksEnabled'] %> +<% if @config[@configtype]['global'].has_key?('chunksEnabled') -%> +chunksEnabled=<%= @config[@configtype]['global']['chunksEnabled'] %> <% end -%> -<% if @config[@configtype].has_key?('recombineHistory') -%> +<% if @config[@configtype]['global'].has_key?('recombineHistory') -%> recombineHistory=<%= @config[@configtype]['recombineHistory'] %> <% end -%> -<% if @config[@configtype].has_key?('checkpointTime') -%> +<% if @config[@configtype]['global'].has_key?('checkpointTime') -%> checkpointTime=<%= @config[@configtype]['checkpointTime'] %> <% end -%> -<% if @config[@configtype].has_key?('revsPerJob') -%> +<% if @config[@configtype]['global'].has_key?('revsPerJob') -%> revsPerJob=<%= @config[@configtype]['revsPerJob'] %> <% end -%> -<% if @config[@configtype].has_key?('retryWait') -%> +<% if @config[@configtype]['global'].has_key?('retryWait') -%> retryWait=<%= @config[@configtype]['retryWait'] %> <% end -%> -<% if @config[@configtype].has_key?('maxRetries') -%> +<% if @config[@configtype]['global'].has_key?('maxRetries') -%> maxRetries=<%= @config[@configtype]['maxRetries'] %> <% end -%> -<% if @config[@configtype].has_key?('revsMargin') -%> +<% if @config[@configtype]['global'].has_key?('revsMargin') -%> revsMargin=<%= @config[@configtype]['revsMargin'] %> <% end -%> @@ -79,27 +79,40 @@ minpages=1 maxrevs=100000 -<% if @config[@configtype].has_key?('wikis') %> + +######################## +# wiki-specific settings + + +<% if @config[@configtype].has_key?('big') %> +<% if @config[@configtype]['big'].has_key?('wikis') %> <% @config[@configtype]['wikis'].keys.sort.each do |w| -%> [<%= w %>] -<% if @config[@configtype]['wikis'][w].has_key?('jobsperbatch') -%> -jobsperbatch=<%= @config[@configtype]['wikis'][w]['jobsperbatch'] %> +# generic settings for big wikis +<% @config[@configtype]['big'].keys.sort.each do |bigsetting| -%> +<%= bigsetting %>=<%= @config[@configtype]['big'][bigsetting] %> <% end -%> -pagesPerChunkHistory=<%= @config[@configtype]['wikis'][w]['pagesPerChunkHistory'] %> -pagesPerChunkAbstract=<%= @config[@configtype]['wikis'][w]['pagesPerChunkAbstract'] %> -chunksForAbstract=<%= @config[@configtype]['wikis'][w]['chunksForAbstract'] %> -<% if @config[@configtype]['wikis'][w].has_key?('checkpointTime') -%> -checkpointTime=<%= @config[@configtype]['wikis'][w]['checkpointTime'] %> +# specific settings for wiki <%= w %> +<% @config[@configtype]['big'][w].keys.sort.each do |bigwikisetting| -%> +<%= bigwikisetting %>=<%= @config[@configtype]['big'][bigwikisetting] %> <% end -%> -<% if @config[@configtype]['wikis'][w].has_key?('orderrevs') -%> -orderrevs=<%= @config[@configtype]['wikis'][w]['orderrevs'] %> <% end -%> -<% if @config[@configtype]['wikis'][w].has_key?('minpages') -%> -minpages=<%= @config[@configtype]['wikis'][w]['minpages'] %> <% end -%> -<% if @config[@configtype]['wikis'][w].has_key?('maxrevs') -%> -maxrevs=<%= @config[@configtype]['wikis'][w]['maxrevs'] %> <% end -%> + +<% if @config[@configtype].has_key?('huge') %> +<% if @config[@configtype]['huge'].has_key?('wikis') %> +<% @config[@configtype]['wikis'].keys.sort.each do |w| -%> +[<%= w %>] +# generic settings for huge wikis +<% @config[@configtype]['huge'].keys.sort.each do |hugesetting| -%> +<%= hugesetting %>=<%= @config[@configtype]['big'][hugesetting] %> +<% end -%> +# specific settings for wiki <%= w %> +<% @config[@configtype]['huge'][w].keys.sort.each do |hugewikisetting| -%> +<%= hugewikisetting %>=<%= @config[@configtype]['big'][hugewikisetting] %> +<% end -%> +<% end -%> <% end -%> <% end -%> -- To view, visit https://gerrit.wikimedia.org/r/386388 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I6fc98e3152693b073591da1c582aa9cb9c9b1411 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: ArielGlenn <ar...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits