ArielGlenn has uploaded a new change for review. https://gerrit.wikimedia.org/r/119006
Change subject: dumps templates, configs, dblists added to snapshot module ...................................................................... dumps templates, configs, dblists added to snapshot module these were formerly unpuppetized, though sample templates live in the dumps repo this is the start of moving things to /srv on snapshot hosts Change-Id: I1c8f2f61ff2d8986514bc75f80b678c4f4e4697a --- A modules/snapshot/files/dumps/templates/download-index.html A modules/snapshot/files/dumps/templates/dvd.html A modules/snapshot/files/dumps/templates/errormail.txt A modules/snapshot/files/dumps/templates/feed.xml A modules/snapshot/files/dumps/templates/legal.html A modules/snapshot/files/dumps/templates/progress.html A modules/snapshot/files/dumps/templates/report.html A modules/snapshot/manifests/dirs.pp A modules/snapshot/manifests/dumps.pp A modules/snapshot/manifests/dumps/configs.pp A modules/snapshot/manifests/dumps/dblists.pp A modules/snapshot/manifests/dumps/templates.pp A modules/snapshot/manifests/dumps/wikiconf.pp A modules/snapshot/templates/wikidump.conf.erb 14 files changed, 897 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/06/119006/1 diff --git a/modules/snapshot/files/dumps/templates/download-index.html b/modules/snapshot/files/dumps/templates/download-index.html new file mode 100644 index 0000000..e1150cb --- /dev/null +++ b/modules/snapshot/files/dumps/templates/download-index.html @@ -0,0 +1,127 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd";> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en" dir="ltr"> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> + <title>Wikimedia Downloads</title> + <style type="text/css" media="screen">/*<![CDATA[*/ +body { + font: x-small sans-serif; + background: #f9f9f9 url(http://en.wikipedia.org/skins-1.5/monobook/headbg.jpg) 0 0 no-repeat; + color: black; + margin: 0; + padding: 0; +} +/* scale back up to a sane default */ +#globalWrapper { + font-size: 127%; + width: 100%; + margin: 0; + padding: 0; +} +#content { + margin: 2em 2em 0 2em; + padding: 0 1em 1.5em 1em; + background: white; + color: black; + border: 1px solid #aaa; + line-height: 1.5em; + position: relative; + z-index: 2; +} +h1, h2, h3, h4, h5, h6 { + color: black; + background: none; + font-weight: normal; + margin: 0; + padding-top: .5em; + padding-bottom: .17em; + border-bottom: 1px solid #aaa; +} +h1 { + font-size: 188%; + margin-bottom: .6em; +} +h2 { + font-size: 150%; + margin-bottom: .6em; +} +h3, h4, h5, h6 { + border-bottom: none; + font-weight: bold; + margin-bottom: .3em; +} + +ul { + line-height: 1.5em; + list-style-type: square; + margin: .3em 0 0 1.5em; + padding: 0; + list-style-image: url(bullet.gif); +} +ol { + line-height: 1.5em; + margin: .3em 0 0 3.2em; + padding: 0; + list-style-image: none; +} +li { + margin-bottom: .1em; +} +dt { + font-weight: bold; + margin-bottom: .05em; +} +dl { + margin-top: .2em; + margin-bottom: .5em; +} +dd { + line-height: 1.5em; + margin-left: 2em; + margin-bottom: .5em; +} +a { + text-decoration: none; + color: #002bb8; + background: none; +} +a:visited { + color: #5a3696; +} +a:active { + color: #faa700; +} +a:hover { + text-decoration: underline; +} + +.visualClear { + clear: both; +} + +/*]]>*/</style> + +</head> +<body> + <div id="globalWrapper"> + <div id="content"> + <h1>Wikimedia Downloads</h1> + <p>The following kinds of downloads are available:</p> + <dl> + <dt><a href="backup-index.html">Database backup dumps</a></dt> + <dd>A complete copy of all Wikimedia wikis, in the form of wikitext source and metadata embedded in XML. A number of raw database tables in SQL form are also available.</dd> + + <dt><a href="http://static.wikipedia.org/";>Static HTML dumps</a></dt> + <dd>A copy of all pages from all Wikipedia wikis, in HTML form.</dd> + + <dt><a href="dvd.html">DVD distributions</a></dt> + <dd>Available for some Wikipedia editions.</dd> + + <dt>Image tarballs</dt> + <dd>There are currently no image dumps available.</dd> + </dl> + <div class="visualClear"></div> + </div> + </div> +</body></html> + diff --git a/modules/snapshot/files/dumps/templates/dvd.html b/modules/snapshot/files/dumps/templates/dvd.html new file mode 100644 index 0000000..83f60e9 --- /dev/null +++ b/modules/snapshot/files/dumps/templates/dvd.html @@ -0,0 +1,128 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd";> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en" dir="ltr"> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> + <title>Wikimedia Downloads</title> + <style type="text/css" media="screen">/*<![CDATA[*/ +body { + font: x-small sans-serif; + background: #f9f9f9 url(http://en.wikipedia.org/skins-1.5/monobook/headbg.jpg) 0 0 no-repeat; + color: black; + margin: 0; + padding: 0; +} +/* scale back up to a sane default */ +#globalWrapper { + font-size: 127%; + width: 100%; + margin: 0; + padding: 0; +} +#content { + margin: 2em 2em 0 2em; + padding: 0 1em 1.5em 1em; + background: white; + color: black; + border: 1px solid #aaa; + line-height: 1.5em; + position: relative; + z-index: 2; +} +h1, h2, h3, h4, h5, h6 { + color: black; + background: none; + font-weight: normal; + margin: 0; + padding-top: .5em; + padding-bottom: .17em; + border-bottom: 1px solid #aaa; +} +h1 { + font-size: 188%; + margin-bottom: .6em; +} +h2 { + font-size: 150%; + margin-bottom: .6em; +} +h3, h4, h5, h6 { + border-bottom: none; + font-weight: bold; + margin-bottom: .3em; +} + +ul { + line-height: 1.5em; + list-style-type: square; + margin: .3em 0 0 1.5em; + padding: 0; + list-style-image: url(bullet.gif); +} +ol { + line-height: 1.5em; + margin: .3em 0 0 3.2em; + padding: 0; + list-style-image: none; +} +li { + margin-bottom: .1em; +} +dt { + font-weight: bold; + margin-bottom: .05em; +} +dl { + margin-top: .2em; + margin-bottom: .5em; +} +dd { + line-height: 1.5em; + margin-left: 2em; + margin-bottom: .5em; +} +a { + text-decoration: none; + color: #002bb8; + background: none; +} +a:visited { + color: #5a3696; +} +a:active { + color: #faa700; +} +a:hover { + text-decoration: underline; +} + +.visualClear { + clear: both; +} + +/*]]>*/</style> + +</head> +<body> + <div id="globalWrapper"> + <div id="content"> + <h1>Wikimedia DVD Downloads</h1> + <p>The following kinds of DVD downloads are available:</p> + <dl> + <dt><a href="http://dvd.wikimedia.org/";>German Wikipedia 2007/2008</a></dt> + <dd>German Wikipedia, using the ZenoReader (Windows) and + <a href="http://www.tntnet.org/tntreader.hms";>TntReader</a> (Linux / Unix / MacOS X, GPL). + </dd> + + <dt><a href="http://www.wikipediaondvd.com/";>Wikipedia 0.5</a></dt> + <dd>Selected articles from the English Wikipedia.</dd> + + <dt><a href="http://dvd.wikimedia.pl/";>Polska Wikipedia na DVD</a></dt> + <dd>Polish Wikipedia.</dd> + <dt><a href="http://www.mlwiki.in/";>Malayalam Wikisource 1.0</a></dt> + <dd>Selected books from Malayalam Wikisource on CD</dd> + </dl> + <div class="visualClear"></div> + </div> + </div> +</body></html> + diff --git a/modules/snapshot/files/dumps/templates/errormail.txt b/modules/snapshot/files/dumps/templates/errormail.txt new file mode 100644 index 0000000..1d8c2a5 --- /dev/null +++ b/modules/snapshot/files/dumps/templates/errormail.txt @@ -0,0 +1,8 @@ +This is the Wikimedia dump generator, reporting a failure on the +%(db)s database at %(time)s UTC. + +You may wish to check the status page for more details: +%(url)s + +-- ever your dear friend, + WikiBackup.py diff --git a/modules/snapshot/files/dumps/templates/feed.xml b/modules/snapshot/files/dumps/templates/feed.xml new file mode 100644 index 0000000..c28f023 --- /dev/null +++ b/modules/snapshot/files/dumps/templates/feed.xml @@ -0,0 +1,14 @@ +<?xml version="1.0"?> +<rss version="2.0"> + <channel> + <title>%(chantitle)s</title> + <link>%(chanlink)s</link> + <description>%(chandesc)s</description> + <item> + <title>%(title)s</title> + <link>%(link)s</link> + <description>%(description)s</description> + <pubDate>%(date)s</pubDate> + </item> + </channel> +</rss> diff --git a/modules/snapshot/files/dumps/templates/legal.html b/modules/snapshot/files/dumps/templates/legal.html new file mode 100644 index 0000000..aa67c3a --- /dev/null +++ b/modules/snapshot/files/dumps/templates/legal.html @@ -0,0 +1,44 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd";> + +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> +<head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/> + <title>Copyrights information</title> +</head> + +<body> +<h1>Copyright and license</h1> +<p>All original textual content except Wikinews original textual content is +licensed under the <a href="http://www.wikipedia.org/wiki/Wikipedia:Copyrights"; title="Wikipedia Copyrights"> +GNU Free Documentation License</a> (GFDL) and the +<a href="http://creativecommons.org/licenses/by-sa/3.0/"; title="Creative Commons Attribution-Share-Alike 3.0 License">Creative Commons Attribution-Share-Alike 3.0 License</a>. Some text may be available only under the Creative Commons license; see our +<a href="http://wikimediafoundation.org/wiki/Terms_of_use";>Terms of Use</a> for details. +Text written by some authors may be released under additional licenses +or into the public domain. Some text (including quotations) may +be used under fair use, usually where it is believed that the use +will also be fair dealing outside the United States (US). Note that material +used as "fair use" under United States law may not be legal to +reproduce outside the US. See +<a href="http://en.wikipedia.org/wiki/Fair_use";>Fair use</a> for more information.</p> + +<p>As of 2005-09-25 all Wikinews textual content is licensed under the +<a href="http://creativecommons.org/licenses/by/2.5/"; title="Creative Commons Attribution 2.5 License">Creative Commons Attribution 2.5 License</a>, +All Wikinews material published prior to that date (2005-09-25) is in the public domain.</p> + +<p>The Wikimedia Foundation Inc. does not hold copyright on any text +except where such text was produced by the Foundation.</p> + +<p>Image copyright information is contained in the image description page +inside the text dumps. Many images are NOT released under the GFDL; +these may be fair use or non-commercial images and their use may be +illegal outside Wikipedia or in a different context than they are used +on Wikipedia.</p> + +<p>Both the text and images are almost certain to contain some +copyright infringements which have not yet been noticed and removed. +Your use of this material is AT YOUR OWN RISK. Please let us know about +any copyright infringements you find, so they can be removed.</p> +</body> + +</html> diff --git a/modules/snapshot/files/dumps/templates/progress.html b/modules/snapshot/files/dumps/templates/progress.html new file mode 100644 index 0000000..a86d3d6 --- /dev/null +++ b/modules/snapshot/files/dumps/templates/progress.html @@ -0,0 +1,77 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd";> + +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> +<head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/> + <title>Database dump progress</title> + <style type="text/css"> + html, body { + background-color: #eee; + color: black; + } + .siteinfo { + text-align: center; + } + li { + background-color: #f4f4f4; + list-style-type: none; + } + li li { + background-color: white; + } + li ul { + margin-top: 4px; + margin-bottom: 8px; + } + .detail { + font-weight: normal; + font-style: italic; + } + .updates { + font: monospace; + font-size: smaller; + } + .status { + font-weight: bold; + padding-left: 1em; + padding-right: 1em; + } + .in-progress { + font-weight: bold; + } + .failed { + color: Maroon; + font-weight: bold; + } + .waiting { + color: Silver; /* Gray ? */ + } + .progress { + font-family: monospace; + font-size: 80%%; + margin-left: .5in; + } + </style> +</head> + +<body> + <h1>Database dump progress</h1> + + <p class="siteinfo"> + This is the Wikimedia dump service. + Please read the <a href='legal.html'>copyrights</a> information. + See <a href="http://meta.wikimedia.org/wiki/Data_dumps";>Meta:Data dumps</a> + for documentation on the provided data formats. + </p> + + <div class="status"> + %(status)s + </div> + + <ul> + %(items)s + </ul> + +</body> +</html> diff --git a/modules/snapshot/files/dumps/templates/report.html b/modules/snapshot/files/dumps/templates/report.html new file mode 100644 index 0000000..81fe37f --- /dev/null +++ b/modules/snapshot/files/dumps/templates/report.html @@ -0,0 +1,96 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd";> + +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> +<head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/> + <title>%(db)s dump progress on %(date)s</title> + <style type="text/css"> + html, body { + background-color: #eee; + color: black; + } + .siteinfo { + text-align: center; + } + li { + background-color: #f4f4f4; + list-style-type: none; + } + li li { + background-color: white; + } + li ul { + margin-top: 4px; + margin-bottom: 8px; + } + .detail { + font-weight: normal; + font-style: italic; + } + .updates { + font: monospace; + font-size: smaller; + } + .status { + font-weight: bold; + } + .status, .checksum { + padding-left: 1em; + padding-right: 1em; + } + .in-progress { + font-weight: bold; + } + .failed { + color: Maroon; + font-weight: bold; + } + .notice { + color: Maroon; + font-weight: normal; + } + .waiting { + color: Silver; /* Gray ? */ + } + .progress { + font-family: monospace; + font-size: 80%%; + margin-left: .5in; + } + </style> +</head> + +<body> + <h1>%(db)s dump progress on %(date)s</h1> + + <p class="siteinfo"> + This is the Wikimedia dump service. + Please read the <a href='../../legal.html'>copyrights</a> information. + See <a href="http://meta.wikimedia.org/wiki/Data_dumps";>Meta:Data dumps</a> + for documentation on the provided data formats. + </p> + + <p class="notice">%(notice)s</p> + + <p>See <a href="../../%(index)s">all databases list</a>.</p> + + <p class="previous"> + %(previous)s + </p> + + <p class="status"> + %(status)s + </p> + + <p class="checksum"> + Verify downloaded files against the <a href="%(checksum)s">MD5 checksums</a> + to check for corrupted files. + </p> + + <ul> + %(items)s + </ul> + +</body> +</html> diff --git a/modules/snapshot/manifests/dirs.pp b/modules/snapshot/manifests/dirs.pp new file mode 100644 index 0000000..a8bb847 --- /dev/null +++ b/modules/snapshot/manifests/dirs.pp @@ -0,0 +1,34 @@ +class snapshot::dirs { + $dumpsdir = '/srv/dumps' + $datadir = '/mnt/data/xmldatadumps' + $apachedir = '/apache/common' + + file { $dumpsdir: + ensure => 'directory', + path => $dumpsdir, + mode => '0644', + owner => 'root', + group => 'root', + } + + $addschangesdir = '/srv/addschanges' + + file { $addschangesdir: + ensure => 'directory', + path => $addschangesdir, + mode => '0644', + owner => 'root', + group => 'root', + } + + $wikiqueriesdir = '/srv/wikiqueries' + + file { $wikiqueriesdir: + ensure => 'directory', + path => $wikiqueriesdir, + mode => '0644', + owner => 'root', + group => 'root', + } + +} diff --git a/modules/snapshot/manifests/dumps.pp b/modules/snapshot/manifests/dumps.pp new file mode 100644 index 0000000..22665f4 --- /dev/null +++ b/modules/snapshot/manifests/dumps.pp @@ -0,0 +1,16 @@ +class snapshot::dumps( + $enable = true, + $hugewikis = false, +) { + class { 'snapshot::dumps::configs': + enable => $enable, + hugewikis_enable => $hugewikis, + } + class { 'snapshot::dumps::dblists': + enable => $enable, + hugewikis_enable => $hugewikis, + } + class { 'snapshot::dumps::templates': + enable => 'enable' + } +} diff --git a/modules/snapshot/manifests/dumps/configs.pp b/modules/snapshot/manifests/dumps/configs.pp new file mode 100644 index 0000000..1bc7a7b --- /dev/null +++ b/modules/snapshot/manifests/dumps/configs.pp @@ -0,0 +1,109 @@ +class snapshot::dumps::configs( + $enable = true, + $hugewikis_enable = true, +) { + + include snapshot::dirs + + $config = { + smallwikis => { + dblist => "${snapshot::dirs::apachedir}/all.dblist", + skipdblist => "${snapshot::dirs::dumpsdir}/dblists/skip.dblist", + keep => '14', + chunksEnabled => '0', + }, + bigwikis => { + dblist => "${snapshot::dirs::dumpsdir}/dblists/bigwikis.dblist", + skipdblist => "${snapshot::dirs::dumpsdir}/dblists/skipnone.dblist", + keep => '12', + chunksEnabled => '1', + recombineHistory => '0', + wikis => { + ruwiki => { + pagesPerChunkHistory => '256900,895900,1280900,2000000', + pagesPerChunkAbstract => '1200000', + }, + eswiki => { + pagesPerChunkHistory => '190300,671500,1627200,2000000', + pagesPerChunkAbstract => '1500000', + }, + dewiki => { + pagesPerChunkHistory => '33640,1446760,2569200,3000000', + pagesPerChunkAbstract => '2000000', + }, + ptwiki => { + pagesPerChunkHistory => '137500,668700,1208900,2000000', + pagesPerChunkAbstract => '1000000', + }, + plwiki => { + pagesPerChunkHistory => '208000,422000,818900,2000000', + pagesPerChunkAbstract => '800000', + }, + nlwiki => { + pagesPerChunkHistory => '200800,490800,934600,2000000', + pagesPerChunkAbstract => '1000000', + }, + frwiki => { + pagesPerChunkHistory => '348700,965200,2331100,3000000', + pagesPerChunkAbstract => '1900000', + }, + itwiki => { + pagesPerChunkHistory => '335400,941600,1180900,2000000', + pagesPerChunkAbstract => '1200000', + }, + jawiki => { + pagesPerChunkHistory => '149600,801600,408900,2000000', + pagesPerChunkAbstract => '800000', + }, + }, + }, + hugewikis => { + dblist => "${snapshot::dirs::dumpsdir}/dblists/hugewikis.dblist", + skipdblist => "${snapshot::dirs::dumpsdir}/dblists/skipnone.dblist", + keep => '10', + chunksEnabled => '1', + recombineHistory => '0', + checkpointTime => '720', + wikis => { + enwiki => { + pagesPerChunkHistory => '10000,15000,30000,50000,80000,120000,160000,200000,260000,400000,500000,600000,700000,800000,900000,1200000,1500000,1700000,1900000,2200000,2400000,2500000,2700000,2800000,2900000,3000000,3000000', + pagesPerChunkAbstract => '2000000', + }, + }, + }, + monitor => { + dblist => "${snapshot::dirs::apachedir}/all.dblist", + skipdblist => "${snapshot::dirs::dumpsdir}/dblists/skipmonitor.dblist", + keep => '30', + chunksEnabled => '0', + }, + } + + if ($enable) { + file { "${snapshot::dirs::dumpsdir}/confs": + ensure => 'directory', + path => "${snapshot::dirs::dumpsdir}/confs", + mode => '0755', + owner => 'root', + group => 'root', + } + snapshot::dumps::wikiconf { 'wikidumps.conf': + configtype => 'smallwikis', + config => $config, + } + snapshot::dumps::wikiconf { 'wikidumps.conf.bigwikis': + configtype => 'bigwikis', + config => $config, + } + if ($hugewikis_enable) { + snapshot::dumps::wikiconf { 'wikidumps.conf.hugewikis': + configtype => 'hugewikis', + config => $config, + } + } + snapshot::dumps::wikiconf { 'wikidumps.conf.monitor': + configtype => 'monitor', + config => $config, + } + } +} diff --git a/modules/snapshot/manifests/dumps/dblists.pp b/modules/snapshot/manifests/dumps/dblists.pp new file mode 100644 index 0000000..793baa6 --- /dev/null +++ b/modules/snapshot/manifests/dumps/dblists.pp @@ -0,0 +1,71 @@ +class snapshot::dumps::dblists($enable=true, $hugewikis_enable=false) { + if ($enable) { + $hugewikis = ['enwiki'] + $hugewikis_dblist = join($hugewikis, "\n") + + $bigwikis = ['dewiki', 'eswiki', 'frwiki', 'itwiki', 'jawiki', + 'nlwiki', 'plwiki', 'ptwiki', 'ruwiki'] + $bigwikis_dblist = join($bigwikis, "\n") + + $skip_dblist = "${hugewikis_dblist}\n${bigwikis_dblist}" + + $skipnone_dblist = '' + + include snapshot::dirs + + file { "${snapshot::dirs::dumpsdir}/dblists": + ensure => 'directory', + path => "${snapshot::dirs::dumpsdir}/dblists", + mode => '0755', + owner => 'root', + group => 'root', + } + if ($hugewikis_enable) { + # this host will run enwiki dumps + file { "${snapshot::dirs::dumpsdir}/dblists/hugewikis.dblist": + ensure => 'present', + path => "${snapshot::dirs::dumpsdir}/dblists/hugewikis.dblist", + mode => '0644', + owner => 'root', + group => 'root', + content => "${hugewikis_dblist}\n", + } + } + file { "${snapshot::dirs::dumpsdir}/dblists/bigwikis.dblist": + ensure => 'present', + path => "${snapshot::dirs::dumpsdir}/dblists/bigwikis.dblist", + mode => '0644', + owner => 'root', + group => 'root', + content => "${bigwikis_dblist}\n", + } + file { "${snapshot::dirs::dumpsdir}/dblists/skip.dblist": + ensure => 'present', + path => "${snapshot::dirs::dumpsdir}/dblists/skip.dblist", + mode => '0644', + owner => 'root', + group => 'root', + content => "${skip_dblist}\n", + } + file { "${snapshot::dirs::dumpsdir}/dblists/skipnone.dblist": + ensure => 'present', + path => "${snapshot::dirs::dumpsdir}/dblists/skipnone.dblist", + mode => '0644', + owner => 'root', + group => 'root', + content => "${skipnone_dblist}\n", + } + + $warning = "The files in this directory are maintained by puppet!\n" + $location = "puppet:///modules/snapshot/dumps/dblists.pp\n" + + file { "${snapshot::dirs::dumpsdir}/dblists/README": + ensure => 'present', + path => "${snapshot::dirs::dumpsdir}/dblists/README", + mode => '0644', + owner => 'root', + group => 'root', + content => "${warning}${location}", + } + } +} diff --git a/modules/snapshot/manifests/dumps/templates.pp b/modules/snapshot/manifests/dumps/templates.pp new file mode 100644 index 0000000..c45a1a0 --- /dev/null +++ b/modules/snapshot/manifests/dumps/templates.pp @@ -0,0 +1,82 @@ +class snapshot::dumps::templates($enable=true) { + if ($enable) { + + include snapshot::dirs + + file { "${snapshot::dirs::dumpsdir}/templs": + ensure => 'directory', + path => "${snapshot::dirs::dumpsdir}/templs", + mode => '0755', + owner => 'root', + group => 'root', + } + file { "${snapshot::dirs::dumpsdir}/templs/download-index.html": + ensure => 'present', + path => "${snapshot::dirs::dumpsdir}/templs/download-index.html", + mode => '0644', + owner => 'root', + group => 'root', + source => 'puppet:///modules/snapshot/dumps/templates/download-index.html', + } + file { "${snapshot::dirs::dumpsdir}/templs/dvd.html": + ensure => 'present', + path => "${snapshot::dirs::dumpsdir}/templs/dvd.html", + mode => '0644', + owner => 'root', + group => 'root', + source => 'puppet:///modules/snapshot/dumps/templates/dvd.html', + } + file { "${snapshot::dirs::dumpsdir}/templs/errormail.txt": + ensure => 'present', + path => "${snapshot::dirs::dumpsdir}/templs/errormail.txt", + mode => '0644', + owner => 'root', + group => 'root', + source => 'puppet:///modules/snapshot/dumps/templates/errormail.txt', + } + file { "${snapshot::dirs::dumpsdir}/templs/feed.xml": + ensure => 'present', + path => "${snapshot::dirs::dumpsdir}/templs/feed.xml", + mode => '0644', + owner => 'root', + group => 'root', + source => 'puppet:///modules/snapshot/dumps/templates/feed.xml', + } + file { "${snapshot::dirs::dumpsdir}/templs/legal.html": + ensure => 'present', + path => "${snapshot::dirs::dumpsdir}/templs/legal.html", + mode => '0644', + owner => 'root', + group => 'root', + source => 'puppet:///modules/snapshot/dumps/templates/legal.html', + } + file { "${snapshot::dirs::dumpsdir}/templs/progress.html": + ensure => 'present', + path => "${snapshot::dirs::dumpsdir}/templs/progress.html", + mode => '0644', + owner => 'root', + group => 'root', + source => 'puppet:///modules/snapshot/dumps/templates/progress.html', + } + file { "${snapshot::dirs::dumpsdir}/templs/report.html": + ensure => 'present', + path => "${snapshot::dirs::dumpsdir}/templs/report.html", + mode => '0644', + owner => 'root', + group => 'root', + source => 'puppet:///modules/snapshot/dumps/templates/report.html', + } + + $warning = "The files in this directory are maintained by puppet!\n" + $location = "puppet:///modules/snapshot/dumps/templates\n" + + file { "${snapshot::dirs::dumpsdir}/templs/README": + ensure => 'present', + path => "${snapshot::dirs::dumpsdir}/templs/README", + mode => '0644', + owner => 'root', + group => 'root', + content => "${warning}${location}", + } + } +} diff --git a/modules/snapshot/manifests/dumps/wikiconf.pp b/modules/snapshot/manifests/dumps/wikiconf.pp new file mode 100644 index 0000000..79ff655 --- /dev/null +++ b/modules/snapshot/manifests/dumps/wikiconf.pp @@ -0,0 +1,16 @@ +define snapshot::dumps::wikiconf( + $configtype = 'smallwikis', + $config = undef, + ) { + + include snapshot::dirs + + file { "${snapshot::dirs::dumpsdir}/confs/${title}": + ensure => 'present', + path => "${snapshot::dirs::dumpsdir}/confs/${title}", + mode => '0755', + owner => 'root', + group => 'root', + content => template('snapshot/wikidump.conf.erb'), + } +} diff --git a/modules/snapshot/templates/wikidump.conf.erb b/modules/snapshot/templates/wikidump.conf.erb new file mode 100644 index 0000000..f596958 --- /dev/null +++ b/modules/snapshot/templates/wikidump.conf.erb @@ -0,0 +1,75 @@ +############################################################# +# This file is maintained by puppet! +# puppet:///modules/snapshot/dumps/configs/wikidump.conf.erb +############################################################# + +[wiki] +dblist=<%= @config[configtype]['dblist'] %> +privatelist=<%= scope.lookupvar('snapshot::dirs::apachedir') -%>/private.dblist +flaggedrevslist=<%= scope.lookupvar('snapshot::dirs::apachedir') -%>/flaggedrevs.dblist +biglist=<%= scope.lookupvar('snapshot::dirs::apachedir') -%>/big.dblist +skipdblist=<%= @config[configtype]['skipdblist'] %> +wikidatalist=<%= scope.lookupvar('snapshot::dirs::apachedir') -%>/wikidata.dblist +dir=<%= scope.lookupvar('snapshot::dirs::apachedir') %> +adminsettings=wmf-config/AdminSettings.php +forcenormal=1 + +#testing +#halt=1 + +[output] +public=<%= scope.lookupvar('snapshot::dirs::datadir') -%>/public +private=<%= scope.lookupvar('snapshot::dirs::datadir') -%>/private +temp=<%= scope.lookupvar('snapshot::dirs::datadir') -%>/temp +templatedir=<%= scope.lookupvar('snapshot::dirs::dumpsdir') -%>/templs +index=backup-index.html +webroot=http://download.wikimedia.org +fileperms=0644 + +[reporting] +adminmail=ops-du...@wikimedia.org +mailfrom=r...@wikimedia.org +smtpserver=smtp.pmtpa.wmnet +staleage=3600 + +[database] + +[tools] +php=/usr/bin/php +mysql=/usr/bin/mysql +mysqldump=/usr/bin/mysqldump +gzip=/bin/gzip +bzip2=/bin/bzip2 +sevenzip=/usr/bin/7za +checkforbz2footer=/usr/local/bin/checkforbz2footer +writeuptopageid=/usr/local/bin/writeuptopageid +recompressxml=/usr/local/bin/recompressxml + +[cleanup] +keep=<%= @config[configtype]['keep'] %> + +[chunks] +<% if @config[configtype].has_key?('chunksEnabled') -%> +chunksEnabled=<%= @config[configtype]['chunksEnabled'] %> +<% end -%> +<% if @config[configtype].has_key?('recombineHistory') -%> +recombineHistory=<%= @config[configtype]['recombineHistory'] %> +<% end -%> +<% if @config[configtype].has_key?('checkpointTime') -%> +checkpointTime=<%= @config[configtype]['checkpointTime'] %> +<% end -%> + +[otherformats] +multistream=1 + +[pageslogging] +batchsize=1000000 + +<% if @config[configtype].has_key?('wikis') %> +<% @config[configtype]['wikis'].keys.sort.each do |w| -%> +[<%= [w] %>] +pagesPerChunkHistory=<%= @config[configtype]['wikis'][w]['pagesPerChunkHistory'] %> +pagesPerChunkAbstract=<%= @config[configtype]['wikis'][w]['pagesPerChunkAbstract'] %> + +<% end -%> +<% end -%> -- To view, visit https://gerrit.wikimedia.org/r/119006 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I1c8f2f61ff2d8986514bc75f80b678c4f4e4697a Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: ArielGlenn <ar...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
