ArielGlenn has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/396931 )
Change subject: get rid of datasets1001 mount on snapshot hosts
......................................................................
get rid of datasets1001 mount on snapshot hosts
As part of this cleanup:
get rid of secondary wikidumps.conf file which contained references
to paths on that filesystem
change all references to wikidumps.conf; these should now all point
to wikidumps.conf.dumps
in declaration of managed directories and paths for use by shell
scripts, reoder and compact the code
get rid of all references to 'otherdir' on snapshots, it's no longer
used
Bug: T182540
Change-Id: Iabfd52355aced16f5f4693b7c53be965e72652d2
---
M modules/profile/manifests/dumps/generation/worker/common.pp
M modules/snapshot/files/cron/create-media-per-project-lists.sh
M modules/snapshot/files/cron/dump-global-blocks.sh
M modules/snapshot/files/cron/dumpcategoriesrdf.sh
M modules/snapshot/files/cron/dumpcirrussearch.sh
M modules/snapshot/files/cron/dumpcontentxlation.sh
M modules/snapshot/manifests/cron/categoriesrdf.pp
M modules/snapshot/manifests/cron/cirrussearch.pp
M modules/snapshot/manifests/cron/dump_global_blocks.pp
M modules/snapshot/manifests/cron/pagetitles.pp
M modules/snapshot/manifests/dumps.pp
M modules/snapshot/manifests/dumps/configs.pp
M modules/snapshot/manifests/dumps/cron.pp
M modules/snapshot/manifests/dumps/dirs.pp
M modules/snapshot/templates/set_dump_dirs.sh.erb
15 files changed, 32 insertions(+), 74 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/31/396931/1
diff --git a/modules/profile/manifests/dumps/generation/worker/common.pp
b/modules/profile/manifests/dumps/generation/worker/common.pp
index 1ecf6fb..5c319c5 100644
--- a/modules/profile/manifests/dumps/generation/worker/common.pp
+++ b/modules/profile/manifests/dumps/generation/worker/common.pp
@@ -1,11 +1,11 @@
-class profile::dumps::generation::worker::common {
+class profile::dumps::generation::worker::common(
+) {
# mw packages and dependencies
require ::profile::mediawiki::scap_proxy
require ::profile::mediawiki::common
require ::profile::mediawiki::nutcracker
$xmldumpsmount = '/mnt/dumpsdata'
- $miscdumpsmount = '/mnt/data'
class { '::dumpsuser': }
@@ -13,21 +13,14 @@
mountpoint => $xmldumpsmount,
server => 'dumpsdata1001.eqiad.wmnet',
}
- snapshot::dumps::nfsmount { 'datasetmount':
- mountpoint => $miscdumpsmount,
- server => 'dataset1001.wikimedia.org',
- }
-
# dataset server config files,
# stages files, dblists, html templates
class { '::snapshot::dumps::dirs':
user => 'dumpsgen',
xmldumpsmount => $xmldumpsmount,
- miscdumpsmount => $miscdumpsmount,
}
class { '::snapshot::dumps':
xmldumpsmount => $xmldumpsmount,
- miscdumpsmount => $miscdumpsmount,
}
# scap3 deployment of dump scripts
diff --git a/modules/snapshot/files/cron/create-media-per-project-lists.sh
b/modules/snapshot/files/cron/create-media-per-project-lists.sh
index b51512b..6c5abb6 100755
--- a/modules/snapshot/files/cron/create-media-per-project-lists.sh
+++ b/modules/snapshot/files/cron/create-media-per-project-lists.sh
@@ -9,7 +9,7 @@
DATE=`/bin/date '+%Y%m%d'`
outputdir="${cronsdir}/imageinfo/$DATE"
-configfile="${confsdir}/wikidump.conf:media"
+configfile="${confsdir}/wikidump.conf.dumps:media"
errors=0
cd "$repodir"
diff --git a/modules/snapshot/files/cron/dump-global-blocks.sh
b/modules/snapshot/files/cron/dump-global-blocks.sh
index 90f80ea..4f5416a 100644
--- a/modules/snapshot/files/cron/dump-global-blocks.sh
+++ b/modules/snapshot/files/cron/dump-global-blocks.sh
@@ -76,12 +76,12 @@
echo "Usage: $0 [--config <pathtofile>] [--dryrun]" >& 2
echo >& 2
echo " --config path to configuration file for dump generation" >& 2
- echo " (default value: ${confsdir}/wikidump.conf" >& 2
+ echo " (default value: ${confsdir}/wikidump.conf.dumps" >& 2
echo " --dryrun don't run dump, show what would have been done" >& 2
exit 1
}
-configfile="${confsdir}/wikidump.conf"
+configfile="${confsdir}/wikidump.conf.dumps"
dryrun="false"
while [ $# -gt 0 ]; do
diff --git a/modules/snapshot/files/cron/dumpcategoriesrdf.sh
b/modules/snapshot/files/cron/dumpcategoriesrdf.sh
index abb175e..d55a692 100755
--- a/modules/snapshot/files/cron/dumpcategoriesrdf.sh
+++ b/modules/snapshot/files/cron/dumpcategoriesrdf.sh
@@ -13,13 +13,13 @@
echo "Usage: $0 --list wikis.dblist [--config <pathtofile>] [--dryrun]"
echo
echo " --config path to configuration file for dump generation"
- echo " (default value: ${confsdir}/wikidump.conf"
+ echo " (default value: ${confsdir}/wikidump.conf.dumps"
echo " --list file containing list of the wikis to dump"
echo " --dryrun don't run dump, show what would have been done"
exit 1
}
-configFile="${confsdir}/wikidump.conf"
+configFile="${confsdir}/wikidump.conf.dumps"
dryrun="false"
dumpFormat="ttl"
dbList="categories-rdf"
diff --git a/modules/snapshot/files/cron/dumpcirrussearch.sh
b/modules/snapshot/files/cron/dumpcirrussearch.sh
index 61c2cce0..9a7eeb6 100644
--- a/modules/snapshot/files/cron/dumpcirrussearch.sh
+++ b/modules/snapshot/files/cron/dumpcirrussearch.sh
@@ -13,12 +13,12 @@
echo "Usage: $0 [--config <pathtofile>] [--dryrun]"
echo
echo " --config path to configuration file for dump generation"
- echo " (default value: ${confsdir}/wikidump.conf"
+ echo " (default value: ${confsdir}/wikidump.conf.dumps"
echo " --dryrun don't run dump, show what would have been done"
exit 1
}
-configFile="${confsdir}/wikidump.conf"
+configFile="${confsdir}/wikidump.conf.dumps"
dryrun="false"
while [ $# -gt 0 ]; do
diff --git a/modules/snapshot/files/cron/dumpcontentxlation.sh
b/modules/snapshot/files/cron/dumpcontentxlation.sh
index b40a37b..c580a7e 100644
--- a/modules/snapshot/files/cron/dumpcontentxlation.sh
+++ b/modules/snapshot/files/cron/dumpcontentxlation.sh
@@ -26,7 +26,7 @@
echo "Usage: $0 [--config <pathtofile>] [--dryrun]"
echo
echo " --config path to configuration file for dump generation"
- echo " (default value: ${confsdir}/wikidump.conf"
+ echo " (default value: ${confsdir}/wikidump.conf.dumps"
echo " --dryrun display dump command instead of running it"
exit 1
}
@@ -35,7 +35,7 @@
# MAIN
#####################
-configfile="${confsdir}/wikidump.conf"
+configfile="${confsdir}/wikidump.conf.dumps"
dryrun="false"
#####################
diff --git a/modules/snapshot/manifests/cron/categoriesrdf.pp
b/modules/snapshot/manifests/cron/categoriesrdf.pp
index 8312683..6d21ac1 100644
--- a/modules/snapshot/manifests/cron/categoriesrdf.pp
+++ b/modules/snapshot/manifests/cron/categoriesrdf.pp
@@ -25,7 +25,7 @@
cron { 'categoriesrdf-dump':
ensure => 'present',
- command => "${scriptpath} --config ${confsdir}/wikidump.conf
--list ${apachedir}/dblists/categories-rdf.dblist",
+ command => "${scriptpath} --config ${confsdir}/wikidump.conf.dumps
--list ${apachedir}/dblists/categories-rdf.dblist",
environment => 'MAILTO=ops-du...@wikimedia.org',
user => $user,
minute => '0',
diff --git a/modules/snapshot/manifests/cron/cirrussearch.pp
b/modules/snapshot/manifests/cron/cirrussearch.pp
index 1babe11..ec17dd5 100644
--- a/modules/snapshot/manifests/cron/cirrussearch.pp
+++ b/modules/snapshot/manifests/cron/cirrussearch.pp
@@ -19,7 +19,7 @@
cron { 'cirrussearch-dump':
ensure => 'present',
- command => "${scriptpath} --config ${confsdir}/wikidump.conf",
+ command => "${scriptpath} --config
${confsdir}/wikidump.conf.dumps",
environment => 'MAILTO=ops-du...@wikimedia.org',
user => $user,
minute => '15',
diff --git a/modules/snapshot/manifests/cron/dump_global_blocks.pp
b/modules/snapshot/manifests/cron/dump_global_blocks.pp
index 25b4e7f..97e6dd2 100644
--- a/modules/snapshot/manifests/cron/dump_global_blocks.pp
+++ b/modules/snapshot/manifests/cron/dump_global_blocks.pp
@@ -12,7 +12,7 @@
cron { 'global_blocks_dump':
ensure => 'present',
- command => "/usr/local/bin/dump-global-blocks.sh --config
${confsdir}/wikidump.conf",
+ command => "/usr/local/bin/dump-global-blocks.sh --config
${confsdir}/wikidump.conf.dumps",
environment => 'MAILTO=ops-du...@wikimedia.org',
user => $user,
minute => '15',
diff --git a/modules/snapshot/manifests/cron/pagetitles.pp
b/modules/snapshot/manifests/cron/pagetitles.pp
index 3574143..0ddb782 100644
--- a/modules/snapshot/manifests/cron/pagetitles.pp
+++ b/modules/snapshot/manifests/cron/pagetitles.pp
@@ -9,7 +9,7 @@
ensure => 'present',
environment => 'MAILTO=ops-du...@wikimedia.org',
user => $user,
- command => "cd ${repodir}; python onallwikis.py --configfile
${confsdir}/wikidump.conf:monitor --filenameformat
'{w}-{d}-all-titles-in-ns-0.gz' --outdir '${cronsdir}/pagetitles/{d}' --query
\"'select page_title from page where page_namespace=0;'\"",
+ command => "cd ${repodir}; python onallwikis.py --configfile
${confsdir}/wikidump.conf.dumps:monitor --filenameformat
'{w}-{d}-all-titles-in-ns-0.gz' --outdir '${cronsdir}/pagetitles/{d}' --query
\"'select page_title from page where page_namespace=0;'\"",
minute => '10',
hour => '8',
}
@@ -18,7 +18,7 @@
ensure => 'present',
environment => 'MAILTO=ops-du...@wikimedia.org',
user => $user,
- command => "cd ${repodir}; python onallwikis.py --configfile
${confsdir}/wikidump.conf:monitor --filenameformat
'{w}-{d}-all-media-titles.gz' --outdir '${cronsdir}/mediatitles/{d}' --query
\"'select page_title from page where page_namespace=6;'\"",
+ command => "cd ${repodir}; python onallwikis.py --configfile
${confsdir}/wikidump.conf.dumps:monitor --filenameformat
'{w}-{d}-all-media-titles.gz' --outdir '${cronsdir}/mediatitles/{d}' --query
\"'select page_title from page where page_namespace=6;'\"",
minute => '50',
hour => '8',
}
diff --git a/modules/snapshot/manifests/dumps.pp
b/modules/snapshot/manifests/dumps.pp
index e2248cc..5c678e6 100644
--- a/modules/snapshot/manifests/dumps.pp
+++ b/modules/snapshot/manifests/dumps.pp
@@ -1,11 +1,9 @@
class snapshot::dumps(
$xmldumpsmount = undef,
- $miscdumpsmount = undef,
) {
include ::snapshot::dumps::packages
class { '::snapshot::dumps::configs':
xmldumpsmount => $xmldumpsmount,
- miscdumpsmount => $miscdumpsmount,
}
include ::snapshot::dumps::dblists
include ::snapshot::dumps::templates
diff --git a/modules/snapshot/manifests/dumps/configs.pp
b/modules/snapshot/manifests/dumps/configs.pp
index 1fb7ed4..e77eeaa 100644
--- a/modules/snapshot/manifests/dumps/configs.pp
+++ b/modules/snapshot/manifests/dumps/configs.pp
@@ -1,6 +1,5 @@
class snapshot::dumps::configs(
$xmldumpsmount = undef,
- $miscdumpsmount = undef,
) {
$dblistsdir = $snapshot::dumps::dirs::dblistsdir
$apachedir = $snapshot::dumps::dirs::apachedir
@@ -134,14 +133,6 @@
},
}
- # for jobs running on dataset host
- snapshot::dumps::wikiconf { 'wikidump.conf':
- configtype => 'allwikis',
- config => $config,
- publicdir => "${miscdumpsmount}/xmldatadumps/public",
- privatedir => "${miscdumpsmount}/xmldatadumps/private",
- tempdir => "${miscdumpsmount}/xmldatadumps/temp",
- }
# for xml/sql dumps running on dumpsdata host
snapshot::dumps::wikiconf { 'wikidump.conf.dumps':
configtype => 'allwikis',
diff --git a/modules/snapshot/manifests/dumps/cron.pp
b/modules/snapshot/manifests/dumps/cron.pp
index 3adb2b8..dee3c72 100644
--- a/modules/snapshot/manifests/dumps/cron.pp
+++ b/modules/snapshot/manifests/dumps/cron.pp
@@ -28,7 +28,7 @@
}
# fixme there is an implicit dependency on
- # wikidump.conf.* plus some stage files, make explicit
+ # wikidump.conf.dumps plus some stage files, make explicit
cron { 'fulldumps_rest':
ensure => 'present',
diff --git a/modules/snapshot/manifests/dumps/dirs.pp
b/modules/snapshot/manifests/dumps/dirs.pp
index d54bd13..72603bd 100644
--- a/modules/snapshot/manifests/dumps/dirs.pp
+++ b/modules/snapshot/manifests/dumps/dirs.pp
@@ -1,72 +1,49 @@
class snapshot::dumps::dirs(
$user = undef,
$xmldumpsmount = undef,
- $miscdumpsmount = undef,
) {
+ # need to create and manage these, and have them
+ # available for a shell script that sets vars with
+ # their values for inclusion by other scripts
$dumpsdir = '/etc/dumps'
file { $dumpsdir:
ensure => 'directory',
- path => $dumpsdir,
mode => '0755',
owner => 'root',
group => 'root',
}
- $miscdumpsdir = "${miscdumpsmount}/xmldatadumps"
- $xmldumpsdir = "${xmldumpsmount}/xmldatadumps"
- $cronsdir = "${xmldumpsmount}/otherdumps"
-
- $apachedir = '/srv/mediawiki'
$confsdir = "${dumpsdir}/confs"
-
- file { $confsdir:
- ensure => 'directory',
- path => $confsdir,
- mode => '0755',
- owner => 'root',
- group => 'root',
- }
-
$dblistsdir = "${dumpsdir}/dblists"
- file { $dblistsdir:
- ensure => 'directory',
- path => $dblistsdir,
- mode => '0755',
- owner => 'root',
- group => 'root',
- }
-
$stagesdir = "${dumpsdir}/stages"
- file { $stagesdir:
+ $templsdir = "${dumpsdir}/templs"
+ file { [ $confsdir, $dblistsdir, $stagesdir,
+ $templsdir ]:
ensure => 'directory',
- path => $stagesdir,
mode => '0755',
owner => 'root',
group => 'root',
}
+ # dir will be written in by the user
$cachedir = "${dumpsdir}/cache"
file { $cachedir:
ensure => 'directory',
- path => $cachedir,
mode => '0755',
owner => $user,
group => 'root',
}
- $templsdir = "${dumpsdir}/templs"
- file { $templsdir:
- ensure => 'directory',
- path => $templsdir,
- mode => '0755',
- owner => 'root',
- group => 'root',
- }
-
- $otherdir = "${miscdumpsdir}/public/other"
-
+ # need these only for the shell script that sets
+ # vars with their values for other scripts
+ $xmldumpsdir = "${xmldumpsmount}/xmldatadumps"
+ $cronsdir = "${xmldumpsmount}/otherdumps"
+ $apachedir = '/srv/mediawiki'
$repodir = '/srv/deployment/dumps/dumps/xmldumps-backup'
+ # here's that script; it gets sourced by
+ # various cron jobs so they know where to
+ # write output, where to find dump scripts, etc.
file { '/usr/local/etc/set_dump_dirs.sh':
ensure => 'present',
path => '/usr/local/etc/set_dump_dirs.sh',
diff --git a/modules/snapshot/templates/set_dump_dirs.sh.erb
b/modules/snapshot/templates/set_dump_dirs.sh.erb
index 6313c23..5a846da 100644
--- a/modules/snapshot/templates/set_dump_dirs.sh.erb
+++ b/modules/snapshot/templates/set_dump_dirs.sh.erb
@@ -5,7 +5,6 @@
repodir="<%= scope.lookupvar('snapshot::dumps::dirs::repodir') -%>"
xmldumpsdir="<%= scope.lookupvar('snapshot::dumps::dirs::xmldumpsdir') -%>"
miscdumpsdir="<%= scope.lookupvar('snapshot::dumps::dirs::miscdumpsdir') -%>"
-otherdir="<%= scope.lookupvar('snapshot::dumps::dirs::otherdir') -%>"
cronsdir="<%= scope.lookupvar('snapshot::dumps::dirs::cronsdir') -%>"
dumpsdir="<%= scope.lookupvar('snapshot::dumps::dirs::dumpsdir') -%>"
dblistsdir="<%= scope.lookupvar('snapshot::dumps::dirs::dblistsdir') -%>"
--
To view, visit https://gerrit.wikimedia.org/r/396931
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Iabfd52355aced16f5f4693b7c53be965e72652d2
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <ar...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
