ArielGlenn has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/390000 )
Change subject: move hardcoded references to dump server mount points out to
profile
......................................................................
move hardcoded references to dump server mount points out to profile
Bug: T179942
Change-Id: I34e3d9261d279af2e5da3d7384f8508e094bc412
---
D modules/dumps/manifests/generation/client/nfs.pp
M modules/profile/manifests/dumps/generation/worker/common.pp
M modules/snapshot/files/cron/list-last-good-dumps.sh
M modules/snapshot/files/cron/wikidatadumps-shared.sh
M modules/snapshot/manifests/dumps.pp
M modules/snapshot/manifests/dumps/configs.pp
M modules/snapshot/manifests/dumps/dirs.pp
A modules/snapshot/manifests/dumps/nfsmount.pp
M modules/snapshot/manifests/dumps/packages.pp
M modules/snapshot/manifests/dumps/wikiconf.pp
M modules/snapshot/templates/addschanges.conf.erb
M modules/snapshot/templates/dumps/wikidump.conf.erb
M modules/snapshot/templates/set_dump_dirs.sh.erb
13 files changed, 77 insertions(+), 103 deletions(-)
Approvals:
ArielGlenn: Looks good to me, approved
jenkins-bot: Verified
diff --git a/modules/dumps/manifests/generation/client/nfs.pp
b/modules/dumps/manifests/generation/client/nfs.pp
deleted file mode 100644
index 7f2ea1b..0000000
--- a/modules/dumps/manifests/generation/client/nfs.pp
+++ /dev/null
@@ -1,22 +0,0 @@
-class dumps::generation::client::nfs {
- require_package('nfs-common')
-
- file { [ '/mnt/dumpsdata' ]:
- ensure => 'directory',
- }
-
- $dumpsdataserver = $::site ? {
- 'eqiad' => 'dumpsdata1001.eqiad.wmnet',
- default => 'dumpsdata1001.eqiad.wmnet',
- }
-
- mount { '/mnt/dumpsdata':
- ensure => 'mounted',
- device => "${dumpsdataserver}:/data",
- fstype => 'nfs',
- name => '/mnt/dumpsdata',
- options => 'bg,hard,tcp,rsize=8192,wsize=8192,intr,nfsvers=3',
- require => File['/mnt/dumpsdata'],
- remounts => false,
- }
-}
diff --git a/modules/profile/manifests/dumps/generation/worker/common.pp
b/modules/profile/manifests/dumps/generation/worker/common.pp
index 29acf43..d6fa82b 100644
--- a/modules/profile/manifests/dumps/generation/worker/common.pp
+++ b/modules/profile/manifests/dumps/generation/worker/common.pp
@@ -4,14 +4,32 @@
require ::profile::mediawiki::common
require ::profile::mediawiki::nutcracker
- # dataset server nfs mount, config files,
- # stages files, dblists, html templates
+ $xmldumpsmount = '/mnt/dumpsdata'
+ $miscdumpsmount = '/mnt/data'
+
class { '::dumpsuser': }
class { '::dumps::deprecated::user': }
- class { '::snapshot::dumps::dirs':
- user => 'dumpsgen',
+
+ snapshot::dumps::nfsmount { 'dumpsdatamount':
+ mountpoint => $xmldumpsmount,
+ server => 'dumpsdata1001.eqiad.wmnet',
}
- class { '::snapshot::dumps': }
+ snapshot::dumps::nfsmount { 'datasetmount':
+ mountpoint => $miscdumpsmount,
+ server => 'dataset1001.wikimedia.org',
+ }
+
+ # dataset server config files,
+ # stages files, dblists, html templates
+ class { '::snapshot::dumps::dirs':
+ user => 'dumpsgen',
+ xmldumpsmount => $xmldumpsmount,
+ miscdumpsmount => $miscdumpsmount,
+ }
+ class { '::snapshot::dumps':
+ xmldumpsmount => $xmldumpsmount,
+ miscdumpsmount => $miscdumpsmount,
+ }
# scap3 deployment of dump scripts
scap::target { 'dumps/dumps':
diff --git a/modules/snapshot/files/cron/list-last-good-dumps.sh
b/modules/snapshot/files/cron/list-last-good-dumps.sh
index d00d3e7..4f78473 100644
--- a/modules/snapshot/files/cron/list-last-good-dumps.sh
+++ b/modules/snapshot/files/cron/list-last-good-dumps.sh
@@ -7,7 +7,7 @@
source /usr/local/etc/set_dump_dirs.sh
# generate lists of most recent completed successful dumps for rsync (dirs,
files)
-python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber 1,2,3,4,5
--dirlisting 'rsync-dirlist-last-%s-good.txt' --configfile
"${confsdir}/wikidump.conf:monitor" --rsynclists --relpath --outputdir
"${datadir}/public/"
-python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber 1,2,3,4,5
--filelisting 'rsync-filelist-last-%s-good.txt' --configfile
"${confsdir}/wikidump.conf:monitor" --rsynclists --relpath --outputdir
"${datadir}/public/" --toplevel
+python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber 1,2,3,4,5
--dirlisting 'rsync-dirlist-last-%s-good.txt' --configfile
"${confsdir}/wikidump.conf:monitor" --rsynclists --relpath --outputdir
"${miscdumpsdir}/public/"
+python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber 1,2,3,4,5
--filelisting 'rsync-filelist-last-%s-good.txt' --configfile
"${confsdir}/wikidump.conf:monitor" --rsynclists --relpath --outputdir
"${miscdumpsdir}/public/" --toplevel
# these lists can be used for rsync excl/incl on our side, providing shares
that "just work" for the mirrors
-python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber 1,2,3,4,5
--rsynclisting 'rsync-inc-last-%s.txt' --configfile
"${confsdir}/wikidump.conf:monitor" --relpath --outputdir "${datadir}/public/"
--toplevel
+python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber 1,2,3,4,5
--rsynclisting 'rsync-inc-last-%s.txt' --configfile
"${miscdumpsdir}/wikidump.conf:monitor" --relpath --outputdir
"${miscdumpsdir}/public/" --toplevel
diff --git a/modules/snapshot/files/cron/wikidatadumps-shared.sh
b/modules/snapshot/files/cron/wikidatadumps-shared.sh
index 460ed02..1a15df2 100644
--- a/modules/snapshot/files/cron/wikidatadumps-shared.sh
+++ b/modules/snapshot/files/cron/wikidatadumps-shared.sh
@@ -18,10 +18,7 @@
results=`python "${repodir}/getconfigvals.py" --configfile "$configfile"
--args "$args"`
apacheDir=`getsetting "$results" "wiki" "dir"` || exit 1
-#tempDir=`getsetting "$results" "output" "temp"` || exit 1
-# while jobs are split between dumpsdata and dataset hosts, fix this path
-# for those jobs remaining on dataset1001 for now
-tempDir="/mnt/data/xmldatadumps/temp"
+tempDir=`getsetting "$results" "output" "temp"` || exit 1
for settingname in "apacheDir" "tempDir"; do
checkval "$settingname" "${!settingname}"
diff --git a/modules/snapshot/manifests/dumps.pp
b/modules/snapshot/manifests/dumps.pp
index c96b929..e2248cc 100644
--- a/modules/snapshot/manifests/dumps.pp
+++ b/modules/snapshot/manifests/dumps.pp
@@ -1,6 +1,12 @@
-class snapshot::dumps {
+class snapshot::dumps(
+ $xmldumpsmount = undef,
+ $miscdumpsmount = undef,
+) {
include ::snapshot::dumps::packages
- include ::snapshot::dumps::configs
+ class { '::snapshot::dumps::configs':
+ xmldumpsmount => $xmldumpsmount,
+ miscdumpsmount => $miscdumpsmount,
+ }
include ::snapshot::dumps::dblists
include ::snapshot::dumps::templates
include ::snapshot::dumps::stagesconfig
diff --git a/modules/snapshot/manifests/dumps/configs.pp
b/modules/snapshot/manifests/dumps/configs.pp
index 318efe4..1fb7ed4 100644
--- a/modules/snapshot/manifests/dumps/configs.pp
+++ b/modules/snapshot/manifests/dumps/configs.pp
@@ -1,4 +1,7 @@
-class snapshot::dumps::configs {
+class snapshot::dumps::configs(
+ $xmldumpsmount = undef,
+ $miscdumpsmount = undef,
+) {
$dblistsdir = $snapshot::dumps::dirs::dblistsdir
$apachedir = $snapshot::dumps::dirs::apachedir
$confsdir = $snapshot::dumps::dirs::confsdir
@@ -135,17 +138,17 @@
snapshot::dumps::wikiconf { 'wikidump.conf':
configtype => 'allwikis',
config => $config,
- publicdir => '/mnt/data/xmldatadumps/public',
- privatedir => '/mnt/data/xmldatadumps/private',
- tempdir => '/mnt/data/xmldatadumps/temp',
+ publicdir => "${miscdumpsmount}/xmldatadumps/public",
+ privatedir => "${miscdumpsmount}/xmldatadumps/private",
+ tempdir => "${miscdumpsmount}/xmldatadumps/temp",
}
# for xml/sql dumps running on dumpsdata host
snapshot::dumps::wikiconf { 'wikidump.conf.dumps':
configtype => 'allwikis',
config => $config,
- publicdir => '/mnt/dumpsdata/xmldatadumps/public',
- privatedir => '/mnt/dumpsdata/xmldatadumps/private',
- tempdir => '/mnt/dumpsdata/xmldatadumps/temp',
+ publicdir => "${xmldumpsmount}/xmldatadumps/public",
+ privatedir => "${xmldumpsmount}/xmldatadumps/private",
+ tempdir => "${xmldumpsmount}/xmldatadumps/temp",
}
file { "${confsdir}/table_jobs.yaml":
diff --git a/modules/snapshot/manifests/dumps/dirs.pp
b/modules/snapshot/manifests/dumps/dirs.pp
index 33b93db..01736b3 100644
--- a/modules/snapshot/manifests/dumps/dirs.pp
+++ b/modules/snapshot/manifests/dumps/dirs.pp
@@ -1,5 +1,7 @@
class snapshot::dumps::dirs(
$user = undef,
+ $xmldumpsmount = undef,
+ $miscdumpsmount = undef,
) {
$dumpsdir = '/etc/dumps'
file { $dumpsdir:
@@ -10,7 +12,9 @@
group => 'root',
}
- $datadir = '/mnt/data/xmldatadumps'
+ $miscdumpsdir = "${miscdumpsmount}/xmldatadumps"
+ $xmldumpsdir = "${xmldumpsmount}/xmldatadumps"
+
$apachedir = '/srv/mediawiki'
$confsdir = "${dumpsdir}/confs"
@@ -58,12 +62,7 @@
group => 'root',
}
- # maintained on the NFS fileserver, not here
- # but we need to know the path, used for
- # dumps and datasets other than the main
- # xml/sql dumps
- # make this explicit for now
- $otherdir = '/mnt/data/xmldatadumps/public/other'
+ $otherdir = "${miscdumpsdir}/public/other"
$repodir = '/srv/deployment/dumps/dumps/xmldumps-backup'
@@ -75,5 +74,4 @@
group => 'root',
content => template('snapshot/set_dump_dirs.sh.erb'),
}
-
}
diff --git a/modules/snapshot/manifests/dumps/nfsmount.pp
b/modules/snapshot/manifests/dumps/nfsmount.pp
new file mode 100644
index 0000000..94f76a8
--- /dev/null
+++ b/modules/snapshot/manifests/dumps/nfsmount.pp
@@ -0,0 +1,20 @@
+define snapshot::dumps::nfsmount(
+ $mountpoint = undef,
+ $server = undef,
+) {
+ require_package('nfs-common')
+
+ file { [ $mountpoint ]:
+ ensure => 'directory',
+ }
+
+ mount { $mountpoint:
+ ensure => 'mounted',
+ device => "${server}:/data",
+ fstype => 'nfs',
+ name => $mountpoint,
+ options => 'bg,hard,tcp,rsize=8192,wsize=8192,intr,nfsvers=3',
+ require => File[$mountpoint],
+ remounts => false,
+ }
+}
diff --git a/modules/snapshot/manifests/dumps/packages.pp
b/modules/snapshot/manifests/dumps/packages.pp
index ca55cc0..5305ca2 100644
--- a/modules/snapshot/manifests/dumps/packages.pp
+++ b/modules/snapshot/manifests/dumps/packages.pp
@@ -2,43 +2,4 @@
require_package('mwbzutils')
require_package('mysql-client')
require_package('p7zip-full')
- require_package('nfs-common')
-
- file { [ '/mnt/data' ]:
- ensure => 'directory',
- }
-
- $datasetserver = $::site ? {
- 'eqiad' => 'dataset1001.wikimedia.org',
- default => 'dataset1001.wikimedia.org',
- }
-
- mount { '/mnt/data':
- ensure => 'mounted',
- device => "${datasetserver}:/data",
- fstype => 'nfs',
- name => '/mnt/data',
- options => 'bg,hard,tcp,rsize=8192,wsize=8192,intr,nfsvers=3',
- require => File['/mnt/data'],
- remounts => false,
- }
-
- file { [ '/mnt/dumpsdata' ]:
- ensure => 'directory',
- }
-
- $dumpsgenserver = $::site ? {
- 'eqiad' => 'dumpsdata1001.eqiad.wmnet',
- default => 'dumpsdata1001.eqiad.wmnet',
- }
-
- mount { '/mnt/dumpsdata':
- ensure => 'mounted',
- device => "${dumpsgenserver}:/data",
- fstype => 'nfs',
- name => '/mnt/dumpsdata',
- options => 'bg,hard,tcp,rsize=8192,wsize=8192,intr,nfsvers=3',
- require => File['/mnt/dumpsdata'],
- remounts => false,
- }
}
diff --git a/modules/snapshot/manifests/dumps/wikiconf.pp
b/modules/snapshot/manifests/dumps/wikiconf.pp
index 32b3ea8..84f8956 100644
--- a/modules/snapshot/manifests/dumps/wikiconf.pp
+++ b/modules/snapshot/manifests/dumps/wikiconf.pp
@@ -1,9 +1,9 @@
define snapshot::dumps::wikiconf(
$configtype = 'allwikis',
$config = undef,
- $publicdir = '/mnt/data/xmldatadumps/public',
- $privatedir = '/mnt/data/xmldatadumps/private',
- $tempdir = '/mnt/data/xmldatadumps/temp',
+ $publicdir = undef,
+ $privatedir = undef,
+ $tempdir = undef,
) {
$confsdir = $snapshot::dumps::dirs::confsdir
diff --git a/modules/snapshot/templates/addschanges.conf.erb
b/modules/snapshot/templates/addschanges.conf.erb
index ecdfd84..540d16b 100644
--- a/modules/snapshot/templates/addschanges.conf.erb
+++ b/modules/snapshot/templates/addschanges.conf.erb
@@ -17,10 +17,7 @@
dumpdir=<%= scope.lookupvar('snapshot::dumps::dirs::otherdir') -%>/incr
templatedir=<%= scope.lookupvar('snapshot::dumps::dirs::templsdir') %>
indextmpl=<%= scope.lookupvar('snapshot::dumps::dirs::templsdir')
-%>/incrs-index.html
-#temp=<%= scope.lookupvar('snapshot::dumps::dirs::datadir') -%>/temp
-# hardcode this during the period where some jobs will be moved to dumpsdata
-# hosts but this one will not
-temp=/mnt/data/xmldatadumps/temp
+temp=<%= scope.lookupvar('snapshot::dumps::dirs::miscdumpsdir') -%>/temp
webroot=http://download.wikimedia.org
fileperms=0644
# revisions must be at least this much older than time of current run
diff --git a/modules/snapshot/templates/dumps/wikidump.conf.erb
b/modules/snapshot/templates/dumps/wikidump.conf.erb
index 00e614c..2024be7 100644
--- a/modules/snapshot/templates/dumps/wikidump.conf.erb
+++ b/modules/snapshot/templates/dumps/wikidump.conf.erb
@@ -14,11 +14,6 @@
tablejobs=<%= scope.lookupvar('snapshot::dumps::dirs::confsdir')
-%>/table_jobs.yaml
[output]
-#public=<%= scope.lookupvar('snapshot::dumps::dirs::datadir') -%>/public
-#private=<%= scope.lookupvar('snapshot::dumps::dirs::datadir') -%>/private
-#temp=<%= scope.lookupvar('snapshot::dumps::dirs::datadir') -%>/temp
-# we have these take on values from the defined resource, until move to
-# dumpsdata host of all jobs is complete
public=<%= @publicdir %>
private=<%= @privatedir %>
temp=<%= @tempdir %>
diff --git a/modules/snapshot/templates/set_dump_dirs.sh.erb
b/modules/snapshot/templates/set_dump_dirs.sh.erb
index 03ac0e6..fbb73e9 100644
--- a/modules/snapshot/templates/set_dump_dirs.sh.erb
+++ b/modules/snapshot/templates/set_dump_dirs.sh.erb
@@ -3,7 +3,8 @@
# set some directory paths many cron jobs need
confsdir="<%= scope.lookupvar('snapshot::dumps::dirs::confsdir') -%>"
repodir="<%= scope.lookupvar('snapshot::dumps::dirs::repodir') -%>"
-datadir="<%= scope.lookupvar('snapshot::dumps::dirs::datadir') -%>"
+xmldumpsdir="<%= scope.lookupvar('snapshot::dumps::dirs::xmldumpsdir') -%>"
+miscdumpsdir="<%= scope.lookupvar('snapshot::dumps::dirs::miscdumpsdir') -%>"
otherdir="<%= scope.lookupvar('snapshot::dumps::dirs::otherdir') -%>"
dumpsdir="<%= scope.lookupvar('snapshot::dumps::dirs::dumpsdir') -%>"
dblistsdir="<%= scope.lookupvar('snapshot::dumps::dirs::dblistsdir') -%>"
--
To view, visit https://gerrit.wikimedia.org/r/390000
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I34e3d9261d279af2e5da3d7384f8508e094bc412
Gerrit-PatchSet: 6
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <[email protected]>
Gerrit-Reviewer: ArielGlenn <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
