ArielGlenn has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/395612 )
Change subject: move adds-changes (so-called incrementals) dump cron to
dumpsdata hosts
......................................................................
move adds-changes (so-called incrementals) dump cron to dumpsdata hosts
Also fix up ownerships of other directories on web server, for cron
jobs that have been moved to the new nfs server and new user. This saves
puppet from resetting the ownership to the old bad user after every
rsync run from the new nfs server to the web server corrects it.
Bug: T179942
Change-Id: I8bbccbe13226bb3f12754651da6d0a4d37060d5f
---
M modules/dumps/manifests/generation/server/dirs.pp
M modules/dumps/manifests/web/cleanups/miscdumps.pp
M modules/dumps/manifests/web/dirs.pp
M modules/dumps/manifests/web/html.pp
M modules/dumps/manifests/web/xmldumps.pp
M modules/dumps/manifests/web/xmldumps_active.pp
M modules/profile/manifests/dumps/web/xmldumps_active.pp
M modules/profile/manifests/dumps/web/xmldumps_fallback.pp
M modules/snapshot/manifests/addschanges.pp
M modules/snapshot/manifests/cron.pp
M modules/snapshot/templates/addschanges.conf.erb
11 files changed, 69 insertions(+), 48 deletions(-)
Approvals:
ArielGlenn: Looks good to me, approved
jenkins-bot: Verified
diff --git a/modules/dumps/manifests/generation/server/dirs.pp
b/modules/dumps/manifests/generation/server/dirs.pp
index e370766..f534e03 100644
--- a/modules/dumps/manifests/generation/server/dirs.pp
+++ b/modules/dumps/manifests/generation/server/dirs.pp
@@ -15,7 +15,7 @@
$categoriesrdfdir = "${otherdir}/categoriesrdf"
$globalblocksdir = "${otherdir}/globalblocks"
$medialistsdir = "${otherdir}/imageinfo"
- $incrsdir = "${otherdir}/incrs"
+ $incrsdir = "${otherdir}/incr"
$mediatitlesdir = "${otherdir}/mediatitles"
$othermiscdir = "${otherdir}/misc"
$pagetitlesdir = "${otherdir}/pagetitles"
diff --git a/modules/dumps/manifests/web/cleanups/miscdumps.pp
b/modules/dumps/manifests/web/cleanups/miscdumps.pp
index 5a4531c..5ef152c 100644
--- a/modules/dumps/manifests/web/cleanups/miscdumps.pp
+++ b/modules/dumps/manifests/web/cleanups/miscdumps.pp
@@ -12,7 +12,6 @@
}
$keep_generator=['categoriesrdf:3', 'cirrussearch:3',
'contenttranslation:3', 'globalblocks:3', 'imageinfo:3', 'mediatitles:3',
'pagetitles:3', 'wikibase/wikidatawiki:3']
- # FIXME check for imageinfo, mediatitles, pagetitles that we really
want/need all those
$keep_replicas=['categoriesrdf:11', 'cirrussearch:11',
'contenttranslation:14', 'globalblocks:13', 'imageinfo:32', 'mediatitles:90',
'pagetitles:90', 'wikibase/wikidatawiki:20']
if ($isreplica == true) {
$content= join($keep_replicas, "\n")
@@ -29,10 +28,21 @@
content => "${content}\n"
}
+ $cleanup_miscdumps = "/bin/bash /usr/local/bin/cleanup_old_miscdumps.sh
--miscdumpsdir ${miscdumpsdir} --configfile /etc/dumps/confs/cleanup_misc.conf"
+
+ if ($isreplica == true) {
+ $addschanges_keeps = '40'
+ } else {
+ $addschanges_keeps = '7'
+ }
+
+ # adds-changes dumps cleanup; these are in incr/wikiname/YYYYMMDD for each
day, so they can't go into the above config/cron setup
+ $cleanup_addschanges = "find ${miscdumpsdir}/incr -mindepth 2 -maxdepth 2
-type d -mtime +${addschanges_keeps} -exec rm -rf {} \\;"
+
cron { 'cleanup-misc-dumps':
ensure => 'present',
environment => 'MAILTO=ops-du...@wikimedia.org',
- command => "/bin/bash /usr/local/bin/cleanup_old_miscdumps.sh
--miscdumpsdir ${miscdumpsdir} --configfile /etc/dumps/confs/cleanup_misc.conf",
+ command => "${cleanup_miscdumps} ; ${cleanup_addschanges}",
user => root,
minute => '15',
hour => '7',
diff --git a/modules/dumps/manifests/web/dirs.pp
b/modules/dumps/manifests/web/dirs.pp
index e611ae6..0e09bde 100644
--- a/modules/dumps/manifests/web/dirs.pp
+++ b/modules/dumps/manifests/web/dirs.pp
@@ -4,6 +4,8 @@
$otherdir = '/data/xmldatadumps/public/other',
$user = undef,
$group = undef,
+ $deprecated_user = undef,
+ $deprecated_group = undef,
) {
# Please note that this is incomplete, but new directories
# should be defined in puppet (here).
@@ -14,7 +16,6 @@
$otherdir_wikibase = "${otherdir}/wikibase/"
$relative_wikidatawiki = 'other/wikibase/wikidatawiki'
$xlationdir = "${otherdir}/contenttranslation"
- $centralauthdir = '/data/xmldatadumps/private/centralauth'
$cirrussearchdir = "${otherdir}/cirrussearch"
$medialistsdir = "${otherdir}/imageinfo"
$pagetitlesdir = "${otherdir}/pagetitles"
@@ -33,50 +34,50 @@
file { $publicdir:
ensure => 'directory',
mode => '0775',
- owner => $user,
- group => $group,
+ owner => $deprecated_user,
+ group => $deprecated_group,
}
file { $otherdir:
ensure => 'directory',
mode => '0755',
- owner => $user,
- group => $group,
+ owner => $deprecated_user,
+ group => $deprecated_group,
}
file { $analyticsdir:
ensure => 'directory',
mode => '0755',
- owner => $user,
- group => $group,
+ owner => $deprecated_user,
+ group => $deprecated_group,
}
file { $othermiscdir:
ensure => 'directory',
mode => '0755',
- owner => $user,
- group => $group,
+ owner => $deprecated_user,
+ group => $deprecated_group,
}
file { $othertestfilesdir:
ensure => 'directory',
mode => '0755',
- owner => $user,
- group => $group,
+ owner => $deprecated_user,
+ group => $deprecated_group,
}
file { $otherdir_wikibase:
ensure => 'directory',
mode => '0755',
- owner => $user,
- group => $group,
+ owner => $deprecated_user,
+ group => $deprecated_group,
}
file { "${publicdir}/${relative_wikidatawiki}":
ensure => 'directory',
mode => '0755',
- owner => $user,
- group => $group,
+ owner => $deprecated_user,
+ group => $deprecated_group,
}
# T72385
@@ -90,22 +91,15 @@
file { $otherdir_wikidata_legacy:
ensure => 'directory',
mode => '0755',
- owner => $user,
- group => $group,
+ owner => $deprecated_user,
+ group => $deprecated_group,
}
file { $xlationdir:
ensure => 'directory',
mode => '0755',
- owner => $user,
- group => $group,
- }
-
- file { $centralauthdir:
- ensure => 'directory',
- mode => '0755',
- owner => $user,
- group => $group,
+ owner => $deprecated_user,
+ group => $deprecated_group,
}
file { $cirrussearchdir:
diff --git a/modules/dumps/manifests/web/html.pp
b/modules/dumps/manifests/web/html.pp
index 8f88964..db43acc 100644
--- a/modules/dumps/manifests/web/html.pp
+++ b/modules/dumps/manifests/web/html.pp
@@ -4,13 +4,17 @@
$otherdir = undef,
$webuser = undef,
$webgroup = undef,
+ $deprecated_user = undef,
+ $deprecated_group = undef,
) {
class {'dumps::web::dirs':
- datadir => $datadir,
- publicdir => $publicdir,
- otherdir => $otherdir,
- user => $webuser,
- group => $webgroup,
+ datadir => $datadir,
+ publicdir => $publicdir,
+ otherdir => $otherdir,
+ user => $webuser,
+ group => $webgroup,
+ deprecated_user => $deprecated_user,
+ deprecated_group => $deprecated_group,
}
file { "${publicdir}/dumps.css":
diff --git a/modules/dumps/manifests/web/xmldumps.pp
b/modules/dumps/manifests/web/xmldumps.pp
index 5e1d1e1..84d3a15 100644
--- a/modules/dumps/manifests/web/xmldumps.pp
+++ b/modules/dumps/manifests/web/xmldumps.pp
@@ -6,15 +6,19 @@
$otherdir = undef,
$htmldumps_server = undef,
$xmldumps_server = undef,
- $webuser = undef,
- $webgroup = undef,
+ $webuser = undef,
+ $webgroup = undef,
+ $deprecated_user = undef,
+ $deprecated_group = undef,
) {
class {'dumps::web::html':
- datadir => $datadir,
- publicdir => $publicdir,
- otherdir => $otherdir,
- webuser => $webuser,
- webgroup => $webgroup,
+ datadir => $datadir,
+ publicdir => $publicdir,
+ otherdir => $otherdir,
+ webuser => $webuser,
+ webgroup => $webgroup,
+ deprecated_user => $deprecated_user,
+ deprecated_group => $deprecated_group,
}
class { '::nginx':
diff --git a/modules/dumps/manifests/web/xmldumps_active.pp
b/modules/dumps/manifests/web/xmldumps_active.pp
index e8ca5c8..bdf54a1 100644
--- a/modules/dumps/manifests/web/xmldumps_active.pp
+++ b/modules/dumps/manifests/web/xmldumps_active.pp
@@ -8,6 +8,8 @@
$xmldumps_server = undef,
$webuser = undef,
$webgroup = undef,
+ $deprecated_user = undef,
+ $deprecated_group = undef,
) {
# active web server
class {'::dumps::web::xmldumps':
@@ -19,6 +21,8 @@
xmldumps_server => $xmldumps_server,
webuser => $webuser,
webgroup => $webgroup,
+ deprecated_user => $deprecated_user,
+ deprecated_group => $deprecated_group,
}
# only the active web server should be syncing nginx logs
diff --git a/modules/profile/manifests/dumps/web/xmldumps_active.pp
b/modules/profile/manifests/dumps/web/xmldumps_active.pp
index 3110dbc..27a206f 100644
--- a/modules/profile/manifests/dumps/web/xmldumps_active.pp
+++ b/modules/profile/manifests/dumps/web/xmldumps_active.pp
@@ -11,8 +11,10 @@
logs_dest =>
'stat1005.eqiad.wmnet::srv/log/webrequest/archive/dumps.wikimedia.org/',
htmldumps_server => 'francium.eqiad.wmnet',
xmldumps_server => 'dumps.wikimedia.org',
- webuser => 'datasets',
- webgroup => 'datasets',
+ webuser => 'dumpsgen',
+ webgroup => 'dumpsgen',
+ deprecated_user => 'datasets',
+ deprecated_group => 'datasets',
}
# copy dumps and other datasets to fallback host(s) and to labs
class {'::dumps::copying::peers':
diff --git a/modules/profile/manifests/dumps/web/xmldumps_fallback.pp
b/modules/profile/manifests/dumps/web/xmldumps_fallback.pp
index a253023..c7feafb 100644
--- a/modules/profile/manifests/dumps/web/xmldumps_fallback.pp
+++ b/modules/profile/manifests/dumps/web/xmldumps_fallback.pp
@@ -8,7 +8,9 @@
otherdir => '/data/xmldatadumps/public/other',
htmldumps_server => 'francium.eqiad.wmnet',
xmldumps_server => 'dumps.wikimedia.org',
- webuser => 'datasets',
- webgroup => 'datasets',
+ webuser => 'dumpsgen',
+ webgroup => 'dumpsgen',
+ deprecated_user => 'datasets',
+ deprecated_group => 'datasets',
}
}
diff --git a/modules/snapshot/manifests/addschanges.pp
b/modules/snapshot/manifests/addschanges.pp
index 1a55562..62c97b1 100644
--- a/modules/snapshot/manifests/addschanges.pp
+++ b/modules/snapshot/manifests/addschanges.pp
@@ -6,6 +6,7 @@
$apachedir = $snapshot::dumps::dirs::apachedir
$dblistsdir = $snapshot::dumps::dirs::dblistsdir
$templsdir = $snapshot::dumps::dirs::templsdir
+ $cronsdir = $snapshot::dumps::dirs::cronsdir
file { "${confsdir}/addschanges.conf":
ensure => 'present',
diff --git a/modules/snapshot/manifests/cron.pp
b/modules/snapshot/manifests/cron.pp
index f05b378..e356077 100644
--- a/modules/snapshot/manifests/cron.pp
+++ b/modules/snapshot/manifests/cron.pp
@@ -27,5 +27,5 @@
group => $group,
}
class { '::snapshot::cron::contentxlation': user => $depr_user }
- class { '::snapshot::addschanges': user => $depr_user }
+ class { '::snapshot::addschanges': user => $miscdumpsuser }
}
diff --git a/modules/snapshot/templates/addschanges.conf.erb
b/modules/snapshot/templates/addschanges.conf.erb
index 540d16b..14f2a87 100644
--- a/modules/snapshot/templates/addschanges.conf.erb
+++ b/modules/snapshot/templates/addschanges.conf.erb
@@ -14,10 +14,10 @@
adminsettings=private/PrivateSettings.php
[output]
-dumpdir=<%= scope.lookupvar('snapshot::dumps::dirs::otherdir') -%>/incr
+dumpdir=<%= scope.lookupvar('snapshot::dumps::dirs::cronsdir') -%>/incr
templatedir=<%= scope.lookupvar('snapshot::dumps::dirs::templsdir') %>
indextmpl=<%= scope.lookupvar('snapshot::dumps::dirs::templsdir')
-%>/incrs-index.html
-temp=<%= scope.lookupvar('snapshot::dumps::dirs::miscdumpsdir') -%>/temp
+temp=<%= scope.lookupvar('snapshot::dumps::dirs::xmldumpsdir') -%>/temp
webroot=http://download.wikimedia.org
fileperms=0644
# revisions must be at least this much older than time of current run
--
To view, visit https://gerrit.wikimedia.org/r/395612
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I8bbccbe13226bb3f12754651da6d0a4d37060d5f
Gerrit-PatchSet: 7
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <ar...@wikimedia.org>
Gerrit-Reviewer: ArielGlenn <ar...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
