DCausse has uploaded a new change for review.
https://gerrit.wikimedia.org/r/297276
Change subject: Setup CirrusSearch continuous saneitization process to run via
cron
......................................................................
Setup CirrusSearch continuous saneitization process to run via cron
Bug: T139200
Change-Id: I3b934b52b7b67726ba58c3d6c37c605b869202c2
---
A modules/mediawiki/files/maintenance/logrotate.d_cirrus-sanitize
M modules/mediawiki/manifests/maintenance/cirrussearch.pp
2 files changed, 44 insertions(+), 8 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/76/297276/1
diff --git a/modules/mediawiki/files/maintenance/logrotate.d_cirrus-sanitize
b/modules/mediawiki/files/maintenance/logrotate.d_cirrus-sanitize
new file mode 100644
index 0000000..1d4972b
--- /dev/null
+++ b/modules/mediawiki/files/maintenance/logrotate.d_cirrus-sanitize
@@ -0,0 +1,14 @@
+#####################################################################
+### THIS FILE IS MANAGED BY PUPPET
+#####################################################################
+
+/var/log/mediawiki/cirrus-sanitize/*.log {
+ daily
+ notifempty
+ maxage 30
+ rotate 5
+ dateext
+ compress
+ missingok
+ nocreate
+}
diff --git a/modules/mediawiki/manifests/maintenance/cirrussearch.pp
b/modules/mediawiki/manifests/maintenance/cirrussearch.pp
index cb898d8..910130e 100644
--- a/modules/mediawiki/manifests/maintenance/cirrussearch.pp
+++ b/modules/mediawiki/manifests/maintenance/cirrussearch.pp
@@ -1,23 +1,33 @@
class mediawiki::maintenance::cirrussearch( $ensure = present ) {
require mediawiki::users
+ Cron {
+ ensure => $ensure,
+ user => $::mediawiki::users::web,
+ }
+
# Rebuilds the completion suggester indices daily. This is scheduled
# to run during the low period of cirrus usage, which is generally 3am
# to 7am UTC. The `|| true` statement ensures one failing wiki doesn't
# fail the entire job. This job, as of mar 2015, takes around 5 hours
# to run.
- Cron {
- ensure => $ensure,
- user => $::mediawiki::users::web,
+ cron { 'cirrus_build_completion_indices_eqiad':
+ command => '/usr/local/bin/expanddblist all | xargs -I{} -P 4 sh -c
\'/usr/local/bin/mwscript
extensions/CirrusSearch/maintenance/updateSuggesterIndex.php --wiki={}
--masterTimeout=10m --replicationTimeout=5400 --cluster=eqiad --optimize >
/var/log/mediawiki/cirrus-suggest/{}.eqiad.log 2>&1 || true\'',
minute => 30,
hour => 2,
}
- cron {
- 'cirrus_build_completion_indices_eqiad':
- command => '/usr/local/bin/expanddblist all | xargs -I{} -P 4 sh
-c \'/usr/local/bin/mwscript
extensions/CirrusSearch/maintenance/updateSuggesterIndex.php --wiki={}
--masterTimeout=10m --replicationTimeout=5400 --cluster=eqiad --optimize >
/var/log/mediawiki/cirrus-suggest/{}.eqiad.log 2>&1 || true\'';
- 'cirrus_build_completion_indices_codfw':
- command => '/usr/local/bin/expanddblist all | xargs -I{} -P 4 sh
-c \'/usr/local/bin/mwscript
extensions/CirrusSearch/maintenance/updateSuggesterIndex.php --wiki={}
--masterTimeout=10m --replicationTimeout=5400 --cluster=codfw --optimize >
/var/log/mediawiki/cirrus-suggest/{}.codfw.log 2>&1 || true\''
+ cron { 'cirrus_build_completion_indices_codfw':
+ command => '/usr/local/bin/expanddblist all | xargs -I{} -P 4 sh -c
\'/usr/local/bin/mwscript
extensions/CirrusSearch/maintenance/updateSuggesterIndex.php --wiki={}
--masterTimeout=10m --replicationTimeout=5400 --cluster=codfw --optimize >
/var/log/mediawiki/cirrus-suggest/{}.codfw.log 2>&1 || true\'',
+ minute => 30,
+ hour => 2,
+ }
+
+ # Push sanitze jobs to the jobqueue every 2 hours
+ cron { 'cirrus_sanitize_jobs':
+ command => '/usr/local/bin/expanddblist all | xargs -I{} -P 4 sh -c
\'/usr/local/bin/mwscript extensions/CirrusSearch/maintenance/saneitizeJobs.php
--wiki={} --push --refresh-freq=7200 >>
/var/log/mediawiki/cirrus-sanitize/push-jobs.{}.log 2>&1 || true\'',
+ minute => 10,
+ hour => '*/2',
}
file { '/var/log/mediawiki/cirrus-suggest':
@@ -31,4 +41,16 @@
ensure => $ensure,
source =>
'puppet:///modules/mediawiki/maintenance/logrotate.d_cirrus-suggest',
}
+
+ file { '/var/log/mediawiki/cirrus-sanitize':
+ ensure => ensure_directory($ensure),
+ owner => $::mediawiki::users::web,
+ group => $::mediawiki::users::web,
+ mode => '0775'
+ }
+
+ logrotate::conf { 'cirrus-sanitize':
+ ensure => $ensure,
+ source =>
'puppet:///modules/mediawiki/maintenance/logrotate.d_cirrus-sanitize',
+ }
}
--
To view, visit https://gerrit.wikimedia.org/r/297276
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I3b934b52b7b67726ba58c3d6c37c605b869202c2
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: DCausse <dcau...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
