ArielGlenn has submitted this change and it was merged.
Change subject: dumps: set up but don't enable script for dumps to run from cron
......................................................................
dumps: set up but don't enable script for dumps to run from cron
also add separate creation stages for small, big wikis.
we must check each group separately to see if the directories
exist, so we'll be creating each group of dirs separately if
needed.
Bug: T107750
Change-Id: I20bc0c2e7fa0a4781343fe852bdb36ded197343d
---
A modules/snapshot/manifests/dumps/cron.pp
A modules/snapshot/manifests/dumps/cron/huge.pp
A modules/snapshot/manifests/dumps/cron/rest.pp
M modules/snapshot/manifests/dumps/stagesconfig.pp
M modules/snapshot/templates/dumpstages.erb
A modules/snapshot/templates/fulldumps.sh.erb
6 files changed, 177 insertions(+), 0 deletions(-)
Approvals:
ArielGlenn: Looks good to me, approved
jenkins-bot: Verified
diff --git a/modules/snapshot/manifests/dumps/cron.pp
b/modules/snapshot/manifests/dumps/cron.pp
new file mode 100644
index 0000000..f8287f9
--- /dev/null
+++ b/modules/snapshot/manifests/dumps/cron.pp
@@ -0,0 +1,22 @@
+class snapshot::dumps::cron(
+ $enable = true,
+ $user = undef,
+) {
+ include snapshot::dirs
+
+ if ($enable) {
+ $ensure = 'present'
+ }
+ else {
+ $ensure = 'absent'
+ }
+
+ file { '/srv/dumps/fulldumps.sh':
+ ensure => 'present',
+ path => "${snapshot::dirs::dumpsdir}/fulldumps.sh",
+ mode => '0755',
+ owner => $user,
+ group => root,
+ content => template('snapshot/fulldumps.sh.erb'),
+ }
+}
diff --git a/modules/snapshot/manifests/dumps/cron/huge.pp
b/modules/snapshot/manifests/dumps/cron/huge.pp
new file mode 100644
index 0000000..6bad717
--- /dev/null
+++ b/modules/snapshot/manifests/dumps/cron/huge.pp
@@ -0,0 +1,20 @@
+class snapshot::dumps::cron::huge(
+ $enable = true,
+ $user = undef,
+) {
+ include snapshot::dirs
+ include snapshot::dumps::cron
+
+ # fixme there is an implicit dependency on
+ # $dumpsdir/confs/wikidump.conf.* plus some stage files, make explicit
+ cron { 'fulldumps_huge':
+ # not ready yet
+ ensure => 'absent',
+ environment => '[email protected]',
+ user => $user,
+ command => "${snapshot::dirs::dumpsdir}/fulldumps.sh 01 14
hugewikis >/dev/null",
+ minute => '05',
+ hour => '02',
+ monthday => '01-14',
+ }
+}
diff --git a/modules/snapshot/manifests/dumps/cron/rest.pp
b/modules/snapshot/manifests/dumps/cron/rest.pp
new file mode 100644
index 0000000..1395070
--- /dev/null
+++ b/modules/snapshot/manifests/dumps/cron/rest.pp
@@ -0,0 +1,20 @@
+class snapshot::dumps::cron::rest(
+ $enable = true,
+ $user = undef,
+) {
+ include snapshot::dirs
+ include snapshot::dumps::cron
+
+ # fixme there is an implicit dependency on
+ # $dumpsdir/confs/wikidump.conf.* plus some stage files, make explicit
+ cron { 'fulldumps_rest':
+ # not ready yet
+ ensure => 'absent',
+ environment => '[email protected]',
+ user => $user,
+ command => "${snapshot::dirs::dumpsdir}/fulldumps.sh 01 14 regular
> /dev/null",
+ minute => '05',
+ hour => '02',
+ monthday => '01-14',
+ }
+}
diff --git a/modules/snapshot/manifests/dumps/stagesconfig.pp
b/modules/snapshot/manifests/dumps/stagesconfig.pp
index 86d8a92..2f9482b 100644
--- a/modules/snapshot/manifests/dumps/stagesconfig.pp
+++ b/modules/snapshot/manifests/dumps/stagesconfig.pp
@@ -85,6 +85,14 @@
stagestype => 'create',
stages => $stages,
}
+ snapshot::dumps::stagesconf { 'stages_create_smallwikis':
+ stagestype => 'create_small',
+ stages => $stages,
+ }
+ snapshot::dumps::stagesconf { 'stages_create_bigwikis':
+ stagestype => 'create_big',
+ stages => $stages,
+ }
snapshot::dumps::stagesconf { 'stages_create_hugewikis':
stagestype => 'create_huge',
stages => $stages,
diff --git a/modules/snapshot/templates/dumpstages.erb
b/modules/snapshot/templates/dumpstages.erb
index b0cfb90..25e25f2 100644
--- a/modules/snapshot/templates/dumpstages.erb
+++ b/modules/snapshot/templates/dumpstages.erb
@@ -10,7 +10,16 @@
<% if @stagestype == 'normal' or @stagestype == 'partial' or @stagestype ==
'create' %>
# mark the start of the run for all small, big wikis
1 8 continue none <%= @stages['smallwikis']['firststage'] -%> --job createdirs
--sleep 5; <%= @stages['bigwikis']['firststage'] %> --job createdirs --sleep 5
+<% end -%>
+<% if @stagestype == 'create_small' %>
+# mark the start of the run for all small wikis
+1 8 continue none <%= @stages['smallwikis']['firststage'] -%> --job createdirs
--sleep 5
+<% end -%>
+
+<% if @stagestype == 'create_big' %>
+# mark the start of the run for all big wikis
+1 8 continue none <%= @stages['bigwikis']['firststage'] %> --job createdirs
--sleep 5
<% end -%>
<% if @stagestype == 'normal' or @stagestype == 'partial' or @stagestype ==
'normal_nocreate' or @stagestype == 'partial_nocreate' %>
diff --git a/modules/snapshot/templates/fulldumps.sh.erb
b/modules/snapshot/templates/fulldumps.sh.erb
new file mode 100644
index 0000000..d907c4b
--- /dev/null
+++ b/modules/snapshot/templates/fulldumps.sh.erb
@@ -0,0 +1,98 @@
+#!/bin/bash
+#############################################################
+# This file is maintained by puppet!
+# puppet:///modules/snapshot/fulldumps.sh.erb
+#############################################################
+
+# This script is intended to be run out of cron, set to start
+# daily, with a given date range supplied, but it will
+# actually start that dump run only once during that date range.
+# If there is another copy of this script already running
+# or if there has been a run that completed during the date range
+# then this script will simply exit.
+#
+# This permits the window for completion of dump runs to slip
+# some if there are errors or parts that need to be rerun,
+# without requiring manual intervention for the next cron run.
+
+# don't run if there's already an instance running
+pgrep $0 && exit 0
+# if dumpscheduler already running, don't start up a new one, let that run
complete
+pgrep dumpscheduler.py && exit 0
+
+
+if [ -z "$1" -o -z "$2" -o -z "$3" ]; then
+ echo "Usage: $0 startdate enddate huge|regular"
+ echo "where huge or regular is the type of wiki dump to be"
+ echo "run (huge wikis or small/big wikis)"
+ echo "example:"
+ echo "$0 01 10 regular"
+ echo "for a run over the small and big wikis, that should have started"
+ echo "on the first of the month and can be started up to the 10th without"
+ echo "encroaching on the next dump run"
+ exit 1
+fi
+
+startdate="$1"
+enddate="$2"
+wikitypes="$3"
+
+dumpsdir="<%= scope.lookupvar('snapshot::dirs::dumpsdir') %>"
+cd $dumpsdir
+
+today=`date +%d`
+if [[ "$today" < "$startdate" || "$today" > "$enddate" ]]; then
+ # skip, we're not in the run range for this dump
+ exit 0
+fi
+
+# create directories for the dump run for each group of wikis
+# (small, big, huge) as needed
+case $wikitype in
+ 'hugewikis')
+ configfile="${dumpsdir}/confs/wikidump.conf.hugewikis"
+ stagesfile="stages_create_hugewikis"
+ lastrun=`python dumpadmin.py -s lastrun -configfile $configfile`
+ if [[ -z "$lastrun" || "$lastrun" < "$startdate" ]]; then
+ python ./dumpscheduler.py --slots 8 --commands
${dumpsdir}/stages/${stagesfile} --cache ${dumpsdir}/cache/running_cache.txt
--directory $dumpsdir
+ fi
+ ;;
+ 'regular')
+ configfile="${dumpsdir}/confs/wikidump.conf.bigwikis"
+ stagesfile="stages_create_bigwikis"
+ lastrun=`python dumpadmin.py -s lastrun -configfile $configfile`
+ if [[ -z "$lastrun" || "$lastrun" < "$startdate" ]]; then
+ python ./dumpscheduler.py --slots 8 --commands
${dumpsdir}/stages/${stagesfile} --cache ${dumpsdir}/cache/running_cache.txt
--directory $dumpsdir
+ fi
+
+ configfile="${dumpsdir}/confs/wikidump.conf"
+ stagesfile="stages_create_smallwikis"
+ lastrun=`python dumpadmin.py -s lastrun -configfile $configfile`
+ if [[ -z "$lastrun" || "$lastrun" < "$startdate" ]]; then
+ python ./dumpscheduler.py --slots 8 --commands
${dumpsdir}/stages/${stagesfile} --cache ${dumpsdir}/cache/running_cache.txt
--directory $dumpsdir
+ fi
+
+ stagesfile="stages_create_bigwikis"
+ configfile="${dumpsdir}/confs/wikidump.conf.bigwikis"
+ ;;
+esac
+
+case $wikitype in
+ 'huge')
+ configfile="${dumpsdir}/confs/wikidump.conf.hugewikis"
+ alldone=`python dumpadmin.py -s alldone --configfile $configfile`
+ if [ -z "$alldone" ]; then
+ python ./dumpscheduler.py --slots 27 --commands
${dumpsdir}/stages/stages_normal_nocreate_hugewikis --cache
${dumpsdir}/cache/running_cache.txt --directory $dumpsdir
+ fi
+ ;;
+ 'regular')
+ configfile="${dumpsdir}/confs/wikidump.conf.bigwikis"
+ alldonebig=`python dumpadmin.py -s alldone --configfile $configfile`
+ configfile="${dumpsdir}/confs/wikidump.conf"
+ alldonesmall=`python dumpadmin.py -s alldone --configfile $configfile`
+ if [ -z "$alldonebig" -o -z "$alldonesmall" ]; then
+ python ./dumpscheduler.py --slots 8 --commands
${dumpsdir}/stages/stages_normal_nocreate --cache
${dumpsdir}/cache/running_cache.txt --directory $dumpsdir
+ fi
+ ;;
+esac
+exit 0
--
To view, visit https://gerrit.wikimedia.org/r/263807
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I20bc0c2e7fa0a4781343fe852bdb36ded197343d
Gerrit-PatchSet: 13
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <[email protected]>
Gerrit-Reviewer: ArielGlenn <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
