ArielGlenn has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/375791 )
Change subject: convert "other" dump crons to use script to grab config settings
......................................................................
convert "other" dump crons to use script to grab config settings
grep + mawk is shorter but grodier.
Bug: T174929
Change-Id: Ifbfe38201742b0d1e95cc33f2e4d6350777ef340
---
M modules/snapshot/files/cron/dump-global-blocks.sh
A modules/snapshot/files/cron/dump_functions.sh
M modules/snapshot/files/cron/dumpcategoriesrdf.sh
M modules/snapshot/files/cron/dumpcirrussearch.sh
M modules/snapshot/files/cron/dumpcontentxlation.sh
M modules/snapshot/files/cron/wikidatadumps-shared.sh
M modules/snapshot/manifests/cron.pp
7 files changed, 61 insertions(+), 71 deletions(-)
Approvals:
ArielGlenn: Looks good to me, approved
jenkins-bot: Verified
diff --git a/modules/snapshot/files/cron/dump-global-blocks.sh
b/modules/snapshot/files/cron/dump-global-blocks.sh
index 570f27c..9cd3b81 100644
--- a/modules/snapshot/files/cron/dump-global-blocks.sh
+++ b/modules/snapshot/files/cron/dump-global-blocks.sh
@@ -4,23 +4,7 @@
# modules/snapshot/cron/dump-global-blocks.sh
#############################################################
-source /usr/local/etc/set_dump_dirs.sh
-
-checkval() {
- setting=$1
- value=$2
- if [ -z "$value" -o "$value" == "null" ]; then
- echo "failed to retrieve value of $setting from $configfile" >& 2
- exit 1
- fi
-}
-
-getsetting() {
- results=$1
- section=$2
- setting=$3
- echo "$results" | /usr/bin/jq -M -r ".$section.$setting"
-}
+source /usr/local/etc/dump_functions.sh
get_db_host() {
apachedir=$1
diff --git a/modules/snapshot/files/cron/dump_functions.sh
b/modules/snapshot/files/cron/dump_functions.sh
new file mode 100755
index 0000000..99997ba
--- /dev/null
+++ b/modules/snapshot/files/cron/dump_functions.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#############################################################
+# This file is maintained by puppet!
+# modules/snapshot/cron/dump_functions.sh
+#############################################################
+#
+# functions used by "other" dump cron jobs.
+
+source /usr/local/etc/set_dump_dirs.sh
+
+checkval() {
+ setting=$1
+ value=$2
+ if [ -z "$value" -o "$value" == "null" ]; then
+ echo "failed to retrieve value of $setting from $configfile" >& 2
+ exit 1
+ fi
+}
+
+getsetting() {
+ results=$1
+ section=$2
+ setting=$3
+ echo "$results" | /usr/bin/jq -M -r ".$section.$setting"
+}
diff --git a/modules/snapshot/files/cron/dumpcategoriesrdf.sh
b/modules/snapshot/files/cron/dumpcategoriesrdf.sh
index 7fc1e5b..d0b810d 100755
--- a/modules/snapshot/files/cron/dumpcategoriesrdf.sh
+++ b/modules/snapshot/files/cron/dumpcategoriesrdf.sh
@@ -7,23 +7,7 @@
# Generate an RDF dump of categories for all wikis in
# categories-rdf list and remove old ones.
-source /usr/local/etc/set_dump_dirs.sh
-
-checkval() {
- setting=$1
- value=$2
- if [ -z "$value" -o "$value" == "null" ]; then
- echo "failed to retrieve value of $setting from $configfile" >& 2
- exit 1
- fi
-}
-
-getsetting() {
- results=$1
- section=$2
- setting=$3
- echo "$results" | /usr/bin/jq -M -r ".$section.$setting"
-}
+source /usr/local/etc/dump_functions.sh
usage() {
echo "Usage: $0 --list wikis.dblist [--config <pathtofile>] [--dryrun]"
diff --git a/modules/snapshot/files/cron/dumpcirrussearch.sh
b/modules/snapshot/files/cron/dumpcirrussearch.sh
index 29f30cc..0431825 100644
--- a/modules/snapshot/files/cron/dumpcirrussearch.sh
+++ b/modules/snapshot/files/cron/dumpcirrussearch.sh
@@ -7,7 +7,7 @@
# Generate a json dump of cirrussearch indices for all enabled
# wikis and remove old ones.
-source /usr/local/etc/set_dump_dirs.sh
+source /usr/local/etc/dump_functions.sh
usage() {
echo "Usage: $0 [--config <pathtofile>] [--dryrun]"
@@ -40,22 +40,18 @@
exit 1
fi
-deployDir=$(egrep "^dir=" "$configFile" | mawk -Fdir= '{ print $2 }')
-gzip=$(egrep "^gzip=" "$configFile" | mawk -Fgzip= '{ print $2 }')
-allList=$(egrep "^dblist=" "$configFile" | mawk -Fdblist= '{ print $2 }')
-privateList=$(egrep "^privatelist=" "$configFile" | mawk -Fprivatelist= '{
print $2 }')
-publicDir=$(egrep "^public=" "$configFile" | mawk -Fpublic= '{ print $2 }')
+args="wiki:dir,dblist,privatelist;tools:gzip;output:public"
+results=`python "${repodir}/getconfigvals.py" --configfile "$configFile"
--args "$args"`
-if [ -z "$deployDir" -o -z "$gzip" -o -z "$allList" -o -z "$privateList" -o -z
"$publicDir" ]; then
- echo "failed to find value of one of the following from config file
$configFile:"
- echo "gzip: $gzip"
- echo "dir: $deployDir"
- echo "dblist: $allList"
- echo "privatelist: $privateList"
- echo "public: $publicDir"
- echo "exiting..."
- exit 1
-fi
+deployDir=`getsetting "$results" "wiki" "dir"` || exit 1
+allList=`getsetting "$results" "wiki" "dblist"` || exit 1
+privateList=`getsetting "$results" "wiki" "privatelist"` || exit 1
+gzip=`getsetting "$results" "tools" "gzip"` || exit 1
+publicDir=`getsetting "$results" "output" "public"` || exit 1
+
+for settingname in "deployDir" "allList" "privateList" "gzip" "publicDir"; do
+ checkval "$settingname" "${!settingname}"
+done
today=$(date +'%Y%m%d')
targetDirBase="$publicDir/other/cirrussearch"
diff --git a/modules/snapshot/files/cron/dumpcontentxlation.sh
b/modules/snapshot/files/cron/dumpcontentxlation.sh
index 8b9f28f..881a46e 100644
--- a/modules/snapshot/files/cron/dumpcontentxlation.sh
+++ b/modules/snapshot/files/cron/dumpcontentxlation.sh
@@ -4,23 +4,7 @@
# modules/snapshot/cron/dumpcontentxlation.sh
#############################################################
-source /usr/local/etc/set_dump_dirs.sh
-
-checkval() {
- setting=$1
- value=$2
- if [ -z "$value" -o "$value" == "null" ]; then
- echo "failed to retrieve value of $setting from $configfile"
- exit 1
- fi
-}
-
-getsetting() {
- results=$1
- section=$2
- setting=$3
- echo "$results" | jq -M -r ".$section.$setting"
-}
+source /usr/local/etc/dump_functions.sh
do_dump() {
format=$1
diff --git a/modules/snapshot/files/cron/wikidatadumps-shared.sh
b/modules/snapshot/files/cron/wikidatadumps-shared.sh
index 7aa38e5..9714f08 100644
--- a/modules/snapshot/files/cron/wikidatadumps-shared.sh
+++ b/modules/snapshot/files/cron/wikidatadumps-shared.sh
@@ -8,17 +8,25 @@
#
# Marius Hoch < [email protected] >
-source /usr/local/etc/set_dump_dirs.sh
+source /usr/local/etc/dump_functions.sh
configfile="${confsdir}/wikidump.conf"
today=`date +'%Y%m%d'`
daysToKeep=70
-apacheDir=`awk -Fdir= '/^dir=/ { print $2 }' "$configfile"`
-publicDir=`awk -Fpublic= '/^public=/ { print $2 }' "$configfile"`
+args="wiki:dir;output:public,temp"
+results=`python "${repodir}/getconfigvals.py" --configfile "$configfile"
--args "$args"`
+
+apacheDir=`getsetting "$results" "wiki" "dir"` || exit 1
+publicDir=`getsetting "$results" "output" "public"` || exit 1
+tempDir=`getsetting "$results" "output" "temp"` || exit 1
+
+for settingname in "apacheDir" "publicDir" "tempDir"; do
+ checkval "$settingname" "${!settingname}"
+done
+
targetDirBase=$publicDir/other/wikibase/wikidatawiki
targetDir=$targetDirBase/$today
-tempDir=`awk -Ftemp= '/^temp=/ { print $2 }' "$configfile"`
multiversionscript="${apacheDir}/multiversion/MWScript.php"
diff --git a/modules/snapshot/manifests/cron.pp
b/modules/snapshot/manifests/cron.pp
index c5a25a6..1d19fa5 100644
--- a/modules/snapshot/manifests/cron.pp
+++ b/modules/snapshot/manifests/cron.pp
@@ -1,6 +1,15 @@
class snapshot::cron(
$user = undef,
) {
+ file { '/usr/local/etc/dump_functions.sh':
+ ensure => 'present',
+ path => '/usr/local/etc/dump_functions.sh',
+ mode => '0755',
+ owner => 'root',
+ group => 'root',
+ source => 'puppet:///modules/snapshot/cron/dump_functions.sh',
+ }
+
class { '::snapshot::cron::mediaperprojectlists': user => $user }
class { '::snapshot::cron::pagetitles': user => $user }
class { '::snapshot::cron::cirrussearch': user => $user }
--
To view, visit https://gerrit.wikimedia.org/r/375791
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ifbfe38201742b0d1e95cc33f2e4d6350777ef340
Gerrit-PatchSet: 7
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <[email protected]>
Gerrit-Reviewer: ArielGlenn <[email protected]>
Gerrit-Reviewer: Hoo man <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
