ArielGlenn has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/381838 )
Change subject: move remaining hardcoded paths for base dir of misc dumps out
to profile
......................................................................
move remaining hardcoded paths for base dir of misc dumps out to profile
Bug: T175528
Change-Id: I51d7a447d20d37ad3355355449d45fc26a016d90
---
M modules/dumps/files/fetches/kiwix-rsync-cron.sh
M modules/dumps/manifests/copying/labs.pp
M modules/dumps/manifests/web/fetches/kiwix.pp
M modules/dumps/templates/copying/labs-rsync-cron.sh.erb
M modules/profile/manifests/dumps/fetcher.pp
M modules/profile/manifests/dumps/web/xmldumps_active.pp
6 files changed, 33 insertions(+), 15 deletions(-)
Approvals:
ArielGlenn: Looks good to me, approved
jenkins-bot: Verified
diff --git a/modules/dumps/files/fetches/kiwix-rsync-cron.sh
b/modules/dumps/files/fetches/kiwix-rsync-cron.sh
index a8f7498..b54ba91 100644
--- a/modules/dumps/files/fetches/kiwix-rsync-cron.sh
+++ b/modules/dumps/files/fetches/kiwix-rsync-cron.sh
@@ -5,8 +5,6 @@
######################################
sourcehost="download.kiwix.org"
-destroot="/data/xmldatadumps/public/other"
-
bwlimit="--bwlimit=40000"
do_rsync (){
@@ -22,4 +20,11 @@
fi
}
+if [ -z "$1" ]; then
+ echo "Usage: $0 dest_base_dir"
+ exit 1
+fi
+
+destroot="$1"
+
do_rsync "download.kiwix.org/zim/wikipedia/" "kiwix/zim/wikipedia/"
diff --git a/modules/dumps/manifests/copying/labs.pp
b/modules/dumps/manifests/copying/labs.pp
index 44faca9..5dce210 100644
--- a/modules/dumps/manifests/copying/labs.pp
+++ b/modules/dumps/manifests/copying/labs.pp
@@ -1,6 +1,7 @@
class dumps::copying::labs(
$labhost = undef,
$publicdir = undef,
+ $otherdir = undef,
) {
file { '/usr/local/bin/wmfdumpsmirror.py':
ensure => 'present',
@@ -19,7 +20,7 @@
user => 'root',
minute => '50',
hour => '3',
- command => "/usr/local/sbin/labs-rsync-cron.sh ${labhost}
${publicdir}",
+ command => "/usr/local/sbin/labs-rsync-cron.sh ${labhost}
${publicdir} ${otherdir}",
environment => '[email protected]',
require => File['/usr/local/bin/wmfdumpsmirror.py',
'/usr/local/sbin/labs-rsync-cron.sh'],
diff --git a/modules/dumps/manifests/web/fetches/kiwix.pp
b/modules/dumps/manifests/web/fetches/kiwix.pp
index a58d705..2d5c71f 100644
--- a/modules/dumps/manifests/web/fetches/kiwix.pp
+++ b/modules/dumps/manifests/web/fetches/kiwix.pp
@@ -1,13 +1,15 @@
class dumps::web::fetches::kiwix(
$user = undef,
$group = undef,
+ $publicdir = undef,
+ $otherdir = undef,
) {
include dumps::deprecated::user
require_package('rsync')
- file { '/data/xmldatadumps/public/kiwix':
+ file { "${publicdir}/kiwix":
ensure => 'link',
- target => '/data/xmldatadumps/public/other/kiwix',
+ target => "${otherdir}/kiwix",
owner => $user,
group => $group,
mode => '0644',
@@ -23,7 +25,7 @@
cron { 'kiwix-mirror-update':
ensure => 'present',
environment => '[email protected]',
- command => '/bin/bash /usr/local/bin/kiwix-rsync-cron.sh',
+ command => "/bin/bash /usr/local/bin/kiwix-rsync-cron.sh
${otherdir}",
user => $group,
minute => '15',
hour => '*/2',
diff --git a/modules/dumps/templates/copying/labs-rsync-cron.sh.erb
b/modules/dumps/templates/copying/labs-rsync-cron.sh.erb
index 20bacb1..b207472 100755
--- a/modules/dumps/templates/copying/labs-rsync-cron.sh.erb
+++ b/modules/dumps/templates/copying/labs-rsync-cron.sh.erb
@@ -10,20 +10,22 @@
do_rsync (){
srcdir=$1
destdir=$2
+ base_srcdir=$3
running=`pgrep -u root -f -x "/usr/bin/rsync -rlptqgo $bwlimit
${otherdir}/${srcdir} ${desthost}::dumps/${destdir}"`
if [ -z "$running" ]; then
- /usr/bin/rsync -rlptqgo "$bwlimit" "${otherdir}/${srcdir}"
"${desthost}::dumps/${destdir}"
+ /usr/bin/rsync -rlptqgo "$bwlimit" "${base_srcdir}/${srcdir}"
"${desthost}::dumps/${destdir}"
fi
}
-if [ -z "$1" -o -z "$2" ]; then
- echo "Usage: $0 hostname sourcedir"
+if [ -z "$1" -o -z "$2" -o -z "$3" ]; then
+ echo "Usage: $0 hostname sourcedir otherdumpsdir"
exit 1
fi
desthost="$1"
sourcedir="$2"
+otherdumpsdir="$3"
running=`pgrep -u root -f "python /usr/local/bin/wmfdumpsmirror.py
--dest_hostname ${desthost}"`
if [ -z "$running" ]; then
@@ -33,6 +35,6 @@
# fixme need to ensure ${desthost}::dumps/public/wikidatawiki/entities/ exists
#copy from our dumps "other" directory to the labs host copy of dumps
-do_rsync "incr" ""
-do_rsync "wikibase/wikidatawiki/" "public/wikidatawiki/entities/"
-do_rsync "pageviews" ""
+do_rsync "incr" "" $otherdumpsdir
+do_rsync "wikibase/wikidatawiki/" "public/wikidatawiki/entities/"
$otherdumpsdir
+do_rsync "pageviews" "" $otherdumpsdir
diff --git a/modules/profile/manifests/dumps/fetcher.pp
b/modules/profile/manifests/dumps/fetcher.pp
index 8d3b179..9933bca 100644
--- a/modules/profile/manifests/dumps/fetcher.pp
+++ b/modules/profile/manifests/dumps/fetcher.pp
@@ -1,11 +1,16 @@
class profile::dumps::fetcher {
+ $publicdir = '/data/xmldatadumps/public'
+ $otherdir = '/data/xmldatadumps/public/other'
+
class {'dumps::web::fetches::kiwix':
- user => 'datasets',
- group => 'datasets',
+ user => 'datasets',
+ group => 'datasets',
+ publicdir => $publicdir,
+ otherdir => $otherdir,
}
class {'dumps::web::fetches::stats':
src => 'stat1005.eqiad.wmnet::hdfs-archive',
- otherdir => '/data/xmldatadumps/public/other',
+ otherdir => $otherdir,
user => 'datasets',
}
class {'dumps::web::fetches::wikitech_dumps':
diff --git a/modules/profile/manifests/dumps/web/xmldumps_active.pp
b/modules/profile/manifests/dumps/web/xmldumps_active.pp
index bffa0a8..78e0833 100644
--- a/modules/profile/manifests/dumps/web/xmldumps_active.pp
+++ b/modules/profile/manifests/dumps/web/xmldumps_active.pp
@@ -1,6 +1,8 @@
class profile::dumps::web::xmldumps_active {
class { '::dumpsuser': }
$publicdir = '/data/xmldatadumps/public'
+ $otherdir = '/data/xmldatadumps/public/other'
+
class {'::dumps::web::xmldumps_active':
do_acme => hiera('do_acme'),
datadir => '/data/xmldatadumps',
@@ -22,5 +24,6 @@
class {'::dumps::copying::labs':
labhost => 'labstore1003.eqiad.wmnet',
publicdir => $publicdir,
+ otherdir => $otherdir,
}
}
--
To view, visit https://gerrit.wikimedia.org/r/381838
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I51d7a447d20d37ad3355355449d45fc26a016d90
Gerrit-PatchSet: 7
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <[email protected]>
Gerrit-Reviewer: ArielGlenn <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
