Faidon Liambotis has uploaded a new change for review.
https://gerrit.wikimedia.org/r/97021
Change subject: Revert geowiki separate class & backup changes
......................................................................
Revert geowiki separate class & backup changes
The whole "let's prepend $misc::statistics::geowiki::params::
everywhere" is super ugly & hard to read through. Case in point: this
broke puppet on stat1 with:
err: Could not retrieve catalog from remote server: Error 400 on SERVER:
Must pass directory to Git::Clone[geowiki-data-public] at
/etc/puppet/modules/git/manifests/clone.pp:34 on node
stat1.wikimedia.org
We could of course fix it in place, but the fact that it's so unreadable
makes it necessary enough to use a different approach.
This reverts commits ceb34961941e984c0646a761097da05430f1c6fd &
commit 0ec1823e5f9eccf62fd20fee424fb83bdbcfd155.
Change-Id: I7789080a16870ecd50e2a123f89f1ab8bd64e6de
---
M manifests/misc/statistics.pp
M manifests/role/backup.pp
2 files changed, 75 insertions(+), 74 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/21/97021/1
diff --git a/manifests/misc/statistics.pp b/manifests/misc/statistics.pp
index 73f45ae..ceb1f5d 100644
--- a/manifests/misc/statistics.pp
+++ b/manifests/misc/statistics.pp
@@ -39,16 +39,8 @@
# Labs has security groups, and as such, doesn't need firewall rules
}
-# == Class misc::statistics::user::params
-# Parameters for the statistics user
-class misc::statistics::user::params {
- $username = "stats"
-}
-
class misc::statistics::user {
- require misc::statistics::user::params
-
- $username = $misc::statistics::user::params::username
+ $username = "stats"
$homedir = "/var/lib/$username"
generic::systemuser { $username:
@@ -261,8 +253,7 @@
# stats.wikimedia.org
class misc::statistics::sites::stats {
- require misc::statistics::geowiki::data::private,
- misc::statistics::geowiki::params
+ require misc::statistics::geowiki::data::private
$site_name = "stats.wikimedia.org"
$docroot = "/srv/$site_name/htdocs"
@@ -288,7 +279,7 @@
# link geowiki checkout from docroot
file { $geowiki_private_directory:
ensure => "link",
- target =>
"${misc::statistics::geowiki::params::private_data_path}/datafiles",
+ target =>
"${misc::statistics::geowiki::data::private::geowiki_private_data_path}/datafiles",
owner => "root",
group => "www-data",
mode => '0750',
@@ -844,36 +835,21 @@
}
-# == Class misc::statistics::geowiki::params
-# Parameters for geowiki classes
-class misc::statistics::geowiki::params {
- require misc::statistics::user::params
-
- $username = $misc::statistics::user::params::username
- $base_path = '/a/geowiki'
- $log_path = "${base_path}/logs"
- $scripts_path = "${base_path}/scripts"
- $private_data_bare_path = "${base_path}/data-private-bare"
- $private_data_bare_host = "stat1"
- $private_data_bare_host_fqdn =
"${geowiki_private_data_bare_host}.wikimedia.org"
- $private_data_path = "${base_path}/data-private"
- $public_data_path = "${base_path}/data-public"
- $mysql_conf_research_file = "${base_path}/.research.my.cnf"
- $mysql_conf_globaldev_file = "${base_path}/.globaldev.my.cnf"
-}
-
# == Class misc::statistics::geowiki
# Clones analytics/geowiki python scripts
class misc::statistics::geowiki {
- require misc::statistics::user,
- misc::statistics::geowiki::params
+ require misc::statistics::user
+
+ $geowiki_user = $misc::statistics::user::username
+ $geowiki_base_path = '/a/geowiki'
+ $geowiki_scripts_path = "${geowiki_base_path}/scripts"
git::clone { 'geowiki-scripts':
- directory => $misc::statistics::geowiki::params::scripts_path,
+ directory => $geowiki_scripts_path,
origin => "https://gerrit.wikimedia.org/r/p/analytics/geowiki.git";,
ensure => 'latest',
- owner => $misc::statistics::geowiki::params::username,
- group => $misc::statistics::geowiki::params::username,
+ owner => $geowiki_user,
+ group => $geowiki_user,
}
}
@@ -885,12 +861,16 @@
require misc::statistics::geowiki,
passwords::mysql::research
+ $geowiki_user = $misc::statistics::geowiki::geowiki_user
+ $geowiki_base_path = $misc::statistics::geowiki::geowiki_base_path
+
$research_mysql_user = $passwords::mysql::research::user
$research_mysql_pass = $passwords::mysql::research::pass
- file { $misc::statistics::geowiki::params::mysql_conf_research_file:
- owner => $misc::statistics::geowiki::params::username,
- group => $misc::statistics::geowiki::params::username,
+ $conf_file = "${geowiki_base_path}/.research.my.cnf"
+ file { $conf_file:
+ owner => $geowiki_user,
+ group => $geowiki_user,
mode => '0400',
content => "
[client]
@@ -911,24 +891,26 @@
class misc::statistics::geowiki::data::private_bare::sync {
require misc::statistics::geowiki
- file { $misc::statistics::geowiki::params::private_data_bare_path:
+ $geowiki_user = $misc::statistics::geowiki::geowiki_user
+ $geowiki_base_path = $misc::statistics::geowiki::geowiki_base_path
+ $geowiki_private_data_bare_path = "${geowiki_base_path}/data-private-bare"
+ $geowiki_private_data_bare_host = "stat1"
+ $geowiki_private_data_bare_host_fqdn =
"${geowiki_private_data_bare_host}.wikimedia.org"
+
+ file { "$geowiki_private_data_bare_path":
ensure => directory,
- owner => $misc::statistics::geowiki::params::username,
- group => $misc::statistics::geowiki::params::username,
+ owner => $geowiki_user,
+ group => $geowiki_user,
mode => '0640',
}
# The bare repository lives on stat1, so it's available there directly.
- # It only needs backup (as the repo is not living in gerrit)
# Other hosts need to rsync it over
- if $::hostname ==
$misc::statistics::geowiki::params::private_data_bare_host {
- include backup::host
- backup::set { 'a-geowiki-data-private-bare': }
- } else {
+ if $::hostname != $geowiki_private_data_bare_host {
cron { 'geowiki data-private bare sync':
- command => "/usr/bin/rsync -rt --delete
rsync://${misc::statistics::geowiki::params::private_data_bare_host_fqdn}${misc::statistics::geowiki::params::private_data_bare_path}/
${misc::statistics::geowiki::params::private_data_bare_path}/",
- require =>
File[$misc::statistics::geowiki::params::private_data_bare_path],
- user => $misc::statistics::geowiki::params::username,
+ command => "/usr/bin/rsync -rt --delete
rsync://${geowiki_private_data_bare_host_fqdn}${geowiki_private_data_bare_path}/
${geowiki_private_data_bare_path}/",
+ require => File["$geowiki_private_data_bare_path"],
+ user => $geowiki_user,
hour => '17',
minute => '0',
}
@@ -942,11 +924,16 @@
require misc::statistics::geowiki,
misc::statistics::geowiki::data::private_bare::sync
+ $geowiki_user = $misc::statistics::geowiki::geowiki_user
+ $geowiki_base_path = $misc::statistics::geowiki::geowiki_base_path
+ $geowiki_private_data_path = "${geowiki_base_path}/data-private"
+ $geowiki_private_data_bare_path =
$misc::statistics::geowiki::data::private_bare::sync::geowiki_private_data_bare_path
+
git::clone { 'geowiki-data-private':
- directory => $misc::statistics::geowiki::params::private_data_path,
- origin =>
"file://${misc::statistics::geowiki::params::private_data_bare_path}",
+ directory => $geowiki_private_data_path,
+ origin => "file://${geowiki_private_data_bare_path}",
ensure => 'latest',
- owner => $misc::statistics::geowiki::params::username,
+ owner => $geowiki_user,
group => 'www-data',
mode => 0750,
}
@@ -964,13 +951,20 @@
misc::statistics::packages::python,
geoip
+ $geowiki_user = $misc::statistics::geowiki::geowiki_user
+ $geowiki_base_path = $misc::statistics::geowiki::geowiki_base_path
+ $geowiki_scripts_path = $misc::statistics::geowiki::geowiki_scripts_path
+
+ $geowiki_mysql_research_conf_file =
$misc::statistics::geowiki::mysql::conf::research::conf_file
+
# install MySQL conf files for db acccess
$globaldev_mysql_user = $passwords::mysql::globaldev::user
$globaldev_mysql_pass = $passwords::mysql::globaldev::pass
- file { $misc::statistics::geowiki::params::mysql_conf_globaldev_file:
- owner => $misc::statistics::geowiki::params::username,
- group => $misc::statistics::geowiki::params::username,
+ $geowiki_mysql_globaldev_conf_file =
"${geowiki_base_path}/.globaldev.my.cnf"
+ file { $geowiki_mysql_globaldev_conf_file:
+ owner => $geowiki_user,
+ group => $geowiki_user,
mode => '0400',
content => "
[client]
@@ -979,22 +973,23 @@
",
}
- file { $misc::statistics::geowiki::params::log_path:
+ $geowiki_log_path = "${geowiki_base_path}/logs"
+ file { $geowiki_log_path:
ensure => 'directory',
- owner => $misc::statistics::geowiki::params::username,
- group => $misc::statistics::geowiki::params::username,
+ owner => $geowiki_user,
+ group => $geowiki_user,
}
# cron to run geowiki/process_data.py.
# This will query the production slaves and
# store results in the research staging database.
- # Logs will be kept at log path.
+ # Logs will be kept $geowiki_log_path.
cron { 'geowiki-process-data':
minute => 0,
hour => 12,
- user => $misc::statistics::geowiki::params::username,
- command => "/usr/bin/python
${misc::statistics::geowiki::params::scripts_path}/geowiki/process_data.py -o
${misc::statistics::geowiki::params::log_path} --wpfiles
${misc::statistics::geowiki::params::scripts_path}/geowiki/data/all_ids.tsv
--daily --start=`date --date='-2 day' +\\%Y-\\%m-\\%d` --end=`date --date='0
day' +\\%Y-\\%m-\\%d`
--source_sql_cnf=${misc::statistics::geowiki::params::mysql_conf_globaldev_file}
--dest_sql_cnf=${misc::statistics::geowiki::params::mysql_conf_research_file}",
- require => File[$misc::statistics::geowiki::params::log_path],
+ user => $geowiki_user,
+ command => "/usr/bin/python
${geowiki_scripts_path}/geowiki/process_data.py -o ${$geowiki_log_path}
--wpfiles ${geowiki_scripts_path}/geowiki/data/all_ids.tsv --daily
--start=`date --date='-2 day' +\\%Y-\\%m-\\%d` --end=`date --date='0 day'
+\\%Y-\\%m-\\%d` --source_sql_cnf=${geowiki_mysql_globaldev_conf_file}
--dest_sql_cnf=${geowiki_mysql_research_conf_file}",
+ require => File[$geowiki_log_path],
}
}
@@ -1006,12 +1001,19 @@
misc::statistics::geowiki::data::private,
misc::statistics::packages::python
+ $geowiki_user = $misc::statistics::geowiki::geowiki_user
+ $geowiki_base_path = $misc::statistics::geowiki::geowiki_base_path
+ $geowiki_scripts_path = $misc::statistics::geowiki::geowiki_scripts_path
+ $geowiki_public_data_path = "${geowiki_base_path}/data-public"
+ $geowiki_private_data_path =
$misc::statistics::geowiki::data::private::geowiki_private_data_path
+ $geowiki_mysql_research_conf_file =
$misc::statistics::geowiki::mysql::conf::research::conf_file
+
git::clone { 'geowiki-data-public':
- directory => $misc::statistics::user::params::public_data_path,
+ directory => $geowiki_public_data_path,
origin =>
"ssh://gerrit.wikimedia.org:29418/analytics/geowiki/data-public.git",
ensure => 'latest',
- owner => $misc::statistics::geowiki::params::username,
- group => $misc::statistics::geowiki::params::username,
+ owner => $geowiki_user,
+ group => $geowiki_user,
}
# cron job to do the actual fetching from the database, computation of
@@ -1019,13 +1021,13 @@
cron { 'geowiki-process-db-to-limn':
minute => 0,
hour => 15,
- user => $misc::statistics::geowiki::params::username,
- command =>
"${misc::statistics::geowiki::params::scripts_path}/scripts/make_and_push_limn_files.sh
--cron-mode
--basedir_public=${misc::statistics::user::params::public_data_path}
--basedir_private=${misc::statistics::geowiki::params::private_data_path}
--source_sql_cnf=${misc::statistics::geowiki::params::mysql_conf_research_file}",
+ user => $geowiki_user,
+ command =>
"${geowiki_scripts_path}/scripts/make_and_push_limn_files.sh --cron-mode
--basedir_public=${geowiki_public_data_path}
--basedir_private=${geowiki_private_data_path}
--source_sql_cnf=${geowiki_mysql_research_conf_file}",
require => [
Git::Clone['geowiki-scripts'],
Git::Clone['geowiki-data-public'],
Git::Clone['geowiki-data-private'],
- File[$misc::statistics::geowiki::params::mysql_conf_research_file],
+ File[$geowiki_mysql_research_conf_file],
],
}
}
@@ -1039,14 +1041,17 @@
class misc::statistics::geowiki::jobs::monitoring {
require misc::statistics::geowiki
+ $geowiki_user = $misc::statistics::geowiki::geowiki_user
+ $geowiki_scripts_path = $misc::statistics::geowiki::geowiki_scripts_path
+
# cron job to fetch geowiki data via http://gp.wmflabs.org/
# and checks that the files are up-to-date and within
# meaningful ranges.
cron { 'geowiki-monitoring':
minute => 30,
hour => 21,
- user => $misc::statistics::geowiki::params::username,
- command =>
"${misc::statistics::geowiki::params::scripts_path}/scripts/check_web_page.sh",
+ user => $geowiki_user,
+ command => "${geowiki_scripts_path}/scripts/check_web_page.sh",
# Disabled for now due to restructuring of geowiki
ensure => absent,
}
diff --git a/manifests/role/backup.pp b/manifests/role/backup.pp
index 01a6157..03d8105 100644
--- a/manifests/role/backup.pp
+++ b/manifests/role/backup.pp
@@ -11,7 +11,6 @@
include backup::host
include role::backup::config
include passwords::bacula
- require misc::statistics::geowiki::params
system::role { 'role::backup::director': description => 'Backup server' }
@@ -72,9 +71,6 @@
}
bacula::director::fileset { 'a-eventlogging':
includes => [ '/a/eventlogging' ]
- }
- bacula::director::fileset { 'a-geowiki-data-private-bare':
- includes => [
$misc::statistics::geowiki::params::private_data_bare_path ]
}
bacula::director::fileset { 'home':
includes => [ '/home' ]
--
To view, visit https://gerrit.wikimedia.org/r/97021
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I7789080a16870ecd50e2a123f89f1ab8bd64e6de
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Faidon Liambotis <fai...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
