Ottomata has submitted this change and it was merged.
Change subject: Adding misc::statistics::geowiki
......................................................................
Adding misc::statistics::geowiki
Change-Id: I9b4d9a7ab0458143094163d7793c3b8f8e32c202
---
M manifests/misc/statistics.pp
M manifests/role/statistics.pp
2 files changed, 83 insertions(+), 1 deletion(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
jenkins-bot: Verified
diff --git a/manifests/misc/statistics.pp b/manifests/misc/statistics.pp
index fe7a426..6802219 100644
--- a/manifests/misc/statistics.pp
+++ b/manifests/misc/statistics.pp
@@ -829,3 +829,83 @@
minute => 0,
}
}
+
+
+# == Class misc::statistics::geowiki
+# Clones analytics/editor-geocoding python scripts
+# and installs a cron job to get recent editor data
+# from the research slave databases and generate
+# editor geocoding statistics, saved back into a db.
+#
+class misc::statistics::geowiki {
+ require misc::statisics::user,
+ passwords::mysql::research,
+ passwords::mysql::globaldev,
+ misc::statistics::packages::python,
+ misc::geoip
+
+ $geowiki_user = $misc::statisics::user::username
+ $geowiki_path = '/a/geowiki'
+
+ git::clone { 'geowiki':
+ directory => $geowiki_path,
+ origin =>
"https://gerrit.wikimedia.org/r/p/analytics/editor-geocoding.git";,
+ ensure => 'latest',
+ owner => $geowiki_user,
+ }
+
+ # install MySQL conf files for db acccess
+ $research_mysql_user = $passwords::mysql::research::user
+ $research_mysql_pass = $passwords::mysql::research::pass
+
+ $globaldev_mysql_user = $passwords::mysql::globaldev::user
+ $globaldev_mysql_pass = $passwords::mysql::globaldev::pass
+
+ file { "${geowiki_path}/.research.my.cnf":
+ owner => $geowiki_user,
+ group => $geowiki_user,
+ mode => '0400',
+ content => "
+[client]
+user=${research_mysql_user}
+password=${research_mysql_pass}
+host=s1-analytics-slave.eqiad.wmnet'
+",
+ require => Git::Clone['geowiki'],
+ }
+
+ file { "${geowiki_path}/.globaldev.my.cnf":
+ owner => $geowiki_user,
+ group => $geowiki_user,
+ mode => '0400',
+ content => "
+[client]
+user=${globaldev_mysql_user}
+password=${globaldev_mysql_pass}
+",
+ require => Git::Clone['geowiki'],
+ }
+
+ $geowiki_backups_path = "${geowiki_path}/geowiki-backup-data"
+ file { $geowiki_backups_path:
+ ensure => 'directory',
+ owner => $geowiki_user,
+ group => $geowiki_user,
+ require => Git::Clone['geowiki'],
+ }
+
+ # cron to run geowiki/process_data.py.
+ # This will query the production slaves and
+ # store results in the research staging database.
+ # Backup files will be kept $geowiki_backups_path.
+ cron { 'geowiki-process-data':
+ minute => 0,
+ command => "/usr/bin/python
${geowiki_path}/geowiki/process_data.py\
+-o ${$geowiki_backups_path}\
+--wpfiles ${geowiki_path}/geowiki/data/all_ids.tsv --daily\
+--start=`date --date='-1 day' +\\%Y-\\%m-\\%d`\
+--end=`date --date='1 day' +\\%Y-\\%m-\\%d`\
+--source_sql_cnf=${geowiki_path}/.globaldev.my.cnf\
+--dest_sql_cnf=${geowiki_path}/.research.my.cnf",
+ }
+}
diff --git a/manifests/role/statistics.pp b/manifests/role/statistics.pp
index 95a1e99..ed8d0f0 100644
--- a/manifests/role/statistics.pp
+++ b/manifests/role/statistics.pp
@@ -25,7 +25,9 @@
# generate gerrit stats from stat1.
misc::statistics::gerrit_stats,
# rsync logs from logging hosts over to stat1
- misc::statistics::rsync::jobs
+ misc::statistics::rsync::jobs,
+ # editor-geocoding geowiki to generate geographic editor stats
+ misc::statistics::geowiki
}
class role::statistics::www inherits role::statistics {
--
To view, visit https://gerrit.wikimedia.org/r/75881
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I9b4d9a7ab0458143094163d7793c3b8f8e32c202
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <o...@wikimedia.org>
Gerrit-Reviewer: Ottomata <o...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
