Ottomata has submitted this change and it was merged.
Change subject: Statistics::wmde puppetization
......................................................................
Statistics::wmde puppetization
Bug: T125989
Change-Id: Ie6c865a01e5ec00decf82ecaab221e7fa76a95b8
---
M manifests/role/statistics.pp
A modules/statistics/manifests/wmde.pp
A modules/statistics/templates/wmde/config.erb
A modules/statistics/templates/wmde/daily.erb
A modules/statistics/templates/wmde/logrotate.erb
A modules/statistics/templates/wmde/minutely.erb
6 files changed, 237 insertions(+), 0 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
diff --git a/manifests/role/statistics.pp b/manifests/role/statistics.pp
index 3d5d2c1..9110969 100644
--- a/manifests/role/statistics.pp
+++ b/manifests/role/statistics.pp
@@ -145,6 +145,9 @@
# rsync mediawiki logs from logging hosts
include statistics::rsync::mediawiki
+ # WMDE statistics scripts and cron jobs
+ include statistics::wmde
+
# eventlogging logs are not private, but they
# are here for convenience
include statistics::rsync::eventlogging
diff --git a/modules/statistics/manifests/wmde.pp
b/modules/statistics/manifests/wmde.pp
new file mode 100644
index 0000000..054a979
--- /dev/null
+++ b/modules/statistics/manifests/wmde.pp
@@ -0,0 +1,159 @@
+# Licence AGPL version 3 or later
+#
+# Module for running WMDE releated analytics scripts.
+#
+# @author Addshore
+#
+# These scripts get metrics from a variety of places including:
+# - Databases
+# - Log files
+# - Hive
+# - Dumps
+#
+# And send the data to statsd or graphite directly
+class statistics::wmde(
+ $user = 'analytics-wmde',
+ $dir = '/srv/analytics-wmde/'
+) {
+ Class['::statistics'] -> Class['::statistics::wmde']
+
+ # Path in which all crons will log to.
+ $log_dir = "${dir}/log"
+
+ $wmde_secrets = hiera('wmde_secrets')
+
+ require_package(
+ 'openjdk-7-jdk',
+ 'php5',
+ 'php5-cli',
+ 'git')
+
+ group { $user:
+ ensure => present,
+ name => $user,
+ }
+
+ user { $user:
+ ensure => present,
+ shell => '/bin/bash',
+ managehome => false,
+ home => $dir,
+ system => true,
+ require => Group[$user],
+ }
+
+ include passwords::mysql::research
+ # This file will render at
+ # /etc/mysql/conf.d/research-wmde-client.cnf.
+ mysql::config::client { 'research-wmde':
+ user => $::passwords::mysql::research::user,
+ pass => $::passwords::mysql::research::pass,
+ group => $user,
+ mode => '0440',
+ require => User[$user],
+ }
+
+ $directories = [
+ $dir,
+ "${dir}/src",
+ "${dir}/data",
+ $log_dir
+ ]
+
+ file { $directories:
+ ensure => 'directory',
+ owner => $user,
+ group => $user,
+ mode => '0644',
+ require => User[$user],
+ }
+
+ git::clone { 'wmde/scripts':
+ ensure => '73c88575345d63115230a6f4ca7c75852fb735f0',
+ directory => "${dir}/src/scripts",
+ origin => 'https://gerrit.wikimedia.org/r/analytics/wmde/scripts',
+ owner => $user,
+ group => $user,
+ require => File["${dir}/src"],
+ }
+
+ git::clone { 'wmde/toolkit-analyzer-build':
+ ensure => '5030ff6f98bf1ef463e726883ae03a67819815e8',
+ directory => "${dir}/src/toolkit-analyzer-build",
+ origin =>
'https://gerrit.wikimedia.org/r/analytics/wmde/toolkit-analyzer-build',
+ owner => $user,
+ group => $user,
+ require => File["${dir}/src"],
+ }
+
+ logrotate::conf { 'statistics-wmde':
+ ensure => present,
+ content => template('statistics/wmde/logrotate.erb'),
+ require => File[$log_dir],
+ }
+
+ file { "${dir}/src/config":
+ ensure => 'file',
+ owner => 'root',
+ group => $user,
+ mode => '0440',
+ content => template('statistics/wmde/config.erb'),
+ require => User["${dir}/src"],
+ }
+
+ file { "${dir}/daily.sh":
+ ensure => 'file',
+ owner => $user,
+ group => $user,
+ mode => '0644',
+ content => template('statistics/wmde/daily.erb'),
+ require => Git::Clone['wmde/scripts'],
+ }
+
+ file { "${dir}/minutely.sh":
+ ensure => 'file',
+ owner => $user,
+ group => $user,
+ mode => '0644',
+ content => template('statistics/wmde/minutely.erb'),
+ require => Git::Clone['wmde/scripts'],
+ }
+
+ Cron {
+ user => $user,
+ }
+
+ cron { 'minutely':
+ command => "${dir}/minutely.sh >> ${log_dir}/minutely.log 2>&1",
+ hour => '*',
+ minute => '*',
+ require => File["${dir}/minutely.sh"],
+ }
+
+ # Note: some of the scripts run by this cron need access to secrets!
+ # Docs can be seen at
https://github.com/wikimedia/analytics-wmde-scripts/blob/master/README.md
+ cron { 'daily':
+ command => "${dir}/daily.sh >> ${log_dir}/daily.log 2>&1",
+ hour => '3',
+ minute => '0',
+ require => [
+ File["${dir}/daily.sh"],
+ File["${dir}/src/config"],
+ mysql::config::client['research-wmde'],
+ ],
+ }
+
+ # Logrotate is at 6:25, + time for rsync (hourly?), 12 gives us roughly 6
hours
+ cron { 'graphite/api/logScanner':
+ command => "${dir}/src/scripts/graphite/api/logScanner.sh >>
${log_dir}/graphite_api_logScanner.log 2>&1",
+ hour => '12',
+ minute => '0',
+ }
+
+ cron { 'wmde/toolkit-analyzer-build':
+ command => "java -Xmx2g -jar
${dir}/src/toolkit-analyzer-build/toolkit-analyzer.jar --processors Metric
--store ${dir}/data --latest >> ${log_dir}/toolkit-analyzer.log 2>&1",
+ hour => '12',
+ minute => '0',
+ }
+
+}
diff --git a/modules/statistics/templates/wmde/config.erb
b/modules/statistics/templates/wmde/config.erb
new file mode 100644
index 0000000..60ef2c2
--- /dev/null
+++ b/modules/statistics/templates/wmde/config.erb
@@ -0,0 +1,3 @@
+<% @wmde_secrets.keys.sort.each do |k| -%>
+<%= k %> <%= @wmde_secrets[k] %>
+<% end -%>
\ No newline at end of file
diff --git a/modules/statistics/templates/wmde/daily.erb
b/modules/statistics/templates/wmde/daily.erb
new file mode 100644
index 0000000..eb806ca
--- /dev/null
+++ b/modules/statistics/templates/wmde/daily.erb
@@ -0,0 +1,50 @@
+#!/bin/bash
+# This file is managed by Puppet.
+# puppet:///modules/statistics/wmde/daily
+#
+# @author Addshore
+#
+
+# Data model
+<%= @dir %>/scripts/graphite/datamodel/properties_by_datatype.php
+<%= @dir %>/scripts/graphite/datamodel/terms_by_language.php
+<%= @dir %>/scripts/graphite/datamodel/sitelinks_per_site.php
+<%= @dir %>/scripts/graphite/datamodel/sitelinks_per_item.php
+<%= @dir %>/scripts/graphite/datamodel/statements_per_entity.php
+<%= @dir %>/scripts/graphite/sparql/ranks.php
+<%= @dir %>/scripts/graphite/sparql/instanceof.php
+<%= @dir %>/scripts/graphite/wikidata-analysis/metrics.php <%= @dir %>/data
+
+# Entity usage
+<%= @dir %>/scripts/graphite/entityUsage.php
+
+# Social
+<%= @dir %>/scripts/graphite/social/facebook.php
+<%= @dir %>/scripts/graphite/social/googleplus.php
+<%= @dir %>/scripts/graphite/social/identica.php
+<%= @dir %>/scripts/graphite/social/irc.php
+<%= @dir %>/scripts/graphite/social/mail.php
+<%= @dir %>/scripts/graphite/social/newsletter.php
+<%= @dir %>/scripts/graphite/social/techmail.php
+<%= @dir %>/scripts/graphite/social/twitter.php
+
+# Misc
+<%= @dir %>/scripts/graphite/development/phabricator.php
+<%= @dir %>/scripts/graphite/showcase.php
+<%= @dir %>/scripts/graphite/hive/entitydata.php
+<%= @dir %>/scripts/graphite/catwatch.php
+
+# Site Stats
+<%= @dir %>/scripts/graphite/site_stats/good_articles.sh
+<%= @dir %>/scripts/graphite/site_stats/total_edits.sh
+<%= @dir %>/scripts/graphite/site_stats/total_pages.sh
+<%= @dir %>/scripts/graphite/site_stats/active_users.php
+<%= @dir %>/scripts/graphite/site_stats/users.sh
+<%= @dir %>/scripts/graphite/site_stats/admins.sh
+<%= @dir %>/scripts/graphite/site_stats/bureaucrats.sh
+<%= @dir %>/scripts/graphite/site_stats/bots.sh
+<%= @dir %>/scripts/graphite/site_stats/propertycreators.sh
+<%= @dir %>/scripts/graphite/site_stats/rolling_rc.php
+<%= @dir %>/scripts/graphite/site_stats/pages_by_namespace.php
+<%= @dir %>/scripts/graphite/site_stats/user_languages.php
+<%= @dir %>/scripts/graphite/site_stats/page_size.php
diff --git a/modules/statistics/templates/wmde/logrotate.erb
b/modules/statistics/templates/wmde/logrotate.erb
new file mode 100644
index 0000000..ec919e0
--- /dev/null
+++ b/modules/statistics/templates/wmde/logrotate.erb
@@ -0,0 +1,12 @@
+# This file is managed by Puppet.
+# puppet:///modules/statistics/wmde/logrotate
+# Rotate statistics/wmde cron output logs
+#
+<%= @log_dir %>/*.log {
+ notifempty
+ maxage 10
+ rotate 2
+ dateext
+ compress
+ missingok
+}
diff --git a/modules/statistics/templates/wmde/minutely.erb
b/modules/statistics/templates/wmde/minutely.erb
new file mode 100644
index 0000000..e3badc4
--- /dev/null
+++ b/modules/statistics/templates/wmde/minutely.erb
@@ -0,0 +1,10 @@
+#!/bin/bash
+# This file is managed by Puppet.
+# puppet:///modules/statistics/wmde/minutely
+#
+# @author Addshore
+#
+
+<%= @dir %>/scripts/graphite/dispatch.php
+<%= @dir %>/scripts/graphite/rc.php
+<%= @dir %>/scripts/graphite/sparql/minutely.php
--
To view, visit https://gerrit.wikimedia.org/r/269467
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ie6c865a01e5ec00decf82ecaab221e7fa76a95b8
Gerrit-PatchSet: 20
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Addshore <[email protected]>
Gerrit-Reviewer: Addshore <[email protected]>
Gerrit-Reviewer: Aude <[email protected]>
Gerrit-Reviewer: Hoo man <[email protected]>
Gerrit-Reviewer: Jakob <[email protected]>
Gerrit-Reviewer: JanZerebecki <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: Reedy <[email protected]>
Gerrit-Reviewer: Tobias Gritschacher <[email protected]>
Gerrit-Reviewer: WMDE-Fisch <[email protected]>
Gerrit-Reviewer: WMDE-leszek <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
