Addshore has uploaded a new change for review.
https://gerrit.wikimedia.org/r/298487
Change subject: Move stats::wmde cron files to analytics/wmde/scripts repo
......................................................................
Move stats::wmde cron files to analytics/wmde/scripts repo
Bug: T140095
Change-Id: Ib9c541e67cfe7580e2432a2cb813ea1de36863e6
---
M modules/statistics/manifests/wmde.pp
M modules/statistics/templates/wmde/config.erb
D modules/statistics/templates/wmde/daily.erb
D modules/statistics/templates/wmde/minutely.erb
4 files changed, 40 insertions(+), 82 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/87/298487/1
diff --git a/modules/statistics/manifests/wmde.pp
b/modules/statistics/manifests/wmde.pp
index 573ca1c..4f116d8 100644
--- a/modules/statistics/manifests/wmde.pp
+++ b/modules/statistics/manifests/wmde.pp
@@ -15,9 +15,13 @@
Class['::statistics'] -> Class['::statistics::wmde']
$user = 'analytics-wmde'
+
$dir = "${::statistics::working_path}/analytics-wmde"
$data_dir = "${dir}/data"
$scripts_dir = "${dir}/src/scripts"
+
+ $statsd_host = 'statsd.eqiad.wmnet'
+ $graphite_host = 'graphite.eqiad.wmnet'
# Path in which all crons will log to.
$log_dir = "${dir}/log"
@@ -106,56 +110,67 @@
}
file { "${dir}/daily.sh":
- ensure => 'file',
- owner => $user,
- group => $user,
- mode => '0754',
- content => template('statistics/wmde/daily.erb'),
- require => Git::Clone['wmde/scripts'],
+ ensure => 'absent',
}
file { "${dir}/minutely.sh":
- ensure => 'file',
- owner => $user,
- group => $user,
- mode => '0754',
- content => template('statistics/wmde/minutely.erb'),
- require => Git::Clone['wmde/scripts'],
+ ensure => 'absent',
}
Cron {
user => $user,
}
+ cron { 'daily':
+ ensure => 'absent',
+ }
+ cron { 'graphite/api/logScanner':
+ ensure => 'absent',
+ }
+
cron { 'minutely':
- command => "${dir}/minutely.sh >> ${log_dir}/minutely.log 2>&1",
+ command => "${scripts_dir}/cron/minutely.sh >> ${log_dir}/minutely.log
2>&1",
hour => '*',
minute => '*',
- require => File["${dir}/minutely.sh"],
+ require => Git::Clone['wmde/scripts'],
}
# Note: some of the scripts run by this cron need access to secrets!
# Docs can be seen at
https://github.com/wikimedia/analytics-wmde-scripts/blob/master/README.md
- cron { 'daily':
- command => "${dir}/daily.sh >> ${log_dir}/daily.log 2>&1",
+ cron { 'daily.03':
+ command => "time ${scripts_dir}/cron/daily.03.sh >>
${log_dir}/daily.03.log 2>&1",
hour => '3',
minute => '0',
require => [
- File["${dir}/daily.sh"],
+ Git::Clone['wmde/scripts'],
File["${dir}/src/config"],
mysql::config::client['research-wmde'],
],
}
- # Logrotate is at 6:25, + time for rsync (hourly?), 12 gives us roughly 6
hours
- cron { 'graphite/api/logScanner':
- command => "${scripts_dir}/src/wikidata/apiLogScanner.sh >>
${log_dir}/wd-apiLogScanner.log 2>&1",
+ cron { 'daily.12':
+ command => "time ${scripts_dir}/cron/daily.12.sh >>
${log_dir}/daily.12.log 2>&1",
hour => '12',
minute => '0',
+ require => [
+ Git::Clone['wmde/scripts'],
+ File["${dir}/src/config"],
+ ],
+ }
+
+ cron { 'weekly':
+ command => "time ${scripts_dir}/cron/weekly.sh >>
${log_dir}/weekly.log 2>&1",
+ weekday => '7',
+ hour => '01',
+ minute => '0',
+ require => [
+ Git::Clone['wmde/scripts'],
+ File["${dir}/src/config"],
+ ],
}
cron { 'wmde/toolkit-analyzer-build':
- command => "java -Xmx2g -jar
${dir}/src/toolkit-analyzer-build/toolkit-analyzer.jar --processors Metric
--store ${dir}/data --latest >> ${log_dir}/toolkit-analyzer.log 2>&1",
+ command => "time java -Xmx2g -jar
${dir}/src/toolkit-analyzer-build/toolkit-analyzer.jar --processors Metric
--store ${dir}/data --latest >> ${log_dir}/toolkit-analyzer.log 2>&1",
hour => '12',
minute => '0',
}
diff --git a/modules/statistics/templates/wmde/config.erb
b/modules/statistics/templates/wmde/config.erb
index 60ef2c2..5379745 100644
--- a/modules/statistics/templates/wmde/config.erb
+++ b/modules/statistics/templates/wmde/config.erb
@@ -1,3 +1,6 @@
<% @wmde_secrets.keys.sort.each do |k| -%>
<%= k %> <%= @wmde_secrets[k] %>
-<% end -%>
\ No newline at end of file
+<% end -%>
+dump-dir <%= @data_dir %>
+statsd_host <%= @statsd_host %>
+graphite_host <%= @graphite_host %>
\ No newline at end of file
diff --git a/modules/statistics/templates/wmde/daily.erb
b/modules/statistics/templates/wmde/daily.erb
deleted file mode 100755
index d8956b5..0000000
--- a/modules/statistics/templates/wmde/daily.erb
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/bin/bash
-# This file is managed by Puppet.
-# puppet:///modules/statistics/wmde/daily
-#
-# @author Addshore
-#
-
-# Data model
-<%= @scripts_dir %>/src/wikidata/datamodel/properties_by_datatype.php
-<%= @scripts_dir %>/src/wikidata/datamodel/terms_by_language.php
-<%= @scripts_dir %>/src/wikidata/datamodel/sitelinks_per_site.php
-<%= @scripts_dir %>/src/wikidata/datamodel/sitelinks_per_item.php
-<%= @scripts_dir %>/src/wikidata/datamodel/statements_per_entity.php
-<%= @scripts_dir %>/src/wikidata/sparql/ranks.php
-<%= @scripts_dir %>/src/wikidata/sparql/instanceof.php
-<%= @scripts_dir %>/src/wikidata/wikidata-analysis/metrics.php <%= @data_dir
%>/data
-
-# Entity usage
-<%= @scripts_dir %>/src/wikidata/entityUsage.php
-
-# Social
-<%= @scripts_dir %>/src/wikidata/social/facebook.php
-<%= @scripts_dir %>/src/wikidata/social/googleplus.php
-<%= @scripts_dir %>/src/wikidata/social/identica.php
-<%= @scripts_dir %>/src/wikidata/social/irc.php
-<%= @scripts_dir %>/src/wikidata/social/mail.php
-<%= @scripts_dir %>/src/wikidata/social/newsletter.php
-<%= @scripts_dir %>/src/wikidata/social/techmail.php
-<%= @scripts_dir %>/src/wikidata/social/twitter.php
-
-# Misc
-<%= @scripts_dir %>/src/wikidata/phabricatorTasks.php
-<%= @scripts_dir %>/src/wikidata/showcaseItems.php
-<%= @scripts_dir %>/src/wikidata/specialEntityData.php
-<%= @scripts_dir %>/src/catwatch/userprops.php
-
-# Site Stats
-<%= @scripts_dir %>/src/wikidata/site_stats/good_articles.sh
-<%= @scripts_dir %>/src/wikidata/site_stats/total_edits.sh
-<%= @scripts_dir %>/src/wikidata/site_stats/total_pages.sh
-<%= @scripts_dir %>/src/wikidata/site_stats/active_users.php
-<%= @scripts_dir %>/src/wikidata/site_stats/users.sh
-<%= @scripts_dir %>/src/wikidata/site_stats/admins.sh
-<%= @scripts_dir %>/src/wikidata/site_stats/bureaucrats.sh
-<%= @scripts_dir %>/src/wikidata/site_stats/bots.sh
-<%= @scripts_dir %>/src/wikidata/site_stats/propertycreators.sh
-<%= @scripts_dir %>/src/wikidata/site_stats/rolling_rc.php
-<%= @scripts_dir %>/src/wikidata/site_stats/pages_by_namespace.php
-<%= @scripts_dir %>/src/wikidata/site_stats/user_languages.php
-<%= @scripts_dir %>/src/wikidata/site_stats/page_size.php
diff --git a/modules/statistics/templates/wmde/minutely.erb
b/modules/statistics/templates/wmde/minutely.erb
deleted file mode 100755
index b8ec5d3..0000000
--- a/modules/statistics/templates/wmde/minutely.erb
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-# This file is managed by Puppet.
-# puppet:///modules/statistics/wmde/minutely
-#
-# @author Addshore
-#
-
-<%= @scripts_dir %>/src/wikidata/dispatch.php
-<%= @scripts_dir %>/src/wikidata/recentChanges.php
-<%= @scripts_dir %>/src/wikidata/sparql/minutely.php
--
To view, visit https://gerrit.wikimedia.org/r/298487
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib9c541e67cfe7580e2432a2cb813ea1de36863e6
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Addshore <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
