Ottomata has submitted this change and it was merged.
Change subject: Move stats::wmde cron files to analytics/wmde/scripts repo
......................................................................
Move stats::wmde cron files to analytics/wmde/scripts repo
The changes in the scripts repo should automatically merge
once this puppet change has merged.
This change does not ensure that the cron or the files removed
are absent, but the deployer should check this.
Relates to the change at I7d05126bc62b7606b3f16d2d2e38b64f8ae5c03d
Bug: T140095
Change-Id: Ib9c541e67cfe7580e2432a2cb813ea1de36863e6
---
M modules/statistics/manifests/wmde.pp
M modules/statistics/templates/wmde/config.erb
D modules/statistics/templates/wmde/daily.erb
D modules/statistics/templates/wmde/minutely.erb
4 files changed, 32 insertions(+), 88 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
diff --git a/modules/statistics/manifests/wmde.pp
b/modules/statistics/manifests/wmde.pp
index 573ca1c..7f9d97b 100644
--- a/modules/statistics/manifests/wmde.pp
+++ b/modules/statistics/manifests/wmde.pp
@@ -15,9 +15,14 @@
Class['::statistics'] -> Class['::statistics::wmde']
$user = 'analytics-wmde'
+
$dir = "${::statistics::working_path}/analytics-wmde"
$data_dir = "${dir}/data"
$scripts_dir = "${dir}/src/scripts"
+
+ $statsd_host = hiera('statsd')
+ # TODO graphite hostname should be in hiera
+ $graphite_host = 'graphite.eqiad.wmnet'
# Path in which all crons will log to.
$log_dir = "${dir}/log"
@@ -105,57 +110,53 @@
require => File["${dir}/src"],
}
- file { "${dir}/daily.sh":
- ensure => 'file',
- owner => $user,
- group => $user,
- mode => '0754',
- content => template('statistics/wmde/daily.erb'),
- require => Git::Clone['wmde/scripts'],
- }
-
- file { "${dir}/minutely.sh":
- ensure => 'file',
- owner => $user,
- group => $user,
- mode => '0754',
- content => template('statistics/wmde/minutely.erb'),
- require => Git::Clone['wmde/scripts'],
- }
-
Cron {
user => $user,
}
cron { 'minutely':
- command => "${dir}/minutely.sh >> ${log_dir}/minutely.log 2>&1",
+ command => "${scripts_dir}/cron/minutely.sh >> ${log_dir}/minutely.log
2>&1",
hour => '*',
minute => '*',
- require => File["${dir}/minutely.sh"],
+ require => Git::Clone['wmde/scripts'],
}
# Note: some of the scripts run by this cron need access to secrets!
# Docs can be seen at
https://github.com/wikimedia/analytics-wmde-scripts/blob/master/README.md
- cron { 'daily':
- command => "${dir}/daily.sh >> ${log_dir}/daily.log 2>&1",
+ cron { 'daily.03':
+ command => "time ${scripts_dir}/cron/daily.03.sh >>
${log_dir}/daily.03.log 2>&1",
hour => '3',
minute => '0',
require => [
- File["${dir}/daily.sh"],
+ Git::Clone['wmde/scripts'],
File["${dir}/src/config"],
mysql::config::client['research-wmde'],
],
}
- # Logrotate is at 6:25, + time for rsync (hourly?), 12 gives us roughly 6
hours
- cron { 'graphite/api/logScanner':
- command => "${scripts_dir}/src/wikidata/apiLogScanner.sh >>
${log_dir}/wd-apiLogScanner.log 2>&1",
+ cron { 'daily.12':
+ command => "time ${scripts_dir}/cron/daily.12.sh >>
${log_dir}/daily.12.log 2>&1",
hour => '12',
minute => '0',
+ require => [
+ Git::Clone['wmde/scripts'],
+ File["${dir}/src/config"],
+ ],
+ }
+
+ cron { 'weekly':
+ command => "time ${scripts_dir}/cron/weekly.sh >>
${log_dir}/weekly.log 2>&1",
+ weekday => '7',
+ hour => '01',
+ minute => '0',
+ require => [
+ Git::Clone['wmde/scripts'],
+ File["${dir}/src/config"],
+ ],
}
cron { 'wmde/toolkit-analyzer-build':
- command => "java -Xmx2g -jar
${dir}/src/toolkit-analyzer-build/toolkit-analyzer.jar --processors Metric
--store ${dir}/data --latest >> ${log_dir}/toolkit-analyzer.log 2>&1",
+ command => "time java -Xmx2g -jar
${dir}/src/toolkit-analyzer-build/toolkit-analyzer.jar --processors Metric
--store ${dir}/data --latest >> ${log_dir}/toolkit-analyzer.log 2>&1",
hour => '12',
minute => '0',
}
diff --git a/modules/statistics/templates/wmde/config.erb
b/modules/statistics/templates/wmde/config.erb
index 60ef2c2..5379745 100644
--- a/modules/statistics/templates/wmde/config.erb
+++ b/modules/statistics/templates/wmde/config.erb
@@ -1,3 +1,6 @@
<% @wmde_secrets.keys.sort.each do |k| -%>
<%= k %> <%= @wmde_secrets[k] %>
-<% end -%>
\ No newline at end of file
+<% end -%>
+dump-dir <%= @data_dir %>
+statsd_host <%= @statsd_host %>
+graphite_host <%= @graphite_host %>
\ No newline at end of file
diff --git a/modules/statistics/templates/wmde/daily.erb
b/modules/statistics/templates/wmde/daily.erb
deleted file mode 100755
index d8956b5..0000000
--- a/modules/statistics/templates/wmde/daily.erb
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/bin/bash
-# This file is managed by Puppet.
-# puppet:///modules/statistics/wmde/daily
-#
-# @author Addshore
-#
-
-# Data model
-<%= @scripts_dir %>/src/wikidata/datamodel/properties_by_datatype.php
-<%= @scripts_dir %>/src/wikidata/datamodel/terms_by_language.php
-<%= @scripts_dir %>/src/wikidata/datamodel/sitelinks_per_site.php
-<%= @scripts_dir %>/src/wikidata/datamodel/sitelinks_per_item.php
-<%= @scripts_dir %>/src/wikidata/datamodel/statements_per_entity.php
-<%= @scripts_dir %>/src/wikidata/sparql/ranks.php
-<%= @scripts_dir %>/src/wikidata/sparql/instanceof.php
-<%= @scripts_dir %>/src/wikidata/wikidata-analysis/metrics.php <%= @data_dir
%>/data
-
-# Entity usage
-<%= @scripts_dir %>/src/wikidata/entityUsage.php
-
-# Social
-<%= @scripts_dir %>/src/wikidata/social/facebook.php
-<%= @scripts_dir %>/src/wikidata/social/googleplus.php
-<%= @scripts_dir %>/src/wikidata/social/identica.php
-<%= @scripts_dir %>/src/wikidata/social/irc.php
-<%= @scripts_dir %>/src/wikidata/social/mail.php
-<%= @scripts_dir %>/src/wikidata/social/newsletter.php
-<%= @scripts_dir %>/src/wikidata/social/techmail.php
-<%= @scripts_dir %>/src/wikidata/social/twitter.php
-
-# Misc
-<%= @scripts_dir %>/src/wikidata/phabricatorTasks.php
-<%= @scripts_dir %>/src/wikidata/showcaseItems.php
-<%= @scripts_dir %>/src/wikidata/specialEntityData.php
-<%= @scripts_dir %>/src/catwatch/userprops.php
-
-# Site Stats
-<%= @scripts_dir %>/src/wikidata/site_stats/good_articles.sh
-<%= @scripts_dir %>/src/wikidata/site_stats/total_edits.sh
-<%= @scripts_dir %>/src/wikidata/site_stats/total_pages.sh
-<%= @scripts_dir %>/src/wikidata/site_stats/active_users.php
-<%= @scripts_dir %>/src/wikidata/site_stats/users.sh
-<%= @scripts_dir %>/src/wikidata/site_stats/admins.sh
-<%= @scripts_dir %>/src/wikidata/site_stats/bureaucrats.sh
-<%= @scripts_dir %>/src/wikidata/site_stats/bots.sh
-<%= @scripts_dir %>/src/wikidata/site_stats/propertycreators.sh
-<%= @scripts_dir %>/src/wikidata/site_stats/rolling_rc.php
-<%= @scripts_dir %>/src/wikidata/site_stats/pages_by_namespace.php
-<%= @scripts_dir %>/src/wikidata/site_stats/user_languages.php
-<%= @scripts_dir %>/src/wikidata/site_stats/page_size.php
diff --git a/modules/statistics/templates/wmde/minutely.erb
b/modules/statistics/templates/wmde/minutely.erb
deleted file mode 100755
index b8ec5d3..0000000
--- a/modules/statistics/templates/wmde/minutely.erb
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-# This file is managed by Puppet.
-# puppet:///modules/statistics/wmde/minutely
-#
-# @author Addshore
-#
-
-<%= @scripts_dir %>/src/wikidata/dispatch.php
-<%= @scripts_dir %>/src/wikidata/recentChanges.php
-<%= @scripts_dir %>/src/wikidata/sparql/minutely.php
--
To view, visit https://gerrit.wikimedia.org/r/298487
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ib9c541e67cfe7580e2432a2cb813ea1de36863e6
Gerrit-PatchSet: 9
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Addshore <[email protected]>
Gerrit-Reviewer: Addshore <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
