Ottomata has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/364808 )
Change subject: Apply role statistics::private_new to stat1005
......................................................................
Apply role statistics::private_new to stat1005
Bug: T152712
Change-Id: I4e4a84f23457e0ecdba484b47acdca489dfbcda9
---
M manifests/site.pp
M modules/profile/manifests/statistics/private.pp
A modules/role/manifests/statistics/private_new.pp
3 files changed, 54 insertions(+), 37 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
diff --git a/manifests/site.pp b/manifests/site.pp
index 9ac72e9..857b867 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -2108,8 +2108,7 @@
# Deploy wikimedia/discovery/analytics repository
# to this node.
- elasticsearch::analytics
- )
+ elasticsearch::analytics)
}
# stat1003 is a general purpose number cruncher for
@@ -2131,7 +2130,8 @@
# WIP: stat1002 replacement (T152712)
node 'stat1005.eqiad.wmnet' {
- role(# This is a Hadoop client, and should
+ role(statistics::private_new,
+ # This is a Hadoop client, and should
# have any special analytics system users on it
# for interacting with HDFS.
analytics_cluster::users,
@@ -2142,22 +2142,24 @@
analytics_cluster::client,
# Include analytics/refinery deployment target.
- analytics_cluster::refinery)
+ analytics_cluster::refinery,
- include ::standard
+ # TODO: include this role piecemeal: T152712
+ # Include analytics/refinery checks that send email about
+ # webrequest partitions faultyness.
+ # analytics_cluster::refinery::job::data_check,
- # Include the MySQL research password at
- # /etc/mysql/conf.d/analytics-research-client.cnf
- # and only readable by users in the
- # analytics-privatedata-users group.
- statistics::mysql_credentials { 'analytics-research':
- group => 'analytics-privatedata-users',
- }
+ # Include analytics/refinery/source guard checks
+ # Disabled due to T166937
+ # analytics_cluster::refinery::job::guard,
- # The eventlogging code is useful for scripting
- # EventLogging consumers. Install this on
- # stat1002, but don't run any daemons.
- include ::eventlogging
+ # Set up a read only rsync module to allow access
+ # to public data generated by the Analytics Cluster.
+ analytics_cluster::rsyncd,
+
+ # Deploy wikimedia/discovery/analytics repository
+ # to this node.
+ elasticsearch::analytics)
}
# stat1006 is a general purpose number cruncher for
diff --git a/modules/profile/manifests/statistics/private.pp
b/modules/profile/manifests/statistics/private.pp
index 3530960..a6d24c7 100644
--- a/modules/profile/manifests/statistics/private.pp
+++ b/modules/profile/manifests/statistics/private.pp
@@ -36,31 +36,33 @@
group => 'analytics-privatedata-users',
}
- # eventlogging logs are not private, but they
- # are here for convenience
+ # The eventlogging codebase is useful for scripting
+ # EventLogging consumers. Install this on
+ # stat1002, but don't run any daemons.
+ include ::eventlogging
+
+ # EventLogging Analytics data logs are not private, but they
+ # are rsynced here for convenience and backup redundancy.
include ::statistics::rsync::eventlogging
- # TODO remove this after stat1002 is gone: T152712
+ # TODO: Piecemeal migrate cron jobs from stat1002 -> stat1005: T152712
if $::hostname == 'stat1002' {
+ # TODO remove this after stat1002 is gone: T152712
+
# backup eventlogging logs.
backup::set { 'a-eventlogging' : }
+
+ # rsync mediawiki logs from logging hosts
+ include ::statistics::rsync::mediawiki
+
+ # WMDE statistics scripts and cron jobs
+ include ::statistics::wmde
+
+ # Discovery statistics generating scripts
+ include ::statistics::discovery
+
+ # Although it is in the "private" profile, the dataset actually isn't
+ # private. We just keep it here to spare adding a separate role.
+ include ::statistics::aggregator::projectview
}
-
- # rsync mediawiki logs from logging hosts
- include ::statistics::rsync::mediawiki
-
- # WMDE statistics scripts and cron jobs
- include ::statistics::wmde
-
- # Discovery statistics generating scripts
- include ::statistics::discovery
-
- # Although it is in the "private" profile, the dataset actually isn't
- # private. We just keep it here to spare adding a separate role.
- include ::statistics::aggregator::projectview
-
- # The eventlogging code is useful for scripting
- # EventLogging consumers. Install this,
- # but don't run any daemons.
- include ::eventlogging
}
diff --git a/modules/role/manifests/statistics/private_new.pp
b/modules/role/manifests/statistics/private_new.pp
new file mode 100644
index 0000000..0dedce6
--- /dev/null
+++ b/modules/role/manifests/statistics/private_new.pp
@@ -0,0 +1,13 @@
+# == Class role::statistics::cruncher_new
+# TODO: rename to private after stat1002 is gone.
+# (stat1005)
+class role::statistics::private_new {
+ system::role { 'statistics::private':
+ description => 'Statistics private data host and general compute node'
+ }
+
+ include ::profile::statistics::private
+
+ # Run Hadoop/Hive reportupdater jobs here.
+ # include ::profile::reportupdater::jobs::hadoop
+}
--
To view, visit https://gerrit.wikimedia.org/r/364808
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I4e4a84f23457e0ecdba484b47acdca489dfbcda9
Gerrit-PatchSet: 3
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
