Ottomata has submitted this change and it was merged.
Change subject: Adding role/analytics/kafka.pp Also adding modules/kafka
......................................................................
Adding role/analytics/kafka.pp
Also adding modules/kafka
Change-Id: Ia4eb812341705e6beb32143ca7eb1b06781ee6cf
---
M .gitmodules
A manifests/role/analytics/kafka.pp
M manifests/role/analytics/zookeeper.pp
M manifests/site.pp
A modules/kafka
5 files changed, 79 insertions(+), 7 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
jenkins-bot: Verified
diff --git a/.gitmodules b/.gitmodules
index 043236c..a5b3306 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -7,3 +7,6 @@
[submodule "modules/jmxtrans"]
path = modules/jmxtrans
url = https://gerrit.wikimedia.org/r/operations/puppet/jmxtrans
+[submodule "modules/kafka"]
+ path = modules/kafka
+ url = https://gerrit.wikimedia.org/r/p/operations/puppet/kafka
diff --git a/manifests/role/analytics/kafka.pp
b/manifests/role/analytics/kafka.pp
new file mode 100644
index 0000000..b34d2a6
--- /dev/null
+++ b/manifests/role/analytics/kafka.pp
@@ -0,0 +1,72 @@
+# role/analytics/kafka.pp
+#
+# Role classes for Analytics Kakfa nodes.
+# These role classes will configure Kafka properly in either
+# the Analytics labs or Analytics production environments.
+#
+# Usage:
+#
+# If you only need the Kafka package and configs to use the
+# Kafka client to talk to Kafka Broker Servers:
+#
+# include role::analytics::kafka::client
+#
+# If you want to set up a Kafka Broker Server
+# include role::analytics::kafka::server
+#
+class role::analytics::kafka::config {
+ # This allows labs to set the $::kafka_cluster global,
+ # which will conditionally select labs hosts to include
+ # in a Kafka cluster. This allows us to test cross datacenter
+ # broker mirroring with multiple clusters.
+ $kafka_cluster_name = $::kafka_cluster ? {
+ undef => $::site,
+ default => $::kafka_cluster,
+ }
+
+ if ($::realm == 'labs') {
+ # TODO: Make hostnames configurable via labs global variables.
+ $cluster = {
+ 'main' => {
+ 'kraken-kafka.pmtpa.wmflabs' => { 'id' => 1 },
+ 'kraken-kafka1.pmtpa.wmflabs' => { 'id' => 2 },
+ },
+ 'external' => {
+ 'kraken-kafka-external.pmtpa.wmflabs' => { 'id' => 10 },
+ },
+ }
+ }
+ # else Kafka cluster is based on $::site.
+ else {
+ $cluster = {
+ 'eqiad' => {
+ 'analytics1021.eqiad.wmnet' => { 'id' => 21 },
+ 'analytics1022.eqiad.wmnet' => { 'id' => 22 },
+ },
+ # 'ulsfo' => { },
+ # 'pmtpa' => { },
+ # 'esams' => { },
+ }
+ }
+
+ $hosts = $cluster[$kafka_cluster_name]
+ $zookeeper_chroot = "/kafka-${kafka_cluster_name}"
+}
+
+# == Class role::analytics::kafka::client
+#
+class role::analytics::kafka::client inherits role::analytics::kafka::config {
+ require role::analytics::zookeeper::config
+
+ class { '::kafka':
+ hosts => $hosts,
+ zookeeper_hosts => $role::analytics::zookeeper::config::hosts_array,
+ zookeeper_chroot => $zookeeper_chroot,
+ }
+}
+
+# == Class role::analytics::kafka::server
+#
+class role::analytics::kafka::server inherits role::analytics::kafka::client {
+ class { '::kafka::server': }
+}
diff --git a/manifests/role/analytics/zookeeper.pp
b/manifests/role/analytics/zookeeper.pp
index 4520846..99ba6b5 100644
--- a/manifests/role/analytics/zookeeper.pp
+++ b/manifests/role/analytics/zookeeper.pp
@@ -1,7 +1,7 @@
# role/analytics/zookeeper.pp
#
# Role classes for Analytics Zookeeper nodes.
-# These role classes will configure Zooekeeper properly in either
+# These role classes will configure Zookeeper properly in either
# the Analytics labs or Analytics production environments.
#
# Usage:
@@ -20,10 +20,9 @@
# $role::analytics::zookeeper::hosts_array variables.
#
class role::analytics::zookeeper::config {
- # TODO: Make this configurable via labs
- # global variables.
+ # TODO: Make this configurable via labs global variables.
$labs_hosts = {
- 'kraken-puppet.pmtpa.wmflabs' => 1,
+ 'kraken-zookeeper.pmtpa.wmflabs' => 1,
}
$production_hosts = {
diff --git a/manifests/site.pp b/manifests/site.pp
index f175961..28a3773 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -154,7 +154,6 @@
include role::analytics::hadoop::master
}
-
# analytics1011-analytics1020 are Hadoop worker nodes
node /analytics10(1[1-9]|20).eqiad.wmnet/ {
# ganglia aggregator for the Analytics cluster.
@@ -170,10 +169,9 @@
}
# analytics1021 and analytics1022 are Kafka Brokers.
-# NOTE: These still need reinstalled as of 2013-08-26
node /analytics102[12]\.eqiad\.wmnet/ {
include role::analytics
- # include role::analytics::kafka::server
+ include role::analytics::kafka::server
}
# analytics1023-1025 are zookeeper server nodes
diff --git a/modules/kafka b/modules/kafka
new file mode 160000
index 0000000..ee8667d
--- /dev/null
+++ b/modules/kafka
+Subproject commit ee8667d98f4bf0beba135535619f3236cefcb1c2
--
To view, visit https://gerrit.wikimedia.org/r/77971
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ia4eb812341705e6beb32143ca7eb1b06781ee6cf
Gerrit-PatchSet: 13
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <o...@wikimedia.org>
Gerrit-Reviewer: Ottomata <o...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
