Giuseppe Lavagetto has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/402784 )
Change subject: kafka: create compound roles, one role() call per node
definition
......................................................................
kafka: create compound roles, one role() call per node definition
Change-Id: If38f8b18967c883fa7982b5b16751f68e1d69d03
---
A hieradata/role/common/kafka/analytics.yaml
D hieradata/role/common/kafka/analytics/broker.yaml
A hieradata/role/common/kafka/main.yaml
M manifests/site.pp
A modules/role/manifests/kafka/analytics.pp
A modules/role/manifests/kafka/main.pp
6 files changed, 51 insertions(+), 30 deletions(-)
Approvals:
Giuseppe Lavagetto: Looks good to me, approved
jenkins-bot: Verified
diff --git a/hieradata/role/common/kafka/analytics.yaml
b/hieradata/role/common/kafka/analytics.yaml
new file mode 100644
index 0000000..d714f2f
--- /dev/null
+++ b/hieradata/role/common/kafka/analytics.yaml
@@ -0,0 +1,11 @@
+admin::groups:
+ - analytics-admins
+cluster: analytics_kafka
+nagios_group: analytics_eqiad
+burrow::lagcheck_intervals: 100
+burrow::httpserver_port: 8000
+burrow::consumer_groups:
+ - eventlogging_processor_client_side_00
+ - eventlogging_consumer_mysql_00
+ - eventlogging_consumer_mysql_eventbus_00
+ - eventlogging_consumer_files_00
diff --git a/hieradata/role/common/kafka/analytics/broker.yaml
b/hieradata/role/common/kafka/analytics/broker.yaml
deleted file mode 100644
index 79be700..0000000
--- a/hieradata/role/common/kafka/analytics/broker.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-admin::groups:
- - analytics-admins
-cluster: analytics_kafka
-nagios_group: analytics_eqiad
diff --git a/hieradata/role/common/kafka/main.yaml
b/hieradata/role/common/kafka/main.yaml
new file mode 100644
index 0000000..201622d
--- /dev/null
+++ b/hieradata/role/common/kafka/main.yaml
@@ -0,0 +1,5 @@
+cluster: eventbus
+admin::groups:
+ - eventbus-admins
+role::lvs::realserver::pools:
+ eventbus: {}
diff --git a/manifests/site.pp b/manifests/site.pp
index edf431e..15e9db2 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -1066,38 +1066,14 @@
# Analytics Kafka Brokers
node /kafka10(12|13|14|20|22|23)\.eqiad\.wmnet/ {
- # Kafka brokers are routed via IPv6 so that
- # other DCs can address without public IPv4
- # addresses.
- interface::add_ip6_mapped { 'main': }
-
- role(kafka::analytics::broker,
- # Mirror all other Kafka cluster data into the analytics Kafka cluster.
- kafka::analytics::mirror,
- # Mirror main Kafka cluster data to Jumbo Kafka cluster.
- # NOTE: this is only running on the analytics Kafka brokers because
- # of a 0.11 client compatibility issue. Ideally this role would
- # be included on the jumbo brokers instead. But, since we need to
consume
- # from a 0.9 cluster (main), we need to use a non 0.11 MirrorMaker
version,
- # which is not available on the Kafka jumbo brokers, since they are
0.11.
- kafka::jumbo::mirror,
- ipsec)
-
- include ::standard
- include ::base::firewall
+ role(kafka::analytics)
}
# Kafka Brokers - main-eqiad and main-codfw Kafka clusters.
# For now, eventlogging-service-eventbus is also colocated
# on these brokers.
node /kafka[12]00[123]\.(eqiad|codfw)\.wmnet/ {
- role(kafka::main::broker,
- # Mirror eqiad.* topics from main-eqiad into main-codfw,
- # or mirror codfw.* topics from main-codfw into main-eqiad.
- kafka::main::mirror,
- eventbus::eventbus)
-
- include ::standard
+ role(kafka::main)
}
# kafka-jumbo is a large general purpose Kafka cluster.
diff --git a/modules/role/manifests/kafka/analytics.pp
b/modules/role/manifests/kafka/analytics.pp
new file mode 100644
index 0000000..d9c5fa9
--- /dev/null
+++ b/modules/role/manifests/kafka/analytics.pp
@@ -0,0 +1,23 @@
+# Compound role for analytics kafka
+class role::kafka::analytics {
+ system::role { 'kafka_analytics': }
+ # Kafka brokers are routed via IPv6 so that
+ # other DCs can address without public IPv4
+ # addresses.
+ interface::add_ip6_mapped { 'main': }
+
+ include ::role::kafka::analytics::broker
+ # Mirror all other Kafka cluster data into the analytics Kafka cluster.
+ include ::role::kafka::analytics::mirror
+ # Mirror main Kafka cluster data to Jumbo Kafka cluster.
+ # NOTE: this is only running on the analytics Kafka brokers because
+ # of a 0.11 client compatibility issue. Ideally this role would
+ # be included on the jumbo brokers instead. But, since we need to consume
+ # from a 0.9 cluster (main), we need to use a non 0.11 MirrorMaker version,
+ # which is not available on the Kafka jumbo brokers, since they are 0.11.
+ include ::role::kafka::jumbo::mirror
+ include ::role::ipsec
+
+ include ::standard
+ include ::base::firewall
+}
diff --git a/modules/role/manifests/kafka/main.pp
b/modules/role/manifests/kafka/main.pp
new file mode 100644
index 0000000..8ca7176
--- /dev/null
+++ b/modules/role/manifests/kafka/main.pp
@@ -0,0 +1,10 @@
+# Compound role for the Kafka "main" cluster
+class role::kafka::main {
+ include ::role::kafka::main::broker
+ # Mirror eqiad.* topics from main-eqiad into main-codfw,
+ # or mirror codfw.* topics from main-codfw into main-eqiad.
+ include ::role::kafka::main::mirror
+ include ::role::eventbus::eventbus
+
+ include ::standard
+}
--
To view, visit https://gerrit.wikimedia.org/r/402784
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: If38f8b18967c883fa7982b5b16751f68e1d69d03
Gerrit-PatchSet: 3
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Giuseppe Lavagetto <glavage...@wikimedia.org>
Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
