Ottomata has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/386648 )
Change subject: Run MirrorMaker on analytics Kafka hosts to mirror main -> jumbo
......................................................................
Run MirrorMaker on analytics Kafka hosts to mirror main -> jumbo
Bug: T177216
Change-Id: I9c2f31e3fb1a4669d980906beb2dfbf37dc25abf
---
M manifests/site.pp
A modules/role/manifests/kafka/jumbo/mirror.pp
2 files changed, 72 insertions(+), 0 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
diff --git a/manifests/site.pp b/manifests/site.pp
index 4c31376..13804a4 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -1214,6 +1214,13 @@
role(kafka::analytics::broker,
# Mirror all other Kafka cluster data into the analytics Kafka cluster.
kafka::analytics::mirror,
+ # Mirror main Kafka cluster data to Jumbo Kafka cluster.
+ # NOTE: this is only running on the analytics Kafka brokers because
+ # of a 0.11 client compatibility issue. Ideally this role would
+ # be included on the jumbo brokers instead. But, since we need to
consume
+ # from a 0.9 cluster (main), we need to use a non 0.11 MirrorMaker
version,
+ # which is not available on the Kafka jumbo brokers, since they are
0.11.
+ kafka::jumbo::mirror,
ipsec)
include ::standard
diff --git a/modules/role/manifests/kafka/jumbo/mirror.pp
b/modules/role/manifests/kafka/jumbo/mirror.pp
new file mode 100644
index 0000000..ed485fb
--- /dev/null
+++ b/modules/role/manifests/kafka/jumbo/mirror.pp
@@ -0,0 +1,65 @@
+# == Class role::kafka::jumbo::mirror
+# Defines Kafka MirrorMaker instances to mirror
+# main Kafka cluster data into the jumbo Kafka Cluster
+#
+# In production, the only other Kafka clusters
+# (as of October 2017) are main-eqiad and main-codfw.
+# Those clusters mirror prefixed topics to each other,
+# in a 'master <-> master' (not really) setup.
+# Thus, main-eqiad will have all the topics for
+# itself and main-codfw. It is sufficient
+# to just mirror all topics from main-eqiad into
+# jumbo-eqiad.
+#
+# It is safe to include this class on multiple hosts.
+# Each host it is included on will just fire
+# up a new MirrorMaker consumer that will be auto balanced
+# by Kafka.
+#
+# NOTE: We would much rather just include profile::kafka::mirror here,
+# rather than manually configuring the mirror maker instance in this
+# role class. However, since profile::kafka::mirror is for configuring
+# a 0.11 MirrorMaker instance, we can't yet use it, since
+# 0.11 Kafka clients are incompatible with 0.9 Brokers (which is what
+# main Kafka clusters use). Once main Kafka clusters are upgraded to 0.9,
+# we can use profile::kafka::mirror instead.
+#
+class role::kafka::jumbo::mirror {
+ # For safety, only allow this class to be included in eqiad.
+ if $::site != 'eqiad' {
+ fail('role::kafka::jumbo::mirror must only be used in eqiad')
+ }
+
+ $source_config = kafka_config('main')
+ $source_cluster_name = $source_config['name']
+
+ $destination_config = kafka_config('jumbo')
+ $destination_cluster_name = $destination_config['name']
+
+ # TODO: fix this hiera lookup when this is moved into profiles
+ $kafka_message_max_bytes = hiera('kafka_message_max_bytes', 1048576)
+ # The requests not only contain the message but also a small metadata
overhead.
+ # So if we want to produce a kafka_message_max_bytes payload the max
request size should be a bit higher.
+ # The 48564 value isn't arbitrary - it's the difference between default
message.max.size and default max.request.size
+ $producer_request_max_size = $kafka_message_max_bytes + 48564
+ $producer_properties = {
+ 'max.request.size' => $producer_request_max_size,
+ }
+
+ $consumer_properties = {
+ 'fetch.message.max.bytes' => $producer_request_max_size,
+ # TODO: This property will only be used by new consumers, which we
don't use yet.
+ # 'max.partition.fetch.bytes' => $producer_request_max_size
+ }
+
+ ::confluent::kafka::mirror::instance {
"${source_cluster_name}_to_${destination_cluster_name}":
+ source_zookeeper_url => $source_config['zookeeper']['url'],
+ destination_brokers =>
split($destination_config['brokers']['string'], ','),
+ # Avoid conflict on port 9997 with main -> analytics mirror maker
+ jmx_port => 9996,
+ num_streams => 2,
+ offset_commit_interval_ms => 5000,
+ producer_properties => $producer_properties,
+ consumer_properties => $consumer_properties,
+ }
+}
--
To view, visit https://gerrit.wikimedia.org/r/386648
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I9c2f31e3fb1a4669d980906beb2dfbf37dc25abf
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <ao...@wikimedia.org>
Gerrit-Reviewer: Elukey <ltosc...@wikimedia.org>
Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org>
Gerrit-Reviewer: Ottomata <ao...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
