Ottomata has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/386648 )
Change subject: Run MirrorMaker on analytics Kafka hosts to mirror main -> jumbo ...................................................................... Run MirrorMaker on analytics Kafka hosts to mirror main -> jumbo Bug: T177216 Change-Id: I9c2f31e3fb1a4669d980906beb2dfbf37dc25abf --- M manifests/site.pp A modules/role/manifests/kafka/jumbo/mirror.pp 2 files changed, 72 insertions(+), 0 deletions(-) Approvals: Ottomata: Verified; Looks good to me, approved diff --git a/manifests/site.pp b/manifests/site.pp index 4c31376..13804a4 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -1214,6 +1214,13 @@ role(kafka::analytics::broker, # Mirror all other Kafka cluster data into the analytics Kafka cluster. kafka::analytics::mirror, + # Mirror main Kafka cluster data to Jumbo Kafka cluster. + # NOTE: this is only running on the analytics Kafka brokers because + # of a 0.11 client compatibility issue. Ideally this role would + # be included on the jumbo brokers instead. But, since we need to consume + # from a 0.9 cluster (main), we need to use a non 0.11 MirrorMaker version, + # which is not available on the Kafka jumbo brokers, since they are 0.11. + kafka::jumbo::mirror, ipsec) include ::standard diff --git a/modules/role/manifests/kafka/jumbo/mirror.pp b/modules/role/manifests/kafka/jumbo/mirror.pp new file mode 100644 index 0000000..ed485fb --- /dev/null +++ b/modules/role/manifests/kafka/jumbo/mirror.pp @@ -0,0 +1,65 @@ +# == Class role::kafka::jumbo::mirror +# Defines Kafka MirrorMaker instances to mirror +# main Kafka cluster data into the jumbo Kafka Cluster +# +# In production, the only other Kafka clusters +# (as of October 2017) are main-eqiad and main-codfw. +# Those clusters mirror prefixed topics to each other, +# in a 'master <-> master' (not really) setup. +# Thus, main-eqiad will have all the topics for +# itself and main-codfw. It is sufficient +# to just mirror all topics from main-eqiad into +# jumbo-eqiad. +# +# It is safe to include this class on multiple hosts. +# Each host it is included on will just fire +# up a new MirrorMaker consumer that will be auto balanced +# by Kafka. +# +# NOTE: We would much rather just include profile::kafka::mirror here, +# rather than manually configuring the mirror maker instance in this +# role class. However, since profile::kafka::mirror is for configuring +# a 0.11 MirrorMaker instance, we can't yet use it, since +# 0.11 Kafka clients are incompatible with 0.9 Brokers (which is what +# main Kafka clusters use). Once main Kafka clusters are upgraded to 0.9, +# we can use profile::kafka::mirror instead. +# +class role::kafka::jumbo::mirror { + # For safety, only allow this class to be included in eqiad. + if $::site != 'eqiad' { + fail('role::kafka::jumbo::mirror must only be used in eqiad') + } + + $source_config = kafka_config('main') + $source_cluster_name = $source_config['name'] + + $destination_config = kafka_config('jumbo') + $destination_cluster_name = $destination_config['name'] + + # TODO: fix this hiera lookup when this is moved into profiles + $kafka_message_max_bytes = hiera('kafka_message_max_bytes', 1048576) + # The requests not only contain the message but also a small metadata overhead. + # So if we want to produce a kafka_message_max_bytes payload the max request size should be a bit higher. + # The 48564 value isn't arbitrary - it's the difference between default message.max.size and default max.request.size + $producer_request_max_size = $kafka_message_max_bytes + 48564 + $producer_properties = { + 'max.request.size' => $producer_request_max_size, + } + + $consumer_properties = { + 'fetch.message.max.bytes' => $producer_request_max_size, + # TODO: This property will only be used by new consumers, which we don't use yet. + # 'max.partition.fetch.bytes' => $producer_request_max_size + } + + ::confluent::kafka::mirror::instance { "${source_cluster_name}_to_${destination_cluster_name}": + source_zookeeper_url => $source_config['zookeeper']['url'], + destination_brokers => split($destination_config['brokers']['string'], ','), + # Avoid conflict on port 9997 with main -> analytics mirror maker + jmx_port => 9996, + num_streams => 2, + offset_commit_interval_ms => 5000, + producer_properties => $producer_properties, + consumer_properties => $consumer_properties, + } +} -- To view, visit https://gerrit.wikimedia.org/r/386648 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I9c2f31e3fb1a4669d980906beb2dfbf37dc25abf Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Ottomata <ao...@wikimedia.org> Gerrit-Reviewer: Elukey <ltosc...@wikimedia.org> Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org> Gerrit-Reviewer: Ottomata <ao...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits