Ottomata has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/384586 )
Change subject: Set up Kafka MirrorMaker from main -> jumbo in eqiad
......................................................................
Set up Kafka MirrorMaker from main -> jumbo in eqiad
This includes a new profile::kafka::mirror class. This
class can be used to set up a single Kafka MirrorMaker instance,
and as such, cannot be used to set up multiple MirrorMaker
instances running on a single node. For now, this should
be fine, as we don't have this use case.
TODO:
- Prometheus monitoring
- TLS configuration
Bug: T177216
Change-Id: I020a9c1a5cb1e3c18a33f18c2ad91ea915a74ed7
---
M hieradata/role/common/kafka/jumbo/broker.yaml
M modules/confluent/manifests/kafka/mirror/instance.pp
M modules/confluent/templates/kafka/mirror/consumer.properties.erb
M modules/role/manifests/kafka/jumbo/broker.pp
A modules/role/manifests/kafka/simple/mirror.pp
5 files changed, 39 insertions(+), 3 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/86/384586/1
diff --git a/hieradata/role/common/kafka/jumbo/broker.yaml
b/hieradata/role/common/kafka/jumbo/broker.yaml
index cb776b3..3f58fed 100644
--- a/hieradata/role/common/kafka/jumbo/broker.yaml
+++ b/hieradata/role/common/kafka/jumbo/broker.yaml
@@ -34,3 +34,10 @@
profile::kafka::broker::monitoring::replica_maxlag_warning: 1000000
profile::kafka::broker::monitoring::replica_maxlag_critical: 5000000
+
+# Mirror all topics from main to jumbo.
+profile::kafka::mirror::source_cluster_name: main
+profile::kafka::mirror::destination_cluster_name: jumbo
+profile::kafka::mirror::properties:
+ num_streams: 2
+ offset_commit_interval_ms: 5000
diff --git a/modules/confluent/manifests/kafka/mirror/instance.pp
b/modules/confluent/manifests/kafka/mirror/instance.pp
index f77e36a..64eaa00 100644
--- a/modules/confluent/manifests/kafka/mirror/instance.pp
+++ b/modules/confluent/manifests/kafka/mirror/instance.pp
@@ -8,7 +8,13 @@
# [*source_zookeeper_url*]
# The URL that the source Kafka cluster users for coordination.
# The MirrorMaker consumer users this to look up source cluster
-# metadata and start consuming.
+# metadata and start consuming. NOTE: This parameter is deprecated.
+# 0.11 does not require zookeeper for consumer. Use $source_brokers instead.
+# TODO: Remove this parameter once all clusters are upgraded and using newer
mirror maker.
+#
+# [*source_brokers*]
+# Array of Kafka broker hosts in your source cluster. These brokers
+# will be used for bootstrapping the consumer configs and metadata.
#
# [*destination_brokers*]
# Array of Kafka brokers hosts in your destination cluster. These brokers
@@ -81,10 +87,12 @@
# }
#
define confluent::kafka::mirror::instance(
- # Consumer Settings
- $source_zookeeper_url,
$destination_brokers,
$jmx_port,
+
+ # TODO: make source_brokers required when we remove $source_zookeeper_url
after upgrading mirror maker
+ $source_brokers = undef,
+ $source_zookeeper_url = undef,
$consumer_properties = {},
@@ -109,6 +117,7 @@
$producer_properties_template =
'confluent/kafka/mirror/producer.properties.erb',
$default_template =
'confluent/kafka/mirror/kafka-mirror.default.erb',
$log4j_properties_template =
'confluent/kafka/mirror/log4j.properties.erb',
+
)
{
require ::confluent::kafka::client
diff --git a/modules/confluent/templates/kafka/mirror/consumer.properties.erb
b/modules/confluent/templates/kafka/mirror/consumer.properties.erb
index 80bd9b9..313f259 100644
--- a/modules/confluent/templates/kafka/mirror/consumer.properties.erb
+++ b/modules/confluent/templates/kafka/mirror/consumer.properties.erb
@@ -9,10 +9,16 @@
# Kafka Consumer group id
group.id=kafka-mirror-<%= @mirror_name %>
+<% if @source_zookeeper_url -%>
# Zookeeper connection string
# comma separated host:port pairs, each corresponding to a zk
# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002"
zookeeper.connect=<%= @source_zookeeper_url %>
+<% else -%>
+# list of brokers used for bootstrapping
+# format: host1:port1,host2:port2 ...
+bootstrap.servers=<%= Array(@source_brokers).join(',') %>
+<% end -%>
########################## Additional Propeties ##########################
diff --git a/modules/role/manifests/kafka/jumbo/broker.pp
b/modules/role/manifests/kafka/jumbo/broker.pp
index c9875be..f84b461 100644
--- a/modules/role/manifests/kafka/jumbo/broker.pp
+++ b/modules/role/manifests/kafka/jumbo/broker.pp
@@ -12,4 +12,7 @@
}
include base::firewall
include profile::kafka::broker
+
+ # Mirror from other Kafka cluster to jumbo.
+ include profile::kafka::mirror
}
diff --git a/modules/role/manifests/kafka/simple/mirror.pp
b/modules/role/manifests/kafka/simple/mirror.pp
new file mode 100644
index 0000000..2a67857
--- /dev/null
+++ b/modules/role/manifests/kafka/simple/mirror.pp
@@ -0,0 +1,11 @@
+# == Class role::kafka::simple::mirror
+# This is useful for testing Kafka MirrorMaker in labs.
+#
+class role::kafka::simple::mirror {
+ system::role { 'role::kafka::simple::mirror':
+ description => "Kafka simple MirrorMaker instance"
+ }
+
+ # include profile::kafka::mirrors
+ include profile::kafka::mirror
+}
--
To view, visit https://gerrit.wikimedia.org/r/384586
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I020a9c1a5cb1e3c18a33f18c2ad91ea915a74ed7
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
