Ppchelko has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/372179 )
Change subject: WIP: Increase max kafka message size ...................................................................... WIP: Increase max kafka message size Change-Id: Ice77027db74e67c519ce305e99ea22c3ff2c7b07 --- M hieradata/common.yaml M modules/changeprop/manifests/init.pp M modules/confluent/manifests/kafka/broker.pp M modules/confluent/templates/kafka/server.properties.erb 4 files changed, 15 insertions(+), 1 deletion(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/79/372179/1 diff --git a/hieradata/common.yaml b/hieradata/common.yaml index 4922ec7..d6be87f 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -438,6 +438,7 @@ main-eqiad: api_version: 0.9 zookeeper_cluster_name: main-eqiad + kafka_message_max_bytes: 2000000 brokers: kafka1001.eqiad.wmnet: id: 1001 @@ -449,6 +450,7 @@ main-codfw: api_version: 0.9 zookeeper_cluster_name: main-codfw + kafka_message_max_bytes: 2000000 brokers: kafka2001.codfw.wmnet: id: 2001 diff --git a/modules/changeprop/manifests/init.pp b/modules/changeprop/manifests/init.pp index ef3b397..d83b107 100644 --- a/modules/changeprop/manifests/init.pp +++ b/modules/changeprop/manifests/init.pp @@ -48,7 +48,7 @@ include ::service::configuration require ::changeprop::packages - + service::node { 'changeprop': enable => true, port => 7272, @@ -66,6 +66,7 @@ site => $::site, redis_path => $redis_path, redis_pass => $redis_pass, + kafka_max_bytes => $::kafka_clusters::main_eqiad::kafka_message_max_bytes, }, auto_refresh => false, init_restart => false, diff --git a/modules/confluent/manifests/kafka/broker.pp b/modules/confluent/manifests/kafka/broker.pp index 99a7203..e2d3414 100644 --- a/modules/confluent/manifests/kafka/broker.pp +++ b/modules/confluent/manifests/kafka/broker.pp @@ -104,6 +104,9 @@ # partition in the fetch requests the replicas send to the leader. # Default: undef (1048576) # +# [*kafka_message_max_bytes*] +# The maximum size of a message that the server can receive. Default: undef +# # [*num_network_threads*] # The number of threads handling network requests. Default: undef (3) # @@ -237,6 +240,7 @@ $replica_socket_receive_buffer_bytes = undef, $num_replica_fetchers = 1, $replica_fetch_max_bytes = undef, + $kafka_message_max_bytes = undef, $num_network_threads = undef, $num_io_threads = size($log_dirs), diff --git a/modules/confluent/templates/kafka/server.properties.erb b/modules/confluent/templates/kafka/server.properties.erb index c2a8fe6..aec3cc3 100644 --- a/modules/confluent/templates/kafka/server.properties.erb +++ b/modules/confluent/templates/kafka/server.properties.erb @@ -273,3 +273,10 @@ # that additional support metrics will be collected and reported. # confluent.support.customer.id=anonymous + +<% if @kafka_message_max_bytes -%> + +# For the JobQueue we need larger messages then default +message.max.bytes=<%= @kafka_message_max_bytes %> + +<% end -%> \ No newline at end of file -- To view, visit https://gerrit.wikimedia.org/r/372179 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ice77027db74e67c519ce305e99ea22c3ff2c7b07 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Ppchelko <ppche...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits