Ottomata has uploaded a new change for review. https://gerrit.wikimedia.org/r/111455
Change subject: Parameterizing queue_buffering_max_ms and batch_num_messages ...................................................................... Parameterizing queue_buffering_max_ms and batch_num_messages Change-Id: I2181807d72e8dc9c2a99247b9bc739119704da62 --- M manifests/defaults.pp M manifests/init.pp M templates/varnishkafka.conf.erb 3 files changed, 18 insertions(+), 1 deletion(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet/varnishkafka refs/changes/55/111455/1 diff --git a/manifests/defaults.pp b/manifests/defaults.pp index bacd4ad..a8f76a9 100644 --- a/manifests/defaults.pp +++ b/manifests/defaults.pp @@ -14,6 +14,8 @@ $partition = -1 $queue_buffering_max_messages = 100000 + $queue_buffering_max_ms = 1000 + $batch_num_messages = 1000 $message_send_max_retries = 3 $topic_request_required_acks = 1 $topic_message_timeout_ms = 300000 diff --git a/manifests/init.pp b/manifests/init.pp index 3a30b02..21f3593 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -22,9 +22,13 @@ # $format_key - Kafka message key format string. # Default: undef (disables Kafka message key usage). # $partition - Topic partition number to send to. -1 for random. -# Default: -1. +# Default: -1 # $queue_buffering_max_messages - Maximum number of messages allowed on the # local Kafka producer queue. Default: 100000 +# $queue_buffering_max_ms - Maximum time, in milliseconds, for buffering +# data on the producer queue. Default: 1000 +# $batch_num_messages - Maximum number of messages batched in one MessageSet. +# Default: 1000 # $message_send_max_retries - Maximum number of retries per messageset. # Default: 3 # $topic_request_required_acks - Required ack level. Default: 1 @@ -79,6 +83,9 @@ $partition = $varnishkafka::defaults::partition, $queue_buffering_max_messages = $varnishkafka::defaults::queue_buffering_max_messages, + $queue_buffering_max_ms = $varnishkafka::defaults::queue_buffering_max_ms, + $batch_num_messages = $varnishkafka::defaults::batch_num_messages, + $message_send_max_retries = $varnishkafka::defaults::message_send_max_retries, $topic_request_required_acks = $varnishkafka::defaults::topic_request_required_acks, $topic_message_timeout_ms = $varnishkafka::defaults::topic_message_timeout_ms, diff --git a/templates/varnishkafka.conf.erb b/templates/varnishkafka.conf.erb index a438234..a6df726 100644 --- a/templates/varnishkafka.conf.erb +++ b/templates/varnishkafka.conf.erb @@ -251,6 +251,14 @@ # Defaults to 1000000 kafka.queue.buffering.max.messages = <%= @queue_buffering_max_messages %> +# Maximum time, in milliseconds, for buffering data on the producer queue. +# Defaults to 1000 (1 second) +kafka.queue.buffering.max.ms = <%= @queue_buffering_max_ms %> + +# Maximum number of messages batched in one MessageSet. +# Defaults to 1000 +kafka.batch.num.messages = <%= @batch_num_messages %> + # Maximum number of retries per messageset. kafka.message.send.max.retries = <%= @message_send_max_retries %> -- To view, visit https://gerrit.wikimedia.org/r/111455 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I2181807d72e8dc9c2a99247b9bc739119704da62 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet/varnishkafka Gerrit-Branch: master Gerrit-Owner: Ottomata <o...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits