Ottomata has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/404737 )
Change subject: [WIP] Produce webrequests from varnishkafka to jumbo Kafka
cluster via TLS
......................................................................
[WIP] Produce webrequests from varnishkafka to jumbo Kafka cluster via TLS
This needs a lot of very careful review and coordination to merge in prod.
For now this exists in gerrit and is cherry-picked in deployment-prep.
Bug: T175461
Change-Id: I1760c36ee26f015617472073e4c5ab95d53d3e44
---
M modules/profile/manifests/cache/kafka/webrequest.pp
1 file changed, 27 insertions(+), 18 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/37/404737/1
diff --git a/modules/profile/manifests/cache/kafka/webrequest.pp
b/modules/profile/manifests/cache/kafka/webrequest.pp
index 655779b..50321b1 100644
--- a/modules/profile/manifests/cache/kafka/webrequest.pp
+++ b/modules/profile/manifests/cache/kafka/webrequest.pp
@@ -1,34 +1,39 @@
# === class profile::cache::kafka::webrequest
#
# Sets up a varnishkafka instance producing varnish
-# webrequest logs to the analytics Kafka brokers in eqiad.
+# webrequest logs to a Kafka cluster via TLS.
#
# === Parameters
#
-# [*monitoring_enabled*]
-# True if the varnishkafka instance should be monitored.
-#
# [*cache_cluster*]
-# the name of the cache cluster
+# The name of the cache cluster.
#
# [*statsd*]
# The host:port to send statsd data to.
#
+# [*kafka_cluster_name*]
+# Name of the Kafka cluster in the hiera kafka_clusters hash. This can
+# be unqualified (without DC suffix) or fully qualified. Default: jumbo
+#
+# [*monitoring_enabled*]
+# True if the varnishkafka instance should be monitored. Default: false
+#
class profile::cache::kafka::webrequest(
- $monitoring_enabled =
hiera('profile::cache::kafka::webrequest::monitoring_enabled', false),
$cache_cluster = hiera('cache::cluster'),
$statsd = hiera('statsd'),
+ $kafka_cluster_name =
hiera('profile::cache::kafka::webrequest::kafka_cluster_name', 'jumbo'),
+ $monitoring_enabled =
hiera('profile::cache::kafka::webrequest::monitoring_enabled', false),
) {
- $config = kafka_config('analytics')
- # NOTE: This is used by inheriting classes role::cache::kafka::*
- $kafka_brokers = $config['brokers']['array']
+ # Include this class to get key and certificate for varnishkafka
+ # to produce to Kafka over SSL/TLS.
+ require ::profile::cache::kafka::certificate
- $topic = "webrequest_${cache_cluster}"
- # These used to be parameters, but I don't really see why given we never
change
- # them
- $varnish_name = 'frontend'
- $varnish_svc_name = 'varnish-frontend'
- $kafka_protocol_version = '0.9.0.1'
+ $config = kafka_config($kafka_cluster_name)
+ $kafka_brokers = $config['brokers']['ssl_array']
+
+ $topic = "webrequest_${cache_cluster}"
+ $varnish_name = 'frontend'
+ $varnish_svc_name = 'varnish-frontend'
# Background task: T136314
# Background info about the parameters used:
@@ -88,10 +93,7 @@
$peak_rps_estimate = 9000
varnishkafka::instance { 'webrequest':
- # FIXME - top-scope var without namespace, will break in puppet 2.8
- # lint:ignore:variable_scope
brokers => $kafka_brokers,
- # lint:endignore
topic => $topic,
format_type => 'json',
compression_codec => 'snappy',
@@ -122,6 +124,13 @@
# stats will be fresh when polled from gmetad.
log_statistics_interval => 15,
force_protocol_version => $kafka_protocol_version,
+ #TLS/SSL config
+ ssl_enabled => true,
+ ssl_ca_location =>
$::profile::cache::kafka::certificate::ssl_ca_location,
+ ssl_key_password =>
$::profile::cache::kafka::certificate::ssl_key_password,
+ ssl_key_location =>
$::profile::cache::kafka::certificate::ssl_key_location,
+ ssl_certificate_location =>
$::profile::cache::kafka::certificate::ssl_certificate_location,
+ ssl_cipher_suites =>
$::profile::cache::kafka::certificate::ssl_cipher_suites,
}
if $monitoring_enabled {
--
To view, visit https://gerrit.wikimedia.org/r/404737
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I1760c36ee26f015617472073e4c5ab95d53d3e44
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <ao...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
