Ottomata has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/404773 )
Change subject: [WIP] point eventlogging processes at Kafka jumbo ...................................................................... [WIP] point eventlogging processes at Kafka jumbo This needs to be merged with https://gerrit.wikimedia.org/r/#/c/403067/ It is in gerrit now to be cherry-picked in deployment-prep Bug: T183297 Change-Id: Iaf6f898b58a6564d2b22dce88ececfb415dc232e --- M modules/role/manifests/eventlogging/analytics/files.pp M modules/role/manifests/eventlogging/analytics/mysql.pp M modules/role/manifests/eventlogging/analytics/processor.pp M modules/role/manifests/eventlogging/analytics/server.pp 4 files changed, 6 insertions(+), 36 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/73/404773/1 diff --git a/modules/role/manifests/eventlogging/analytics/files.pp b/modules/role/manifests/eventlogging/analytics/files.pp index 0619e93..bf5e9e9 100644 --- a/modules/role/manifests/eventlogging/analytics/files.pp +++ b/modules/role/manifests/eventlogging/analytics/files.pp @@ -45,16 +45,6 @@ 'eventlogging_consumer_files_00' ) - # Where possible, if this is set, it will be included in client configuration - # to avoid having to do API version for Kafka < 0.10 (where there is not a version API). - $kafka_api_version = $role::eventlogging::analytics::server::kafka_config['api_version'] - - # Append this to query params if set. - $kafka_api_version_param = $kafka_api_version ? { - undef => '', - default => "&api_version=${kafka_api_version}" - } - # These commonly used URIs are defined for DRY purposes in # role::eventlogging::analytics::server. $kafka_client_side_raw_uri = $role::eventlogging::analytics::server::kafka_client_side_raw_uri @@ -62,7 +52,7 @@ # Raw client side events: eventlogging::service::consumer { 'client-side-events.log': - input => "${kafka_client_side_raw_uri}&raw=True${kafka_api_version_param}", + input => "${kafka_client_side_raw_uri}&raw=True", output => "file://${out_dir}/client-side-events.log", sid => $kafka_consumer_group, } @@ -71,7 +61,7 @@ # 'blacklisted' during processing. Events are blacklisted # from these logs for volume reasons. eventlogging::service::consumer { 'all-events.log': - input => "${kafka_mixed_uri}${kafka_api_version_param}", + input => $kafka_mixed_uri, output => "file://${out_dir}/all-events.log", sid => $kafka_consumer_group, } diff --git a/modules/role/manifests/eventlogging/analytics/mysql.pp b/modules/role/manifests/eventlogging/analytics/mysql.pp index cac5874..081f838 100644 --- a/modules/role/manifests/eventlogging/analytics/mysql.pp +++ b/modules/role/manifests/eventlogging/analytics/mysql.pp @@ -32,16 +32,6 @@ ['mysql-m4-master-00'] ) - # Where possible, if this is set, it will be included in client configuration - # to avoid having to do API version for Kafka < 0.10 (where there is not a version API). - $kafka_api_version = $role::eventlogging::analytics::server::kafka_config['api_version'] - - # Append this to query params if set. - $kafka_api_version_param = $kafka_api_version ? { - undef => '', - default => "&api_version=${kafka_api_version}" - } - $kafka_consumer_scheme = $role::eventlogging::analytics::server::kafka_consumer_scheme $kafka_brokers_string = $role::eventlogging::analytics::server::kafka_brokers_string @@ -68,7 +58,7 @@ # Kafka consumer group for this consumer is mysql-m4-master eventlogging::service::consumer { $mysql_consumers: # auto commit offsets to kafka more often for mysql consumer - input => "${map_scheme}${kafka_consumer_uri}&auto_commit_interval_ms=1000${kafka_api_version_param}${map_function}", + input => "${map_scheme}${kafka_consumer_uri}&auto_commit_interval_ms=1000${map_function}", output => "mysql://${mysql_user}:${mysql_pass}@${mysql_db}?charset=utf8&statsd_host=${statsd_host}&replace=True", sid => 'eventlogging_consumer_mysql_00', # Restrict permissions on this config file since it contains a password. diff --git a/modules/role/manifests/eventlogging/analytics/processor.pp b/modules/role/manifests/eventlogging/analytics/processor.pp index dbaf521..cb15c0c 100644 --- a/modules/role/manifests/eventlogging/analytics/processor.pp +++ b/modules/role/manifests/eventlogging/analytics/processor.pp @@ -6,9 +6,6 @@ class role::eventlogging::analytics::processor{ include role::eventlogging::analytics::server - # Where possible, if this is set, it will be included in client configuration - # to avoid having to do API version for Kafka < 0.10 (where there is not a version API). - $kafka_api_version = $role::eventlogging::analytics::server::kafka_config['api_version'] $kafka_brokers_string = $role::eventlogging::analytics::server::kafka_config['brokers']['string'] $kafka_consumer_group = hiera( @@ -47,13 +44,6 @@ default => "${kafka_producer_scheme}/${kafka_brokers_string}?topic=eventlogging-valid-mixed&blacklist=${mixed_schema_blacklist}" } - # Append this to query params of kafka-python writer if set. - # kafka-confluent defaults to setting this to 0.9 anyway. - $kafka_api_version_param = $kafka_api_version ? { - undef => '', - default => "&api_version=${kafka_api_version}" - } - # Increase number and backoff time of retries for async # analytics uses. If metadata changes, we should give # more time to retry. NOTE: testing this in production @@ -65,7 +55,7 @@ # args for kafka-confluent handler writer 'kafka-confluent://' => 'message.send.max.retries=6,retry.backoff.ms=200', # args for kafka-python handler writer - 'kafka://' => "retries=6&retry_backoff_ms=200${kafka_api_version_param}" + 'kafka://' => "retries=6&retry_backoff_ms=200" } # Incoming format from /beacon/event via varnishkafka eventlogging-client-side @@ -79,7 +69,7 @@ $format = '%q %{recvFrom}s %{seqId}d %D %o %u' eventlogging::service::processor { $client_side_processors: format => $format, - input => "${kafka_client_side_raw_uri}${kafka_api_version_param}", + input => "${kafka_client_side_raw_uri}", sid => $kafka_consumer_group, outputs => [ "${kafka_schema_output_uri}&${kafka_producer_args}", diff --git a/modules/role/manifests/eventlogging/analytics/server.pp b/modules/role/manifests/eventlogging/analytics/server.pp index 2c78208..b976841 100644 --- a/modules/role/manifests/eventlogging/analytics/server.pp +++ b/modules/role/manifests/eventlogging/analytics/server.pp @@ -24,7 +24,7 @@ } # Get the Kafka configuration - $kafka_config = kafka_config('analytics') + $kafka_config = kafka_config('jumbo') $kafka_brokers_string = $kafka_config['brokers']['string'] # Using kafka-confluent as a consumer is not currently supported by this puppet module, -- To view, visit https://gerrit.wikimedia.org/r/404773 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Iaf6f898b58a6564d2b22dce88ececfb415dc232e Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Ottomata <ao...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
