Filippo Giunchedi has submitted this change and it was merged.
Change subject: graphite: introduce carbon-c-relay
......................................................................
graphite: introduce carbon-c-relay
The idea behind this change is to have a local carbon-c-relay instance (on port
1903) whose sole task is to forward incoming metrics to the local carbon-cache,
in practice replacing carbon-relay's role.
In turn, this local carbon-c-relay will receive metrics from a frontend
carbon-c-relay listening on standard port 2003 for line/plaintext protocol
metrics.
Bug: T85908
Change-Id: I61f687237baed6674fcc9813acc9df0ce40d4fbb
---
M manifests/role/graphite.pp
M modules/graphite/files/carbonctl
A modules/graphite/manifests/carbon_c_relay.pp
M modules/graphite/manifests/init.pp
A modules/graphite/templates/frontend-relay.conf.erb
A modules/graphite/templates/initscripts/frontend-relay.upstart.erb
A modules/graphite/templates/initscripts/local-relay.upstart.erb
A modules/graphite/templates/local-relay.conf.erb
8 files changed, 142 insertions(+), 13 deletions(-)
Approvals:
Filippo Giunchedi: Verified; Looks good to me, approved
Alexandros Kosiaris: Looks good to me, but someone else must approve
diff --git a/manifests/role/graphite.pp b/manifests/role/graphite.pp
index 461b094..831a960 100644
--- a/manifests/role/graphite.pp
+++ b/manifests/role/graphite.pp
@@ -16,13 +16,14 @@
# Set to true to enable LDAP based authentication to access the graphite
interface
#
class role::graphite::base(
- $storage_dir = '/var/lib/carbon',
- $auth = true,
- $hostname = 'graphite.wikimedia.org',
+ $storage_dir = '/var/lib/carbon',
+ $auth = true,
+ $hostname = 'graphite.wikimedia.org',
+ $c_relay_settings = {},
) {
include ::passwords::graphite
- if ($::realm == 'labs') {
+ if $::realm == 'labs' {
# Mount extra disk on /srv so carbon has somewhere to store metrics
require role::labs::lvm::srv
}
@@ -141,10 +142,9 @@
## Carbon relay ##
'relay' => {
- line_receiver_interface => '0.0.0.0',
pickle_receiver_interface => '0.0.0.0',
- udp_receiver_interface => '0.0.0.0',
- enable_udp_listener => true,
+ # disabled, see ::graphite::carbon_c_relay
+ line_receiver_port => '0',
relay_method => 'consistent-hashing',
max_queue_size => '500000',
destinations => [
@@ -162,6 +162,7 @@
storage_dir => $carbon_storage_dir,
whisper_lock_writes => true,
+ c_relay_settings => $c_relay_settings,
}
class { '::graphite::web':
@@ -218,8 +219,13 @@
#
class role::graphite::production {
class { 'role::graphite::base':
- storage_dir => '/var/lib/carbon',
- auth => true,
+ storage_dir => '/var/lib/carbon',
+ auth => true,
+ c_relay_settings => {
+ backends => [
+ 'graphite1001.eqiad.wmnet:1903',
+ ],
+ }
}
include role::backup::host
diff --git a/modules/graphite/files/carbonctl b/modules/graphite/files/carbonctl
index e6d9437..b4a2afc 100755
--- a/modules/graphite/files/carbonctl
+++ b/modules/graphite/files/carbonctl
@@ -12,6 +12,8 @@
/sbin/status carbon/cache NAME="$name" |& sed 's/ //'
done
grep -Fqx '[relay]' /etc/carbon/carbon.conf && /sbin/status
carbon/relay
+ [ -e /etc/carbon/local-relay.conf ] && /sbin/status
carbon/local-relay
+ [ -e /etc/carbon/frontend-relay.conf ] && /sbin/status
carbon/frontend-relay
} |& { sed 's/, process//' | column -t | tee /dev/stderr | grep -qv
running ; } 2>&1
;;
check)
diff --git a/modules/graphite/manifests/carbon_c_relay.pp
b/modules/graphite/manifests/carbon_c_relay.pp
new file mode 100644
index 0000000..e1f35d8
--- /dev/null
+++ b/modules/graphite/manifests/carbon_c_relay.pp
@@ -0,0 +1,44 @@
+# == Class: graphite::carbon_c_relay
+#
+# This class configures carbon-c-relay to take carbon-relay's place in
+# routing metrics around using line-protocol.
+#
+# Two copies of carbon-c-relay are run:
+# * local-relay: takes metrics on port 1903 and forwards to all configured
+# carbon-cache processes on the local machine ('carbon-cache' list in config)
+# * frontend-relay: listen on standard port 2003 and mirror metrics to all
+# configured backends ('backends' list in config)
+#
+class graphite::carbon_c_relay( $c_relay_settings ) {
+ package { 'carbon-c-relay':
+ ensure => present,
+ }
+
+ base::service_unit { 'carbon/frontend-relay':
+ ensure => present,
+ upstart => true,
+ template_name => 'frontend-relay',
+ }
+
+ file { '/etc/carbon/frontend-relay.conf':
+ content => template('graphite/frontend-relay.conf.erb'),
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ notify => Service['carbon/frontend-relay'],
+ }
+
+ base::service_unit { 'carbon/local-relay':
+ ensure => present,
+ upstart => true,
+ template_name => 'local-relay',
+ }
+
+ file { '/etc/carbon/local-relay.conf':
+ content => template('graphite/local-relay.conf.erb'),
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ notify => Service['carbon/local-relay'],
+ }
+}
diff --git a/modules/graphite/manifests/init.pp
b/modules/graphite/manifests/init.pp
index ad00d1f..c9b36f1 100644
--- a/modules/graphite/manifests/init.pp
+++ b/modules/graphite/manifests/init.pp
@@ -1,21 +1,25 @@
# == Class: graphite
#
# Graphite is a monitoring tool that stores numeric time-series data and
-# renders graphs of this data on demand. It consists of three software
+# renders graphs of this data on demand. It consists of the following software
# components:
#
# - Carbon, a daemon that listens for time-series data
+# - Carbon-c-relay, an high-performance metric router
# - Whisper, a database library for storing time-series data
# - Graphite webapp, a webapp which renders graphs on demand
#
class graphite(
$carbon_settings,
+ $c_relay_settings,
$storage_schemas,
$storage_aggregation = {},
- $storage_dir = '/var/lib/carbon',
+ $storage_dir = '/var/lib/carbon',
$whisper_lock_writes = false,
- ) {
- package { ['graphite-carbon', 'python-whisper']: }
+) {
+ package { [ 'graphite-carbon', 'python-whisper' ]:
+ ensure => installed,
+ }
# force installation of python-twisted-core separatedly, there seem to be a
# race condition with dropin.cache generation when apt-get installing
@@ -26,6 +30,26 @@
before => Package['graphite-carbon'],
}
+ $default_c_relay_settings = {
+ 'carbon-cache' => [
+ '127.0.0.1:2103',
+ '127.0.0.1:2203',
+ '127.0.0.1:2303',
+ '127.0.0.1:2403',
+ '127.0.0.1:2503',
+ '127.0.0.1:2603',
+ '127.0.0.1:2703',
+ '127.0.0.1:2803',
+ ],
+ 'backends' => [
+ 'localhost:1903',
+ ],
+ }
+
+ class { '::graphite::carbon_c_relay':
+ c_relay_settings => merge($default_c_relay_settings,
$c_relay_settings),
+ }
+
$carbon_service_defaults = {
log_updates => false,
log_cache_hits => false,
diff --git a/modules/graphite/templates/frontend-relay.conf.erb
b/modules/graphite/templates/frontend-relay.conf.erb
new file mode 100644
index 0000000..2f95734
--- /dev/null
+++ b/modules/graphite/templates/frontend-relay.conf.erb
@@ -0,0 +1,10 @@
+cluster backends
+ forward
+ <% @c_relay_settings['backends'].each do |b| -%>
+ <%= b %>
+ <% end -%>
+ ;
+
+match *
+ send to backends
+ ;
diff --git a/modules/graphite/templates/initscripts/frontend-relay.upstart.erb
b/modules/graphite/templates/initscripts/frontend-relay.upstart.erb
new file mode 100644
index 0000000..fde487e
--- /dev/null
+++ b/modules/graphite/templates/initscripts/frontend-relay.upstart.erb
@@ -0,0 +1,15 @@
+description "carbon-c-relay instance, used to route metrics"
+
+start on stopped carbon/init
+stop on runlevel [!2345] or carbon.stop
+
+respawn
+
+limit nofile 32768 32768
+
+setuid carbon-c-relay
+setgid carbon-c-relay
+
+# it is possible there's another carbon-c-relay running on the same host, so
+# change the hostname
+exec /usr/bin/carbon-c-relay -H <%= "#{hostname}_frontend" -%> -p 2003 -f
/etc/carbon/frontend-relay.conf
diff --git a/modules/graphite/templates/initscripts/local-relay.upstart.erb
b/modules/graphite/templates/initscripts/local-relay.upstart.erb
new file mode 100644
index 0000000..3ca5694
--- /dev/null
+++ b/modules/graphite/templates/initscripts/local-relay.upstart.erb
@@ -0,0 +1,15 @@
+description "carbon-c-relay instance, used to forward to local carbon-cache
only"
+
+start on stopped carbon/init
+stop on runlevel [!2345] or carbon.stop
+
+respawn
+
+limit nofile 32768 32768
+
+setuid carbon-c-relay
+setgid carbon-c-relay
+
+# it is possible there's another carbon-c-relay running on the same host, so
+# change the hostname
+exec /usr/bin/carbon-c-relay -H <%= "#{hostname}_local" -%> -p 1903 -f
/etc/carbon/local-relay.conf
diff --git a/modules/graphite/templates/local-relay.conf.erb
b/modules/graphite/templates/local-relay.conf.erb
new file mode 100644
index 0000000..31bb7e1
--- /dev/null
+++ b/modules/graphite/templates/local-relay.conf.erb
@@ -0,0 +1,13 @@
+# simple carbon-c-relay configuration to replace carbon-relay, will
+# load-balance metrics sending among all members of the local cluster.
+
+cluster carbon-cache
+ any_of
+ <% @c_relay_settings['carbon-cache'].each do |b| -%>
+ <%= b %>
+ <% end -%>
+ ;
+
+match *
+ send to carbon-cache
+ ;
--
To view, visit https://gerrit.wikimedia.org/r/181080
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I61f687237baed6674fcc9813acc9df0ce40d4fbb
Gerrit-PatchSet: 10
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Filippo Giunchedi <fgiunch...@wikimedia.org>
Gerrit-Reviewer: Alexandros Kosiaris <akosia...@wikimedia.org>
Gerrit-Reviewer: Faidon Liambotis <fai...@wikimedia.org>
Gerrit-Reviewer: Filippo Giunchedi <fgiunch...@wikimedia.org>
Gerrit-Reviewer: Ori.livneh <o...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
