Giuseppe Lavagetto has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/394966 )
Change subject: role::puppetmaster::puppetdb: add Prometheus monitoring for puppetdb ...................................................................... role::puppetmaster::puppetdb: add Prometheus monitoring for puppetdb This change adds only a subset of the Mbeans available since using the JMX agent's whitelist turned out to be more perfomant. The puppetdb's jvm options are now configurable via hiera to allow a more friendly labs deployment. Change-Id: I58f036e85edb98ef4170580d093c42f0bc8ef786 --- M hieradata/role/common/puppetmaster/puppetdb.yaml A modules/profile/files/puppetmaster/puppetdb/jvm_prometheus_puppetdb_jmx_exporter.yaml M modules/profile/manifests/puppetdb.pp M modules/puppetdb/manifests/app.pp M modules/puppetdb/templates/puppetdb.service.erb M modules/puppetmaster/manifests/puppetdb.pp M modules/role/manifests/puppetmaster/puppetdb.pp 7 files changed, 36 insertions(+), 10 deletions(-) Approvals: Giuseppe Lavagetto: Looks good to me, approved jenkins-bot: Verified diff --git a/hieradata/role/common/puppetmaster/puppetdb.yaml b/hieradata/role/common/puppetmaster/puppetdb.yaml index fd8c9a8..b3be4d0 100644 --- a/hieradata/role/common/puppetmaster/puppetdb.yaml +++ b/hieradata/role/common/puppetmaster/puppetdb.yaml @@ -10,3 +10,4 @@ cidr: 10.192.16.184/32 profile::puppetdb::master: nitrogen.eqiad.wmnet profile::puppetdb::slaves: [nihal.codfw.wmnet] +puppetmaster::puppetdb::jvm_opts: '-Xmx6g' diff --git a/modules/profile/files/puppetmaster/puppetdb/jvm_prometheus_puppetdb_jmx_exporter.yaml b/modules/profile/files/puppetmaster/puppetdb/jvm_prometheus_puppetdb_jmx_exporter.yaml new file mode 100644 index 0000000..6ea2bc3 --- /dev/null +++ b/modules/profile/files/puppetmaster/puppetdb/jvm_prometheus_puppetdb_jmx_exporter.yaml @@ -0,0 +1,8 @@ +--- +lowercaseOutputLabelNames: true +lowercaseOutputName: false +whitelistObjectNames: + - 'com.puppetlabs.puppetdb.command:type=global,name=*' + - 'com.puppetlabs.puppetdb.command:type=replace facts.3,name=*' + - 'com.puppetlabs.puppetdb.http.server:type=/v3/commands,name=*' + - 'com.puppetlabs.puppetdb.http.server:type=/v3/nodes,name=*' \ No newline at end of file diff --git a/modules/profile/manifests/puppetdb.pp b/modules/profile/manifests/puppetdb.pp index b8717af..7c9bea8 100644 --- a/modules/profile/manifests/puppetdb.pp +++ b/modules/profile/manifests/puppetdb.pp @@ -1,13 +1,32 @@ class profile::puppetdb( $master = hiera('profile::puppetdb::master'), - $puppetmasters = hiera('puppetmaster::servers') + $puppetmasters = hiera('puppetmaster::servers'), + $jvm_opts = hiera('profile::puppetdb::jvm_opts', '-Xmx4G'), + $prometheus_nodes = hiera('prometheus_nodes'), ) { + # Prometheus JMX agent for the Puppetdb's JVM + $jmx_exporter_config_file = '/etc/puppetdb/jvm_prometheus_puppetdb_jmx_exporter.yaml' + $prometheus_jmx_exporter_port = 9400 + $prometheus_java_opts = "-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=${::ipaddress}:${prometheus_jmx_exporter_port}:${jmx_exporter_config_file}" # The JVM heap size has been raised to 6G for T170740 class { '::puppetmaster::puppetdb': - master => $master, - heap_size => '6G', + master => $master, + jvm_opts => "${jvm_opts} ${prometheus_java_opts}", } + + # Export JMX metrics to prometheus + profile::prometheus::jmx_exporter { "puppetdb_${::hostname}": + hostname => $::hostname, + port => $prometheus_jmx_exporter_port, + prometheus_nodes => $prometheus_nodes, + config_file => $jmx_exporter_config_file, + source => 'puppet:///modules/profile/puppetmaster/puppetdb/jvm_prometheus_puppetdb_jmx_exporter.yaml', + } + + + # Firewall rules + # Only the TLS-terminating nginx proxy will be exposed $puppetmasters_ferm = inline_template('<%= @puppetmasters.values.flatten(1).map { |p| p[\'worker\'] }.sort.join(\' \')%>') diff --git a/modules/puppetdb/manifests/app.pp b/modules/puppetdb/manifests/app.pp index a012ee1..1b64d57 100644 --- a/modules/puppetdb/manifests/app.pp +++ b/modules/puppetdb/manifests/app.pp @@ -11,7 +11,7 @@ $db_user='puppetdb', $db_password=undef, $perform_gc=false, - $heap_size='4G', + $jvm_opts='-Xmx4G', $bind_ip=undef, $ssldir=puppet_ssldir(), $command_processing_threads=16, diff --git a/modules/puppetdb/templates/puppetdb.service.erb b/modules/puppetdb/templates/puppetdb.service.erb index cef26bd..3bcd7ba 100644 --- a/modules/puppetdb/templates/puppetdb.service.erb +++ b/modules/puppetdb/templates/puppetdb.service.erb @@ -6,8 +6,8 @@ Group=puppetdb Environment=CONFIG=/etc/puppetdb/conf.d ExecStartPre=/bin/bash -c "test -e /var/log/puppetdb/puppetdb-oom.hprof && mv /var/log/puppetdb/puppetdb-oom.hprof /var/log/puppetdb/puppetdb-oom.hprof.prev || exit 0" -ExecStart=/usr/bin/java -Xmx<%= @heap_size %> -XX:+ExitOnOutOfMemoryError \ --XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/puppetdb/puppetdb-oom.hprof -Djava.security.egd=file:/dev/urandom \ +ExecStart=/usr/bin/java <%= @jvm_opts %> \ +-XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/puppetdb/puppetdb-oom.hprof -Djava.security.egd=file:/dev/urandom \ -cp /usr/share/puppetdb/puppetdb.jar clojure.main -m com.puppetlabs.puppetdb.core services -c ${CONFIG} ExecReload=/bin/kill -HUP $MAINPID Restart=always diff --git a/modules/puppetmaster/manifests/puppetdb.pp b/modules/puppetmaster/manifests/puppetdb.pp index 685bdd4..5f88710 100644 --- a/modules/puppetmaster/manifests/puppetdb.pp +++ b/modules/puppetmaster/manifests/puppetdb.pp @@ -5,7 +5,7 @@ $master, $port = 443, $jetty_port = 8080, - $heap_size = '4G', + $jvm_opts ='-Xmx4G', ) { requires_os('debian >= jessie') @@ -38,6 +38,6 @@ db_ro_host => $::fqdn, db_password => $puppetdb_pass, perform_gc => ($master == $::fqdn), # only the master must perform GC - heap_size => $heap_size, + jvm_opts => $jvm_opts, } } diff --git a/modules/role/manifests/puppetmaster/puppetdb.pp b/modules/role/manifests/puppetmaster/puppetdb.pp index 71abe47..4f46166 100644 --- a/modules/role/manifests/puppetmaster/puppetdb.pp +++ b/modules/role/manifests/puppetmaster/puppetdb.pp @@ -5,8 +5,6 @@ include ::profile::puppetdb::database include ::profile::puppetdb - # Monitor the Postgresql replication lag - system::role { "puppetmaster::puppetdb (postgres ${::profile::puppetdb::database::role})": ensure => 'present', description => 'PuppetDB server', -- To view, visit https://gerrit.wikimedia.org/r/394966 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I58f036e85edb98ef4170580d093c42f0bc8ef786 Gerrit-PatchSet: 14 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Elukey <ltosc...@wikimedia.org> Gerrit-Reviewer: Alexandros Kosiaris <akosia...@wikimedia.org> Gerrit-Reviewer: Elukey <ltosc...@wikimedia.org> Gerrit-Reviewer: Filippo Giunchedi <fgiunch...@wikimedia.org> Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org> Gerrit-Reviewer: Herron <kher...@wikimedia.org> Gerrit-Reviewer: Volans <rcocci...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits