Filippo Giunchedi has submitted this change and it was merged. Change subject: configure additional Cassandra metric alerts ......................................................................
configure additional Cassandra metric alerts Bug: T101764 Change-Id: I400e7e6a93fcb91872ff396cba7fc3471645e6bf --- M manifests/role/restbase.pp 1 file changed, 82 insertions(+), 2 deletions(-) Approvals: Filippo Giunchedi: Verified; Looks good to me, approved GWicke: Looks good to me, but someone else must approve diff --git a/manifests/role/restbase.pp b/manifests/role/restbase.pp index 588f51d..ac3c598 100644 --- a/manifests/role/restbase.pp +++ b/manifests/role/restbase.pp @@ -22,7 +22,7 @@ class role::restbase::alerts { monitoring::graphite_threshold { 'restbase_request_5xx_rate': - description => 'RESTBase req/s returning 5xx', + description => 'RESTBase req/s returning 5xx (http://grafana.wikimedia.org/#/dashboard/db/restbase)', metric => 'transformNull(restbase.v1_page_html_-title-_-revision--_tid-.GET.5xx.sample_rate, 0)', from => '10min', warning => '1', # 1 5xx/s @@ -32,7 +32,7 @@ } monitoring::graphite_threshold { 'restbase_html_storage_hit_latency': - description => 'RESTBase HTML storage load mean latency ms', + description => 'RESTBase HTML storage load mean latency ms (http://grafana.wikimedia.org/#/dashboard/db/restbase)', metric => 'movingMedian(restbase.sys_key-rev-value_-bucket-_-key--_revision--_tid-.GET.2xx.mean, 15)', from => '10min', warning => '25', # 25ms @@ -40,4 +40,84 @@ percentage => '50', contact_group => 'team-services', } + + monitoring::graphite_threshold { 'restbase_html_storage_hit_latency_99p': + description => 'RESTBase HTML storage load 99p latency ms (http://grafana.wikimedia.org/#/dashboard/db/restbase)', + metric => 'movingMedian(restbase.sys_key-rev-value_-bucket-_-key--_revision--_tid-.GET.2xx.p99, 15)', + from => '10min', + warning => '1500', # 1.5s + critical => '3000', # 3s + percentage => '50', + contact_group => 'team-services', + } + + monitoring::graphite_threshold { 'restbase_cassandra_highest_storage_exceptions': + description => 'RESTBase Cassandra highest storage exceptions (http://grafana.wikimedia.org/#/dashboard/db/restbase-cassandra-storage)', + metric => 'highestMax(nonNegativeDerivative(cassandra.restbase10*.org.apache.cassandra.metrics.Storage.Exceptions.count), 1)', + from => '10min', + warning => '5', + critical => '10', + percentage => '50', + contact_group => 'team-services', + } + + monitoring::graphite_threshold { 'restbase_cassandra_highest_total_hints': + description => 'RESTBase Cassandra highest total hints (http://grafana.wikimedia.org/#/dashboard/db/restbase-cassandra-storage)', + metric => 'highestMax(nonNegativeDerivative(cassandra.restbase10*.org.apache.cassandra.metrics.Storage.TotalHints.count), 1)', + from => '10min', + warning => '600', + critical => '1000', + percentage => '50', + contact_group => 'team-services', + } + + monitoring::graphite_threshold { 'restbase_cassandra_highest_pending_compactions': + description => 'RESTBase Cassandra highest pending compactions (http://grafana.wikimedia.org/#/dashboard/db/restbase-cassandra-compaction)', + metric => 'highestMax(cassandra.restbase10*.org.apache.cassandra.metrics.Compaction.PendingTasks.value, 1)', + from => '10min', + warning => '100', + critical => '400', + percentage => '50', + contact_group => 'team-services', + } + + monitoring::graphite_threshold { 'restbase_cassandra_highest_sstables_per_read': + description => 'RESTBase Cassandra highest SSTables per-read (http://grafana.wikimedia.org/#/dashboard/db/restbase-cassandra-cf-sstables-per-read)', + metric => 'highestMax(cassandra.restbase10*.org.apache.cassandra.metrics.ColumnFamily.all.SSTablesPerReadHistogram.99percentile, 1)', + from => '10min', + warning => '6', + critical => '10', + percentage => '50', + contact_group => 'team-services', + } + + monitoring::graphite_threshold { 'restbase_cassandra_highest_tombstones_scanned': + description => 'RESTBase Cassandra highest tombstones scanned (http://grafana.wikimedia.org/#/dashboard/db/restbase-cassandra-cf-tombstones-scanned)', + metric => 'highestMax(cassandra.restbase10*.org.apache.cassandra.metrics.ColumnFamily.all.TombstoneScannedHistogram.99percentile, 1)', + from => '10min', + warning => '1000', + critical => '1500', + percentage => '50', + contact_group => 'team-services', + } + + monitoring::graphite_threshold { 'restbase_cassandra_highest_pending_internal': + description => 'RESTBase Cassandra highest pending internal thread pool tasks (http://grafana.wikimedia.org/#/dashboard/db/restbase-cassandra-thread-pools)', + metric => 'highestMax(cassandra.restbase10*.org.apache.cassandra.metrics.ThreadPools.internal.*.PendingTasks.value, 1)', + from => '10min', + warning => '500', + critical => '1000', + percentage => '50', + contact_group => 'team-services', + } + + monitoring::graphite_threshold { 'restbase_cassandra_highest_dropped_messages': + description => 'RESTBase Cassandra highest dropped message rate (http://grafana.wikimedia.org/#/dashboard/db/restbase-cassandra-dropped-messages)', + metric => 'highestMax(cassandra.restbase10*.org.apache.cassandra.metrics.DroppedMessage.*.Dropped.1MinuteRate, 1)', + from => '10min', + warning => '50', + critical => '100', + percentage => '50', + contact_group => 'team-services', + } } -- To view, visit https://gerrit.wikimedia.org/r/218408 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I400e7e6a93fcb91872ff396cba7fc3471645e6bf Gerrit-PatchSet: 8 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Eevans <eev...@wikimedia.org> Gerrit-Reviewer: Eevans <eev...@wikimedia.org> Gerrit-Reviewer: Filippo Giunchedi <fgiunch...@wikimedia.org> Gerrit-Reviewer: GWicke <gwi...@wikimedia.org> Gerrit-Reviewer: Mobrovac <mobro...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits