Jerald commented on code in PR #2402: URL: https://github.com/apache/activemq/pull/2402#discussion_r3816151300
########## activemq-prometheus/src/main/java/org/apache/activemq/prometheus/PrometheusMetricsServlet.java: ########## @@ -0,0 +1,243 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.activemq.prometheus; + +import java.io.IOException; +import java.io.PrintWriter; +import java.lang.management.ManagementFactory; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import javax.management.MBeanServer; +import javax.management.ObjectName; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PrometheusMetricsServlet extends HttpServlet { + + private static final long serialVersionUID = 1L; + private static final Logger LOG = LoggerFactory.getLogger(PrometheusMetricsServlet.class); + private static final String CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8"; + + // Metrics can be easily extended by adding them here + private static final MetricDefinition[] BROKER_METRICS = { + new MetricDefinition("connections", "Current number of connections", "CurrentConnectionsCount", MetricType.GAUGE), + new MetricDefinition("connections_total", "Total connections since last start", "TotalConnectionsCount", MetricType.COUNTER), + new MetricDefinition("messages_enqueued_total", "Total messages enqueued since last start", "TotalEnqueueCount", MetricType.COUNTER), + new MetricDefinition("messages_dequeued_total", "Total messages dequeued since last start", "TotalDequeueCount", MetricType.COUNTER), + new MetricDefinition("consumers", "Current number of consumers", "TotalConsumerCount", MetricType.GAUGE), + new MetricDefinition("producers", "Current number of producers", "TotalProducerCount", MetricType.GAUGE), + new MetricDefinition("messages", "Current number of messages across all destinations", "TotalMessageCount", MetricType.GAUGE), + new MetricDefinition("memory_percent_usage", "Percent (0-100) of memory limit used", "MemoryPercentUsage", MetricType.GAUGE), + new MetricDefinition("memory_limit_bytes", "Memory limit in bytes", "MemoryLimit", MetricType.GAUGE), + new MetricDefinition("store_percent_usage", "Percent (0-100) of store limit used", "StorePercentUsage", MetricType.GAUGE), + new MetricDefinition("store_limit_bytes", "Store limit in bytes", "StoreLimit", MetricType.GAUGE), + new MetricDefinition("temp_percent_usage", "Percent (0-100) of temp limit used", "TempPercentUsage", MetricType.GAUGE), + new MetricDefinition("temp_limit_bytes", "Temp limit in bytes", "TempLimit", MetricType.GAUGE), + new MetricDefinition("uptime_milliseconds", "Broker uptime in milliseconds", "UptimeMillis", MetricType.GAUGE), + new MetricDefinition("queues", "Number of queues on the broker", "TotalQueuesCount", MetricType.GAUGE), + new MetricDefinition("topics", "Number of topics on the broker", "TotalTopicsCount", MetricType.GAUGE), + new MetricDefinition("job_scheduler_store_percent_usage", "Percent (0-100) of job scheduler store limit used", "JobSchedulerStorePercentUsage", MetricType.GAUGE), + new MetricDefinition("job_scheduler_store_limit_bytes", "Job scheduler store limit in bytes", "JobSchedulerStoreLimit", MetricType.GAUGE) + }; + + private static final MetricDefinition[] DESTINATION_METRICS = { + new MetricDefinition("messages", "Number of messages in this destination", "QueueSize", MetricType.GAUGE), + new MetricDefinition("enqueued_total", "Total messages enqueued to this destination since last start", "EnqueueCount", MetricType.COUNTER), + new MetricDefinition("dequeued_total", "Total messages dequeued from destination since last start", "DequeueCount", MetricType.COUNTER), + new MetricDefinition("dispatched_total", "Total messages dispatched from destination since last start", "DispatchCount", MetricType.COUNTER), + new MetricDefinition("message_inflight_count", "Messages dispatched but not acknowledged", "InFlightCount", MetricType.GAUGE), + new MetricDefinition("expired_total", "Total messages expired since last start", "ExpiredCount", MetricType.COUNTER), + new MetricDefinition("consumers", "Number of consumers", "ConsumerCount", MetricType.GAUGE), + new MetricDefinition("producers", "Number of producers", "ProducerCount", MetricType.GAUGE), + new MetricDefinition("memory_percent_usage", "Percent (0-100) of destination memory limit used", "MemoryPercentUsage", MetricType.GAUGE), + new MetricDefinition("memory_limit_bytes", "Memory limit for this destination in bytes", "MemoryLimit", MetricType.GAUGE), + new MetricDefinition("memory_usage_bytes", "Memory used by this destination in bytes", "MemoryUsageByteCount", MetricType.GAUGE), + new MetricDefinition("store_message_size_bytes", "Store message size in bytes", "StoreMessageSize", MetricType.GAUGE), + new MetricDefinition("average_enqueue_time_milliseconds", "Average time (since last start) messages waited before dispatch", "AverageEnqueueTime", MetricType.GAUGE) + }; + + @Override + protected void doGet(final HttpServletRequest request, final HttpServletResponse response) throws IOException { + final boolean perObject = request != null && "true".equalsIgnoreCase(request.getParameter("per_object")); + final MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer(); + + // If the mBeanServer is unavailable nothing useful can be produced, so return a 500 + final Set<ObjectName> brokers; + final Set<ObjectName> queues; + final Set<ObjectName> topics; + try { + brokers = mBeanServer.queryNames(new ObjectName("org.apache.activemq:type=Broker,brokerName=*"), null); + if (perObject) { + // Scraping destinations on brokers with many queues or topics can be expensive. + queues = mBeanServer.queryNames(new ObjectName( + "org.apache.activemq:type=Broker,brokerName=*,destinationType=Queue,destinationName=*"), null); + topics = mBeanServer.queryNames(new ObjectName( + "org.apache.activemq:type=Broker,brokerName=*,destinationType=Topic,destinationName=*"), null); + } else { + queues = Collections.emptySet(); + topics = Collections.emptySet(); + } + } catch (final Exception exception) { + LOG.warn("Prometheus scrape failed while querying broker MBeans", exception); + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Metrics collection failed"); + return; + } + + // Stream metrics to avoid keeping the response body in memory + // Failed items are skipped or default to 0 (when not possible) to allow the scrape to continue + response.setContentType(CONTENT_TYPE); + response.setStatus(HttpServletResponse.SC_OK); + final PrintWriter writer = response.getWriter(); + writeBrokerMetrics(mBeanServer, writer, brokers); + if (perObject) { + writeDestinationMetrics(mBeanServer, writer, "Queue", queues); + writeDestinationMetrics(mBeanServer, writer, "Topic", topics); + } + writer.flush(); + } + + private void writeBrokerMetrics(final MBeanServer mBeanServer, final PrintWriter writer, final Set<ObjectName> brokers) { + // In case there is a network of brokers, only use the local one + // Unreadable names are skipped + final Map<ObjectName, String> identified = new LinkedHashMap<>(); + for (final ObjectName broker : brokers) { + final String brokerName = resolveStringAttribute(mBeanServer, broker, "BrokerName"); + if (brokerName != null) { + identified.put(broker, "broker=\"" + sanitizeLabel(brokerName) + "\""); + } + } + + for (final MetricDefinition metric : BROKER_METRICS) { + final String metricName = "activemq_broker_" + metric.name; + writeMetadata(writer, metricName, metric); + for (final Map.Entry<ObjectName, String> entry : identified.entrySet()) { + writeSample(writer, metricName, entry.getValue(), getNumber(mBeanServer, entry.getKey(), metric.attribute)); + } + } + } + + private void writeDestinationMetrics(final MBeanServer mBeanServer, final PrintWriter writer, final String type, + final Set<ObjectName> destinations) { + final String typeLower = type.toLowerCase(); + + for (final MetricDefinition metric : DESTINATION_METRICS) { + final String metricName = "activemq_" + typeLower + "_" + metric.name; + writeMetadata(writer, metricName, metric.withFormattedHelp(typeLower)); + for (final ObjectName destination : destinations) { + final String brokerName = sanitizeLabel(destination.getKeyProperty("brokerName")); + final String destinationName = sanitizeLabel(destination.getKeyProperty("destinationName")); + final String labels = String.format("broker=\"%s\",destination=\"%s\"", brokerName, destinationName); + writeSample(writer, metricName, labels, getNumber(mBeanServer, destination, metric.attribute)); + } Review Comment: I'm realizing there's an alternate way to encode these metrics that could be used. Right now, the destination type (queue/topic) is encoded in the metric name itself, despite all destinations sharing a set of emitted metrics (though queues and topics do have a few metrics unique to themselves). But because the underlying metrics are mostly overlapping, the destination type can also be disambiguated by a label instead of a unique metric name. For example, the current format for a destination metric name is something like this: `activemq_$destType_$metricName` (`activemq_queue_connections`). And it would have labels like this: `broker=$brokerName`, `destination=$destName` (`broker="foo",destination="FOO.BAR"`). But it should be near equivalent to use a destination name like `activemq_destination_$metricName` and then have labels `broker=$brokerName`, `destinationType=$destType`, and `destination=$destName`. Both these approaches are valid (as far as I'm aware), so it's mostly a matter of taste. One upside that I see is having destination type as a label more closely aligns with how the MBeans look. If someone were to convert a tool from using MBeans for metrics and move to Prometheus, I'd expect this approach would be an easier transition (but whether or not that matters is a separate question). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected] For further information, visit: https://activemq.apache.org/contact
