This is an automated email from the ASF dual-hosted git repository. gezapeti pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/oozie.git
The following commit(s) were added to refs/heads/master by this push: new 70f9cbd OOZIE-3305 Prometheus /metrics http endpoint for monitoring integration (qsbao via gezapeti) 70f9cbd is described below commit 70f9cbd697644d6ad9f2cfd64325e2e95f37ac01 Author: Gezapeti Cseh <gezap...@apache.org> AuthorDate: Sat Jan 18 09:07:22 2020 +0100 OOZIE-3305 Prometheus /metrics http endpoint for monitoring integration (qsbao via gezapeti) --- .../org/apache/oozie/client/rest/RestConstants.java | 2 ++ core/pom.xml | 10 ++++++++++ .../java/org/apache/oozie/servlet/BaseAdminServlet.java | 5 +++++ .../java/org/apache/oozie/servlet/V0AdminServlet.java | 5 +++++ .../java/org/apache/oozie/servlet/V1AdminServlet.java | 9 ++++++++- .../java/org/apache/oozie/servlet/V2AdminServlet.java | 17 +++++++++++++++++ .../org/apache/oozie/util/MetricsInstrumentation.java | 5 +++++ .../java/org/apache/oozie/servlet/TestAdminServlet.java | 16 ++++++++++++++++ docs/src/site/markdown/AG_Install.md | 11 +++++++++++ pom.xml | 12 ++++++++++++ release-log.txt | 1 + 11 files changed, 92 insertions(+), 1 deletion(-) diff --git a/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java b/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java index 1353786..fcfd43f 100644 --- a/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java +++ b/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java @@ -165,6 +165,8 @@ public interface RestConstants { String ADMIN_METRICS_RESOURCE = "metrics"; + String ADMIN_PROMETHEUS_RESOURCE = "prometheus"; + String OOZIE_ERROR_CODE = "oozie-error-code"; String OOZIE_ERROR_MESSAGE = "oozie-error-message"; diff --git a/core/pom.xml b/core/pom.xml index fd00ce7..649cdd0 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -99,6 +99,16 @@ </dependency> <dependency> + <groupId>io.prometheus</groupId> + <artifactId>simpleclient_dropwizard</artifactId> + </dependency> + + <dependency> + <groupId>io.prometheus</groupId> + <artifactId>simpleclient_common</artifactId> + </dependency> + + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> </dependency> diff --git a/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java b/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java index 0b873bc..49f5fbb 100644 --- a/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java @@ -186,6 +186,9 @@ public abstract class BaseAdminServlet extends JsonRestServlet { else if (resource.equals(RestConstants.ADMIN_METRICS_RESOURCE)) { sendMetricsResponse(response); } + else if (resource.equals(RestConstants.ADMIN_PROMETHEUS_RESOURCE)) { + sendPrometheusResponse(response); + } } private String schedulePurgeCommand(HttpServletRequest request) throws XServletException { @@ -475,4 +478,6 @@ public abstract class BaseAdminServlet extends JsonRestServlet { protected abstract Map<String, String> getOozieURLs() throws XServletException; protected abstract void sendMetricsResponse(HttpServletResponse response) throws IOException, XServletException; + + protected abstract void sendPrometheusResponse(HttpServletResponse response) throws IOException, XServletException; } diff --git a/core/src/main/java/org/apache/oozie/servlet/V0AdminServlet.java b/core/src/main/java/org/apache/oozie/servlet/V0AdminServlet.java index 6cb48c7..f387803 100644 --- a/core/src/main/java/org/apache/oozie/servlet/V0AdminServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/V0AdminServlet.java @@ -101,4 +101,9 @@ public class V0AdminServlet extends BaseAdminServlet { protected void sendMetricsResponse(HttpServletResponse response) throws IOException, XServletException { throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0302, "Not supported in v0"); } + + @Override + protected void sendPrometheusResponse(HttpServletResponse response) throws IOException, XServletException { + throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0302, "Not supported in v0"); + } } diff --git a/core/src/main/java/org/apache/oozie/servlet/V1AdminServlet.java b/core/src/main/java/org/apache/oozie/servlet/V1AdminServlet.java index 812b385..7134657 100644 --- a/core/src/main/java/org/apache/oozie/servlet/V1AdminServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/V1AdminServlet.java @@ -40,7 +40,7 @@ public class V1AdminServlet extends BaseAdminServlet { private static final long serialVersionUID = 1L; private static final String INSTRUMENTATION_NAME = "v1admin"; - private static final ResourceInfo RESOURCES_INFO[] = new ResourceInfo[14]; + private static final ResourceInfo RESOURCES_INFO[] = new ResourceInfo[15]; static { RESOURCES_INFO[0] = new ResourceInfo(RestConstants.ADMIN_STATUS_RESOURCE, Arrays.asList("PUT", "GET"), @@ -71,6 +71,8 @@ public class V1AdminServlet extends BaseAdminServlet { RESOURCES_INFO[12] = new ResourceInfo(RestConstants.ADMIN_METRICS_RESOURCE, Arrays.asList("GET"), Collections.EMPTY_LIST); RESOURCES_INFO[13] = new ResourceInfo(RestConstants.ADMIN_PURGE, Arrays.asList("PUT"), Collections.EMPTY_LIST); + RESOURCES_INFO[14] = new ResourceInfo(RestConstants.ADMIN_PROMETHEUS_RESOURCE, Arrays.asList("GET"), + Collections.EMPTY_LIST); } protected V1AdminServlet(String name) { @@ -142,4 +144,9 @@ public class V1AdminServlet extends BaseAdminServlet { protected void sendMetricsResponse(HttpServletResponse response) throws IOException, XServletException { throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0302, "Not supported in v1"); } + + @Override + protected void sendPrometheusResponse(HttpServletResponse response) throws IOException, XServletException { + throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0302, "Not supported in v1"); + } } diff --git a/core/src/main/java/org/apache/oozie/servlet/V2AdminServlet.java b/core/src/main/java/org/apache/oozie/servlet/V2AdminServlet.java index 7eadbe7..9617fa6 100644 --- a/core/src/main/java/org/apache/oozie/servlet/V2AdminServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/V2AdminServlet.java @@ -19,12 +19,15 @@ package org.apache.oozie.servlet; import java.io.IOException; +import java.io.Writer; import java.util.Map; import java.util.Properties; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.exporter.common.TextFormat; import org.apache.hadoop.conf.Configuration; import org.apache.oozie.ErrorCode; import org.apache.oozie.client.rest.JMSConnectionInfoBean; @@ -120,6 +123,20 @@ public class V2AdminServlet extends V1AdminServlet { } @Override + protected void sendPrometheusResponse(HttpServletResponse response) throws IOException, XServletException { + if (metricsInstrumentation != null) { + response.setStatus(HttpServletResponse.SC_OK); + response.setContentType(TextFormat.CONTENT_TYPE_004); + Writer writer = response.getWriter(); + TextFormat.write004(writer, CollectorRegistry.defaultRegistry.metricFamilySamples()); + writer.flush(); + } else { + response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE, "MetricsInstrumentationService is not running"); + } + } + + @Override protected void sendInstrumentationResponse(HttpServletResponse response, Instrumentation instr) throws IOException, XServletException { if (metricsInstrumentation == null) { diff --git a/core/src/main/java/org/apache/oozie/util/MetricsInstrumentation.java b/core/src/main/java/org/apache/oozie/util/MetricsInstrumentation.java index 98af977..d5c1028 100644 --- a/core/src/main/java/org/apache/oozie/util/MetricsInstrumentation.java +++ b/core/src/main/java/org/apache/oozie/util/MetricsInstrumentation.java @@ -38,6 +38,8 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import info.ganglia.gmetric4j.gmetric.GMetric; +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.dropwizard.DropwizardExports; import org.apache.oozie.service.ConfigurationService; import java.io.IOException; @@ -202,6 +204,7 @@ public class MetricsInstrumentation extends Instrumentation { jmxReporter = JmxReporter.forRegistry(metricRegistry).build(); jmxReporter.start(); } + CollectorRegistry.defaultRegistry.register(new DropwizardExports(metricRegistry)); } /** @@ -229,6 +232,8 @@ public class MetricsInstrumentation extends Instrumentation { if (jmxReporter != null) { jmxReporter.stop(); } + + CollectorRegistry.defaultRegistry.clear(); } /** diff --git a/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java b/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java index e3bb4c2..d966566 100644 --- a/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java +++ b/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java @@ -18,6 +18,7 @@ package org.apache.oozie.servlet; +import io.prometheus.client.exporter.common.TextFormat; import org.apache.commons.io.IOUtils; import org.apache.oozie.client.rest.JsonTags; import org.apache.oozie.client.rest.RestConstants; @@ -133,6 +134,21 @@ public class TestAdminServlet extends DagServletTestCase { }); } + public void testPrometheus() throws Exception { + runTest("/v2/admin/*", V2AdminServlet.class, IS_SECURITY_ENABLED, new Callable<Void>() { + public Void call() throws Exception { + URL url = createURL(RestConstants.ADMIN_PROMETHEUS_RESOURCE, Collections.EMPTY_MAP); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertEquals(conn.getHeaderField("content-type"), TextFormat.CONTENT_TYPE_004); + String string = IOUtils.toString(conn.getInputStream(), StandardCharsets.UTF_8); + assertTrue("Prometheus metrics does not show up", string.contains("jobstatus_SUCCEEDED")); + return null; + } + }); + } + public void testSafeMode() throws Exception { runTest(new String[]{"/v0/admin/*", "/v0/job/*"}, new Class[]{V0AdminServlet.class, V0JobServlet.class}, IS_SECURITY_ENABLED, new Callable<Void>() { diff --git a/docs/src/site/markdown/AG_Install.md b/docs/src/site/markdown/AG_Install.md index 8996e8a..1bcdb82 100644 --- a/docs/src/site/markdown/AG_Install.md +++ b/docs/src/site/markdown/AG_Install.md @@ -1093,6 +1093,17 @@ in oozie-site.xml : </property>> ``` +We also have a prometheus compatible metrics endpoint `admin/prometheus`, where all the available metrics are exposed in prometheus +format. Here is an example prometheus scrape config: + +```yaml +scrape_configs: + - job_name: 'oozie' + metrics_path: '/oozie/v2/admin/prometheus' + static_configs: + - targets: ['oozie:11000'] +``` + <a name="HA"></a> ### High Availability (HA) diff --git a/pom.xml b/pom.xml index 8c44958..36db31d 100644 --- a/pom.xml +++ b/pom.xml @@ -1569,6 +1569,18 @@ <version>${dropwizard.metrics.version}</version> </dependency> + <dependency> + <groupId>io.prometheus</groupId> + <artifactId>simpleclient_dropwizard</artifactId> + <version>0.8.0</version> + </dependency> + + <dependency> + <groupId>io.prometheus</groupId> + <artifactId>simpleclient_common</artifactId> + <version>0.8.0</version> + </dependency> + <!-- Draw runtime DAG --> <dependency> <groupId>guru.nidi</groupId> diff --git a/release-log.txt b/release-log.txt index cf0f505..ca9042e 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.3.0 release (trunk - unreleased) +OOZIE-3305 Prometheus /metrics http endpoint for monitoring integration (qsbao via gezapeti) OOZIE-3575 Add credential support for cloud file systems (matijhs via gezapeti) OOZIE-3579 [docs] Fix typos in coordinator documentation (qsbao via asalamon74) OOZIE-3066 Possibility to set retry-max globally (qsbao via asalamon74)